def setup_class(cls): cls.registry = Registry() cls.registry.prepare() cls.context_obj = AttribSafeContextObj() cls.registry.register(pylons.c, cls.context_obj) cls.app_globals_obj = app_globals.Globals() cls.registry.register(pylons.g, cls.app_globals_obj) cls.request_obj = Request(dict(HTTP_HOST="nohost")) cls.registry.register(pylons.request, cls.request_obj) cls.translator_obj = MockTranslator() cls.registry.register(pylons.translator, cls.translator_obj) cls.buffet = pylons.templating.Buffet('genshi', template_root='ckan.templates') cls.registry.register(pylons.buffet, cls.buffet) cls.registry.register(pylons.response, Response()) mapper = make_map() cls.registry.register(pylons.url, URLGenerator(mapper, {})) cls.registry.register(pylons.session, TestSession()) # Templates often want to find out the request's routes info, so put # some dummy values into the routes_dict, so the templates that do # this don't cause an exception. pylons.request.environ.update({ 'pylons.routes_dict': { 'action': 'test-action', 'controller': 'test-package::', } })
def mock_pylons(): from paste.registry import Registry import pylons from pylons.util import AttribSafeContextObj import ckan.lib.app_globals as app_globals from ckan.lib.cli import MockTranslator from ckan.config.routing import make_map from pylons.controllers.util import Request, Response from routes.util import URLGenerator class TestPylonsSession(dict): last_accessed = None def save(self): pass registry = Registry() registry.prepare() context_obj = AttribSafeContextObj() registry.register(pylons.c, context_obj) app_globals_obj = app_globals.app_globals registry.register(pylons.g, app_globals_obj) request_obj = Request(dict(HTTP_HOST="nohost", REQUEST_METHOD="GET")) registry.register(pylons.request, request_obj) translator_obj = MockTranslator() registry.register(pylons.translator, translator_obj) registry.register(pylons.response, Response()) mapper = make_map() registry.register(pylons.url, URLGenerator(mapper, {})) registry.register(pylons.session, TestPylonsSession())
def register_translator(): # Workaround until core translation function defaults to Flask from paste.registry import Registry from ckan.lib.cli import MockTranslator from pylons import translator registry = Registry() registry.prepare() registry.register(translator, MockTranslator())
def setup(): # Register a mock translator instead of having ckan domain translations defined try: from ckan.lib.cli import MockTranslator patcher = mock.patch('pylons.i18n.translation._get_translator', return_value=MockTranslator()) patcher.start() except ImportError: # if Pylons isn't present, we don't need it pass
def register_translator(): # Register a translator in this thread so that # the _() functions in logic layer can work from paste.registry import Registry from pylons import translator from ckan.lib.cli import MockTranslator global registry registry = Registry() registry.prepare() global translator_obj translator_obj = MockTranslator() registry.register(translator, translator_obj)
def load_config(path): import paste.deploy conf = paste.deploy.appconfig('config:' + path) import ckan, pylons ckan.config.environment.load_environment(conf.global_conf, conf.local_conf) from ckan.lib.cli import MockTranslator from paste.registry import Registry registry = Registry() registry.prepare() translator_obj = MockTranslator() registry.register(pylons.translator, translator_obj)
def command(dry_run=False): from ckan import model from ckanext.dgu.lib.resource_formats import match from running_stats import StatsList # Register a translator in this thread so that # the _() functions in logic layer can work from ckan.lib.cli import MockTranslator registry = Registry() registry.prepare() translator_obj = MockTranslator() registry.register(translator, translator_obj) if not dry_run: model.repo.new_revision() # Add canonised formats to map for format_ in res_type_map.keys(): res_type_map[canonise(format_)] = res_type_map[format_] log.info('Tidying resource types') stats = StatsList() res_query = model.Session.query(model.Resource) log.info('Tidying formats. Resources=%i Canonised formats=%i', res_query.count(), len(set(res_type_map.values()))) for res in res_query: canonised_fmt = canonise(res.format or '') if canonised_fmt in res_type_map: improved_fmt = res_type_map[canonised_fmt] else: improved_fmt = tidy(res.format) match_ = match(improved_fmt) if match_: improved_fmt = match_ if (improved_fmt or '') != (res.format or ''): if not dry_run: res.format = improved_fmt stats.add(improved_fmt, res.format) else: stats.add('No change', res.format) if not dry_run: model.repo.commit_and_remove() log.info('Stats report: %r', stats.report()) print stats.report() log.info('Warnings (%i): %r', len(warnings), warnings)
def __init__(self, worker_id, heartbeat_interval=60000, action='update'): super(UpdateIntentsWorker, self).__init__(worker_id, heartbeat_interval) self.intents_worker = UserIntentsWorker(intents_extractor) self.action = action # Workaround until the core translation function defaults to the Flask one from paste.registry import Registry from ckan.lib.cli import MockTranslator registry = Registry() registry.prepare() from pylons import translator registry.register(translator, MockTranslator())
def register_translator(): # https://github.com/ckan/ckanext-archiver/blob/master/ckanext/archiver/bin/common.py # If not set (in cli access), patch the a translator with a mock, so the # _() functions in logic layer don't cause failure. from paste.registry import Registry from pylons import translator from ckan.lib.cli import MockTranslator if 'registery' not in globals(): global registry registry = Registry() registry.prepare() if 'translator_obj' not in globals(): global translator_obj translator_obj = MockTranslator() registry.register(translator, translator_obj)
def _register_translator(): """ Register a translator in this thread. """ global registry try: registry except NameError: registry = Registry() registry.prepare() global translator_obj try: translator_obj except NameError: translator_obj = MockTranslator() registry.register(translator, translator_obj)
def authenticate(self, environ, identity): """A username/password authenticator that throttles login request by IP.""" try: login = identity['login'] except KeyError: return None environ['paste.registry'].register(pylons.translator, MockTranslator()) try: remote_addr = Request(environ).headers['X-Forwarded-For'] except KeyError: try: remote_addr = environ['REMOTE_ADDR'] except KeyError: log.critical( 'X-Forwarded-For header/REMOTE_ADDR missing from request.') return None throttle = LoginThrottle(User.by_name(login), remote_addr) if not ('login' in identity and 'password' in identity): return None # Run through the CKAN auth sequence first, so we can hit the DB # in every case and make timing attacks a little more difficult. auth_user = super(CKANLoginThrottle, self).authenticate(environ, identity) # Check if there is a lock on the requested user, and return None if # we have a lock. if throttle.check_attempts() is False: log.info('User %r (%s) locked out by brute force protection.' % (login, remote_addr)) throttle.increment( ) # Increment so we only send an email the first time around return None # If the CKAN authenticator as successfully authenticated the request # and the user wasn't locked out above, reset the throttle counter and # return the user object. if auth_user is not None: throttle.reset() return auth_user # Increment the throttle counter if the login failed. throttle.increment()
def command(config_ini, nodepublisher_csv): config_ini_filepath = os.path.abspath(config_ini) load_config(config_ini_filepath) engine = engine_from_config(config,'sqlalchemy.') from ckan import model from ckan.lib.munge import munge_title_to_name logging.config.fileConfig(config_ini_filepath) log = logging.getLogger(os.path.basename(__file__)) global global_log global_log = log model.init_model(engine) # Register a translator in this thread so that # the _() functions in logic layer can work from ckan.lib.cli import MockTranslator registry=Registry() registry.prepare() translator_obj=MockTranslator() registry.register(translator, translator_obj) model.repo.new_revision() log.info('Reading %s', nodepublisher_csv) with open(nodepublisher_csv, 'rU') as f: reader = csv.reader( f) for row in reader: nid, title = row publishers[ int(nid) ] = munge_title_to_name(title) # Mappings where we are getting rid of duplicate publishers publishers[16268] = publishers[11408] # UKSA -> ONS publishers[11606] = publishers[11408] # ONS publishers[20054] = publishers[16248] # Met Office publishers[33036] = publishers[15255] # Windsor & Maidenhead publishers[32619] = publishers[33245] # Monmouthshire publishers[12662] = publishers[11567] # NHS update_datasets() generate_harvest_publishers() log.info('Warnings: %r', warnings)
def authenticate(self, environ, identity): """A username/password authenticator that throttles login request by user name.""" try: user_name = identity['login'] except KeyError: return None environ['paste.registry'].register(pylons.translator, MockTranslator()) if not ('login' in identity and 'password' in identity): return None # Run through the CKAN auth sequence first, so we can hit the DB # in every case and make timing attacks a little more difficult. auth_user_name = super(CKANLoginThrottle, self).authenticate(environ, identity) login_throttle_key = get_login_throttle_key(Request(environ), user_name) if login_throttle_key is None: return None throttle = LoginThrottle(User.by_name(user_name), login_throttle_key) # Check if there is a lock on the requested user, and return None if # we have a lock. if throttle.is_locked(): return None if auth_user_name is None: # Increment the throttle counter if the login failed. throttle.increment() # if the CKAN authenticator has successfully authenticated the request and the user wasn't locked out above, # then check the TOTP parameter to see if it is valid if auth_user_name is not None: totp_success = self.authenticate_totp(environ, auth_user_name) if totp_success: # if TOTP was successful -- reset the log in throttle throttle.reset() return totp_success
def setup_class(cls): cls.registry = Registry() cls.registry.prepare() cls.context_obj = AttribSafeContextObj() cls.registry.register(pylons.c, cls.context_obj) cls.app_globals_obj = app_globals.Globals() cls.registry.register(pylons.g, cls.app_globals_obj) cls.request_obj = Request(dict(HTTP_HOST="nohost")) cls.registry.register(pylons.request, cls.request_obj) cls.translator_obj = MockTranslator() cls.registry.register(pylons.translator, cls.translator_obj) cls.buffet = pylons.templating.Buffet('genshi', template_root='ckan.templates') cls.registry.register(pylons.buffet, cls.buffet) cls.registry.register(pylons.response, Response()) mapper = make_map() cls.registry.register(pylons.url, URLGenerator(mapper, {})) cls.registry.register(pylons.session, TestSession())
dest='subcatalogs', default=False, help="Enable subcatalogs handling (dct:hasPart support)") args = parser.parse_args() contents = args.file.read() config.update({DCAT_EXPOSE_SUBCATALOGS: args.subcatalogs}) # Workaround until the core translation function defaults to the Flask one from paste.registry import Registry from ckan.lib.cli import MockTranslator registry = Registry() registry.prepare() from pylons import translator registry.register(translator, MockTranslator()) if args.mode == 'produce': serializer = RDFSerializer(profiles=args.profile, compatibility_mode=args.compat_mode) dataset = json.loads(contents) out = serializer.serialize_dataset(dataset, _format=args.format) print(out) else: parser = RDFParser(profiles=args.profile, compatibility_mode=args.compat_mode) parser.parse(contents, _format=args.format) ckan_datasets = [d for d in parser.datasets()]
def migrate_v3_create_datasets(source_ids=None): import pylons from paste.registry import Registry from ckan.lib.cli import MockTranslator registry = Registry() registry.prepare() registry.register(pylons.translator, MockTranslator()) sources = [] if not source_ids: sources = model.Session.query(HarvestSource).all() else: sources = model.Session.query(HarvestSource) \ .filter(HarvestSource.id.in_(source_ids)) \ .all() if not sources: log.debug('No harvest sources to migrate') return site_user_name = logic.get_action('get_site_user')({ 'model': model, 'ignore_auth': True }, {})['name'] context = { 'model': model, 'session': model.Session, 'user': site_user_name, # TODO: auth of existing sources? 'return_id_only': True, 'extras_as_string': True, } def gen_new_name(title): name = munge_title_to_name(title).replace('_', '-') while '--' in name: name = name.replace('--', '-') pkg_obj = Session.query(Package).filter(Package.name == name).first() if pkg_obj: return name + str(uuid.uuid4())[:5] else: return name for source in sources: if 'id' in context: del context['id'] if 'package' in context: del context['package'] # Check if package already exists try: logic.get_action('package_show')(context, {'id': source.id}) continue except logic.NotFound: pass package_dict = { 'id': source.id, 'name': gen_new_name(source.title) if source.title else source.id, 'title': source.title if source.title else source.url, 'notes': source.description, 'url': source.url, 'type': 'harvest', 'source_type': source.type, 'config': source.config, 'frequency': source.frequency, } context['message'] = 'Created package for harvest source {0}'.format( source.id) try: new_package_id = logic.get_action('package_create')(context, package_dict) if new_package_id != source.id or not context['return_id_only']: # this check only makes sense if we are sure we are returning # the package id not the package object raise PackageIdHarvestSourceIdMismatch log.info('Created new package for source {0} ({1})'.format( source.id, source.url)) except logic.ValidationError, e: log.error('Validation Error: %s' % str(e.error_summary))
def mock_objects(config): registry = Registry() registry.prepare() registry.register(pylons.translator, MockTranslator()) h.config = config
def command(config_ini, nodepublisher_csv, users_csv): config_ini_filepath = os.path.abspath(config_ini) load_config(config_ini_filepath) engine = engine_from_config(config, 'sqlalchemy.') from ckan import model from ckan.lib.munge import munge_title_to_name import ckanext.dgu.plugin logging.config.fileConfig(config_ini_filepath) global log log = logging.getLogger(os.path.basename(__file__)) model.init_model(engine) # Register a translator in this thread so that # the _() functions in logic layer can work from ckan.lib.cli import MockTranslator registry = Registry() registry.prepare() translator_obj = MockTranslator() registry.register(translator, translator_obj) with open(nodepublisher_csv, 'rU') as f: reader = csv.reader(f) for row in reader: publishers[int(row[0])] = munge_title_to_name(row[1]) log.info('Opened list of %i publishers', reader.line_num) # get rid of flash message warnings warnings_.filterwarnings('ignore', '.*flash message.*') ckanext.dgu.plugin.log.disabled = True with open(users_csv, 'rU') as f: reader = csv.reader(f) for row in reader: model.repo.new_revision() node_id, name, email, publisher_id = row # create a new user uname, user_id = _add_new_user(int(node_id), name, email) if not user_id: # validation error. warning already printed continue # Find the publisher and add them as editor if node_id: publisher_name = publishers[int(publisher_id)] publisher = model.Group.by_name(publisher_name) if not publisher: warn( 'Could not find publisher %r so skipping making %r editor for it.', publisher_name, name) continue capacity = 'editor' # Check for Member where table_name is u['id'] res = model.Session.query(model.Member).\ from_statement(MEMBER_LOOKUP).\ params(userid=user_id, groupid=publisher.id).all() if len(res) == 0: m = model.Member(group_id=publisher.id, table_id=user_id, table_name='user', capacity=capacity) model.Session.add(m) log.info('Made %r editor for %r', name, publisher_name) else: log.info('%r already editor for %r', name, publisher_name) # Update harvest_source user_id field to new user id. model.Session.execute(HARVEST_QUERY, params={ 'uid': user_id, 'node_id': str(node_id) }) model.Session.commit() log.info('Processed list of %i users', reader.line_num) log.info('Warnings (%i): %r', len(warnings), warnings)
def command(dry_run=False): from ckan import model # Register a translator in this thread so that # the _() functions in logic layer can work from ckan.lib.cli import MockTranslator registry=Registry() registry.prepare() translator_obj=MockTranslator() registry.register(translator, translator_obj) global_log.info('Tidying package fields') stats = StatsList() if not dry_run: rev = model.repo.new_revision() rev.message = 'Package fields migration' for pkg in model.Session.query(model.Package)\ .filter_by(state='active')\ .order_by(model.Package.name): # field map for existing_fields, destination_field in field_map.items(): value = pkg.extras.get(destination_field) if value: continue for existing_field in existing_fields: if hasattr(pkg, existing_field): value = getattr(pkg, existing_field) else: value = pkg.extras.get(existing_field) if value: value = value.strip() if value: # take the first hit continue if not dry_run: pkg.extras[destination_field] = value or '' # delete existing field values for existing_field in existing_fields: if hasattr(pkg, existing_field): setattr(pkg, existing_field, '') elif existing_field in pkg.extras: del pkg.extras[existing_field] if value: stats.add('Merged to field "%s"' % destination_field, pkg.name) else: stats.add('Not merged to field "%s"' % destination_field, pkg.name) # move url to additional resource if pkg.url: stats.add('Url moved to additional resource', value) if not dry_run: if not pkg.resource_groups: res_group = model.ResourceGroup(label="default") pkg.resource_groups.append(res_group) res_group = pkg.resource_groups[0] res = model.Resource(format='HTML', resource_type='documentation', url=pkg.url, description='Web page about the data') res_group.resources.append(res) model.Session.add(res) #pkg.url = '' stats.add('URL moved to additional resource', pkg.name) else: stats.add('No URL to move to additional resource', pkg.name) # delete fields for field in delete_fields: if field in pkg.extras: if not dry_run: del pkg.extras[field] stats.add('Deleted field "%s"' % field, pkg.name) else: stats.add('No field to delete "%s"' % field, pkg.name) if not dry_run: model.repo.commit_and_remove() global_log.info(stats.report())
def _get_or_create_user(self, env): #WSGI Variables #Shib-Application-ID 'default' #Shib-Authentication-Instant '2012-08-13T12:04:22.492Z' #Shib-Authentication-Method 'urn:oasis:names:tc:SAML:2.0:ac:classes:PasswordProtectedTransport' #Shib-AuthnContext-Class 'urn:oasis:names:tc:SAML:2.0:ac:classes:PasswordProtectedTransport' #Shib-Identity-Provider 'https://idp.example.com/idp/shibboleth' #Shib-Session-ID '_7ec5a681e6dbae627c1cefcc7cb4d56a' #Shib-Session-Index '39dafd8477850f5e0b968e3561570197f2109948c1d374a7a2b4c9a7adbf8628' #cn 'My Other Self' #givenName 'My Other Self' #mail '*****@*****.**' eppn = env.get(self.eppn, None) fullname = env.get(self.fullname, None) email = env.get(self.mail, None) if not eppn or not fullname: log.debug( 'Environ does not contain eppn or cn attributes, user not loaded.') return None user = model.Session.query(model.User).autoflush(False) \ .filter_by(name=eppn).first() # Check if user information from shibboleth has changed if user: if (user.fullname != fullname or user.email != email): log.debug('User attributes modified, updating.') user.fullname = fullname user.email = email else: # user is None: log.debug('User does not exists, creating new one.') basename = eppn username = basename suffix = 0 while not model.User.check_name_available(username): suffix += 1 username = basename + str(suffix) user = model.User(name=username, fullname=fullname, email=email, openid=eppn) model.Session.add(user) model.Session.flush() log.info('Created new user {usr}'.format(usr=fullname)) groups = env.get(self.groups, None) if groups: groups = groups.split(";") log.debug("groups: {}".format(sorted(groups))) orgs = toolkit.get_action('group_list')(data_dict={ 'all_fields': True, 'include_extras': True }) log.debug("orgs: {}".format(orgs)) add_member = toolkit.get_action('group_member_create') # Ensure there's a pylons.translator object registry = Registry() registry.prepare() registry.register(pylons.translator, MockTranslator()) for o in orgs: for e in o['extras']: if e['key'] == 'ecgroup' and e['value'] in groups: log.debug("Adding {} to {}".format(user.name, o['name'])) add_member(context={ 'user': '******' }, data_dict={ 'id': o['name'], 'username': user.name, 'role': 'member' }) model.Session.commit() model.Session.remove() return user
def setup(): # Register a mock translator instead of having ckan domain translations defined patcher = mock.patch('pylons.i18n.translation._get_translator', return_value=MockTranslator()) patcher.start()
def command(input_csv, config_ini, commit=False): config_ini_filepath = os.path.abspath(config_ini) load_config(config_ini_filepath) engine = engine_from_config(config, 'sqlalchemy.') logging.config.fileConfig(config_ini_filepath) log = logging.getLogger(os.path.basename(__file__)) from ckan import model from ckan.logic import get_action from ckan.lib.cli import MockTranslator model.init_model(engine) registry = Registry() registry.prepare() translator_obj = MockTranslator() registry.register(translator, translator_obj) ctx = { 'model': model, 'session': model.Session, 'user': get_action('get_site_user')({ 'model': model, 'ignore_auth': True }, {})['name'] } if commit: rev = model.repo.new_revision() packages_to_check = set() reader = csv.reader(open(input_csv, 'r')) for row in reader: # For each URL in the csv, get the list of resources referencing # that URL resources = model.Session.query(model.Resource)\ .filter(model.Resource.state=='active')\ .filter(model.Resource.url==row[0]).all() for resource in resources: # For each resource, add the package to the list packages_to_check.add(resource.get_package_id()) # Delete the resource resource.state = 'deleted' model.Session.add(resource) if commit: model.Session.commit() print "Deleted resource: {0}".format(resource.id) stats.increment("Deleted resource") for pid in packages_to_check: # for each package we need to check, see if it has any # resources left, it not, delete it. pkg = model.Package.get(pid) if len(pkg.resources) == 0: pkg.state = 'deleted' model.Session.add(pkg) if commit: model.Session.commit() stats.increment('Deleted packages') print "Deleted package: {0}".format(pkg.name) if commit: model.repo.commit_and_remove() else: print "" print '*' * 60 print "DON'T PANIC, this was a dry run, nothing was committed" print '*' * 60 print '' print '*' * 60, 'Deletion Report' print stats.report(order_by_title=True)