def main(): parser = argparse.ArgumentParser(description = __doc__) parser.add_argument('config', help = 'path of configuration file') parser.add_argument('-v', '--verbose', action = 'store_true', help = 'increase output verbosity') args = parser.parse_args() # logging.basicConfig(level = logging.DEBUG if args.verbose else logging.WARNING, stream = sys.stdout) logging.basicConfig(level = logging.INFO if args.verbose else logging.WARNING, stream = sys.stdout) site_conf = appconfig('config:{}'.format(os.path.abspath(args.config))) load_environment(site_conf.global_conf, site_conf.local_conf) registry = Registry() registry.prepare() registry.register(pylons.translator, MockTranslator()) plugins.load('synchronous_search') model.repo.new_revision() for package_extra in model.Session.query(model.PackageExtra).filter( model.PackageExtra.key == 'supplier_id', model.PackageExtra.package_id.in_( model.Session.query(model.Package.id).filter(model.Package.name.like('%-fork-%'))), ): model.Session.delete(package_extra) model.repo.commit_and_remove() return 0
class CkanCommand(paste.script.command.Command): parser = paste.script.command.Command.standard_parser(verbose=True) parser.add_option('-c', '--config', dest='config', default='development.ini', help='Config file to use.') parser.add_option('-f', '--file', action='store', dest='file_path', help="File to dump results to (if needed)") default_verbosity = 1 group_name = 'ckan' def _load_config(self): from paste.deploy import appconfig from ckan.config.environment import load_environment if not self.options.config: msg = 'No config file supplied' raise self.BadCommand(msg) self.filename = os.path.abspath(self.options.config) if not os.path.exists(self.filename): raise AssertionError('Config filename %r does not exist.' % self.filename) fileConfig(self.filename) conf = appconfig('config:' + self.filename) load_environment(conf.global_conf, conf.local_conf) self.registry=Registry() self.registry.prepare() import pylons self.translator_obj = MockTranslator() self.registry.register(pylons.translator, self.translator_obj) def _setup_app(self): cmd = paste.script.appinstall.SetupCommand('setup-app') cmd.run([self.filename])
def setup_environment_and_database(env_dir=None, enabled_plugins=''): global_config = {} env_dir = env_dir or '/invalid' app_instance_secret = secret.secret_string() app_config = { 'plugins': enabled_plugins, 'sqlalchemy.url': 'sqlite://', 'layout_template': 'layout', 'external_template': 'false', 'image_dir': os.path.join(env_dir, 'images'), 'media_dir': os.path.join(env_dir, 'media'), 'beaker.session.secret': app_instance_secret, 'sa_auth.cookie_secret': app_instance_secret, } pylons_config = load_environment(global_config, app_config) paste_registry = Registry() paste_registry.prepare() app_globals = pylons_config['pylons.app_globals'] register_instance(paste_registry, 'app_globals', app_globals) pylons_config.update({ 'paste.registry': paste_registry, 'pylons.pylons': pylons, }) metadata.create_all(bind=DBSession.bind, checkfirst=True) return pylons_config
def main(): parser = argparse.ArgumentParser(description = __doc__) parser.add_argument('config', help = 'path of configuration file') parser.add_argument('-v', '--verbose', action = 'store_true', help = 'increase output verbosity') args = parser.parse_args() # logging.basicConfig(level = logging.DEBUG if args.verbose else logging.WARNING, stream = sys.stdout) logging.basicConfig(level = logging.INFO if args.verbose else logging.WARNING, stream = sys.stdout) site_conf = appconfig('config:{}'.format(os.path.abspath(args.config))) load_environment(site_conf.global_conf, site_conf.local_conf) registry = Registry() registry.prepare() registry.register(pylons.translator, MockTranslator()) plugins.load('synchronous_search') model.repo.new_revision() for package_extra in model.Session.query(model.PackageExtra).filter( model.PackageExtra.key == 'territorial_coverage_granularity', model.PackageExtra.value == 'france', ): package = package_extra.package if package.private or package.state != 'active': log.warning(u'Territorial coverage granularity of package {} must be manually corrected'.format( package.name)) continue package_extra.value = 'pays' model.repo.commit_and_remove() return 0
class HelloworldCommand(Command): ''' A base class to use when the full Pylons environment should be loaded ''' summary = __doc__.split("\n") [0] usage = __doc__ group_name = "helloworld" def __init__(self, name): Command.__init__(self, name) self.parser = Command.standard_parser(verbose=False) self.parser.add_option('-c', '--config', dest='config', default='development.ini', help='Config file to use.') return def command(self): pass def _load_config(self): from paste.deploy import appconfig if not self.options.config: msg = 'No config file supplied' raise self.BadCommand(msg) self.filename = os.path.abspath(self.options.config) if not os.path.exists(self.filename): raise AssertionError('Config filename %r does not exist.' % self.filename) fileConfig (self.filename) conf = appconfig('config:' + self.filename) # We have now loaded the config. Now we can import helloworld for the first time. from helloworld.config.environment import load_environment load_environment (conf.global_conf, conf.local_conf) self.registry = Registry() self.registry.prepare() return
def load_config(config: Any, load_site_user: bool = True): conf = _get_config(config) assert 'ckan' not in dir() # otherwise loggers would be disabled # We have now loaded the config. Now we can import ckan for the # first time. from ckan.config.environment import load_environment load_environment(conf) # Set this internal test request context with the configured environment so # it can be used when calling url_for from the CLI. global _cli_test_request_context app = make_app(conf) flask_app = app.apps['flask_app']._wsgi_app _cli_test_request_context = flask_app.test_request_context() registry = Registry() registry.prepare() site_user = None if model.user_table.exists() and load_site_user: site_user = logic.get_action('get_site_user')({ 'ignore_auth': True }, {}) ## give routes enough information to run url_for parsed = urlparse(cast(str, conf.get('ckan.site_url', 'http://0.0.0.0'))) request_config = routes.request_config() request_config.host = parsed.netloc + parsed.path request_config.protocol = parsed.scheme return site_user
def register_translator(): from paste.registry import Registry import pylons registry = Registry() registry.prepare() translator_obj = MockTranslator() registry.register(pylons.translator, translator_obj) log.info('Translator object was registered for the background job')
def main(): parser = argparse.ArgumentParser(description = __doc__) parser.add_argument('config', help = 'path of configuration file') parser.add_argument('-v', '--verbose', action = 'store_true', help = 'increase output verbosity') args = parser.parse_args() # logging.basicConfig(level = logging.DEBUG if args.verbose else logging.WARNING, stream = sys.stdout) logging.basicConfig(level = logging.INFO if args.verbose else logging.WARNING, stream = sys.stdout) site_conf = appconfig('config:{}'.format(os.path.abspath(args.config))) load_environment(site_conf.global_conf, site_conf.local_conf) registry = Registry() registry.prepare() registry.register(pylons.translator, MockTranslator()) plugins.load('synchronous_search') # model.repo.new_revision() # for resource in model.Session.query(model.Resource).filter( # model.Resource.url.like('%etalab2.fr%'), # ): # url = resource.url # if url.startswith('http://ckan.etalab2.fr/'): # resource.url = resource.url.replace('http://ckan.etalab2.fr/', 'http://www.data.gouv.fr/fr/') # elif url.startswith('http://ckan-hetic.etalab2.fr/'): # resource.url = resource.url.replace('http://ckan-hetic.etalab2.fr/', 'http://www.data.gouv.fr/fr/') # elif url.startswith('http://www.etalab2.fr/'): # resource.url = resource.url.replace('http://www.etalab2.fr/', 'http://www.data.gouv.fr/') # else: # print resource.url # model.repo.commit_and_remove() # model.repo.new_revision() # for resource in model.Session.query(model.Resource).filter( # model.Resource.url.like('%www.data.gouv.fr%'), # ): # if resource.url.startswith('http://www.data.gouv.fr/') and not resource.url.startswith('http://www.data.gouv.fr/var'): # resource.url = resource.url.replace('http://www.data.gouv.fr/', 'http://new.data.gouv.fr/') # model.repo.commit_and_remove() model.repo.new_revision() for resource in model.Session.query(model.Resource).filter( model.Resource.url.like('http://new.data.gouv.fr/%'), ): resource.url = resource.url.replace('http://new.data.gouv.fr/', 'https://www.data.gouv.fr/') model.repo.commit_and_remove() model.repo.new_revision() for resource in model.Session.query(model.Resource).filter( model.Resource.url.like('https://new.data.gouv.fr/%'), ): resource.url = resource.url.replace('https://new.data.gouv.fr/', 'https://www.data.gouv.fr/') model.repo.commit_and_remove() return 0
def main(): parser = argparse.ArgumentParser(description = __doc__) parser.add_argument('config', help = 'path of configuration file') parser.add_argument('-t', '--territoria-url', help = 'Territoria URL', required = True) parser.add_argument('-v', '--verbose', action = 'store_true', help = 'increase output verbosity') args = parser.parse_args() # logging.basicConfig(level = logging.DEBUG if args.verbose else logging.WARNING, stream = sys.stdout) logging.basicConfig(level = logging.INFO if args.verbose else logging.WARNING, stream = sys.stdout) site_conf = appconfig('config:{}'.format(os.path.abspath(args.config))) load_environment(site_conf.global_conf, site_conf.local_conf) registry = Registry() registry.prepare() registry.register(pylons.translator, MockTranslator()) plugins.load('synchronous_search') model.repo.new_revision() kind_code_name_by_kind_code = {} for package_extra in model.Session.query(model.PackageExtra).filter( model.PackageExtra.key == 'territorial_coverage', ): if package_extra.value == 'Coutry/FR': kind_code_name = 'Country/FR/FRANCE' elif package_extra.value == 'InternationalOrganization/EU': kind_code_name = 'InternationalOrganization/UE/UNION EUROPEENNE' elif package_extra.value.count('/') == 1: kind_code_name = kind_code_name_by_kind_code.get(package_extra.value) if kind_code_name is None: kind, code = package_extra.value.split('/') try: response = urllib2.urlopen(urlparse.urljoin(args.territoria_url, '/api/v1/territory?{}'.format(urllib.urlencode(dict( code = code, kind = kind, ), doseq = True)))) except urllib2.HTTPError, response: print package_extra.value raise response_dict = json.loads(response.read()) main_postal_distribution = response_dict['data']['main_postal_distribution'] kind_code_name_by_kind_code[package_extra.value] = kind_code_name = u'/'.join([kind, code, main_postal_distribution]) print kind_code_name else: continue package = package_extra.package if package.private or package.state != 'active': log.warning(u'Territorial coverage of package {} must be manually corrected'.format(package.name)) continue package_extra.value = kind_code_name
def _paste_registry(pylons_request): "Internal helper method to get a new registry." paste_registry = None is_request_registered = is_object_registered(pylons_request) if is_request_registered: paste_registry = pylons_request.environ.get('paste.registry') if paste_registry is None: paste_registry = Registry() paste_registry.prepare() if is_request_registered: pylons_request.environ['paste.registry'] = paste_registry return paste_registry
def register_translator(): # Register a translator in this thread so that # the _() functions in logic layer can work from paste.registry import Registry from pylons import translator from ckan.lib.cli import MockTranslator global registry registry = Registry() registry.prepare() global translator_obj translator_obj = MockTranslator() registry.register(translator, translator_obj)
def load_config(path): import paste.deploy conf = paste.deploy.appconfig('config:' + path) import ckan, pylons ckan.config.environment.load_environment(conf.global_conf, conf.local_conf) from ckan.lib.cli import MockTranslator from paste.registry import Registry registry=Registry() registry.prepare() translator_obj=MockTranslator() registry.register(pylons.translator, translator_obj)
def command(dry_run=False): from ckan import model from ckanext.dgu.lib.resource_formats import match from running_stats import StatsList # Register a translator in this thread so that # the _() functions in logic layer can work from ckan.lib.cli import MockTranslator registry=Registry() registry.prepare() translator_obj=MockTranslator() registry.register(translator, translator_obj) if not dry_run: model.repo.new_revision() # Add canonised formats to map for format_ in res_type_map.keys(): res_type_map[canonise(format_)] = res_type_map[format_] log.info('Tidying resource types') stats = StatsList() res_query = model.Session.query(model.Resource) log.info('Tidying formats. Resources=%i Canonised formats=%i', res_query.count(), len(set(res_type_map.values()))) for res in res_query: canonised_fmt = canonise(res.format or '') if canonised_fmt in res_type_map: improved_fmt = res_type_map[canonised_fmt] else: improved_fmt = tidy(res.format) match_ = match(improved_fmt) if match_: improved_fmt = match_ if (improved_fmt or '') != (res.format or ''): if not dry_run: res.format = improved_fmt stats.add(improved_fmt, res.format) else: stats.add('No change', res.format) if not dry_run: model.repo.commit_and_remove() log.info('Stats report: %r', stats.report()) print stats.report() log.info('Warnings (%i): %r', len(warnings), warnings)
def load_translations(lang): # Register a translator in this thread so that # the _() functions in logic layer can work import ckan.lib.i18n from paste.registry import Registry from pylons import translator from pylons import request registry = Registry() registry.prepare() class FakePylons: translator = None fakepylons = FakePylons() class FakeRequest: # Stores details of the translator environ = {'pylons.pylons': fakepylons} registry.register(request, FakeRequest()) # create translator ckan.lib.i18n.set_lang(lang) # pull out translator and register it registry.register(translator, fakepylons.translator)
def load_config(path): import paste.deploy from paste.registry import Registry from command import MockTranslator conf = paste.deploy.appconfig('config:' + os.path.abspath(path)) import ckan ckan.config.environment.load_environment(conf.global_conf, conf.local_conf) registry=Registry() registry.prepare() translator_obj = MockTranslator() registry.register(pylons.translator, translator_obj)
def create_fake_env(): registry = Registry() registry.prepare() registry.register(session, SessionObject({})) registry.register(request, Request.blank('/bootstrap')) request.identity = {} return session, request
def _register_translator(): """ Register a translator in this thread. """ global registry try: registry except NameError: registry = Registry() registry.prepare() global translator_obj try: translator_obj except NameError: translator_obj = MockTranslator() registry.register(translator, translator_obj)
def register_translator(): # https://github.com/ckan/ckanext-archiver/blob/master/ckanext/archiver/bin/common.py # If not set (in cli access), patch the a translator with a mock, so the # _() functions in logic layer don't cause failure. from paste.registry import Registry from pylons import translator from ckan.lib.cli import MockTranslator if 'registery' not in globals(): global registry registry = Registry() registry.prepare() if 'translator_obj' not in globals(): global translator_obj translator_obj = MockTranslator() registry.register(translator, translator_obj)
def __call__(self, environ, start_response): app_iter = None reg = environ.setdefault('paste.registry', Registry()) reg.prepare() if self.streaming: return self.streaming_iter(reg, environ, start_response) try: app_iter = self.application(environ, start_response) except Exception, e: # Regardless of if the content is an iterable, generator, list # or tuple, we clean-up right now. If its an iterable/generator # care should be used to ensure the generator has its own ref # to the actual object if environ.get('paste.evalexception'): # EvalException is present in the WSGI stack expected = False for expect in environ.get('paste.expected_exceptions', []): if isinstance(e, expect): expected = True if not expected: # An unexpected exception: save state for EvalException restorer.save_registry_state(environ) reg.cleanup() raise
def setup_class(cls): cls.registry = Registry() cls.registry.prepare() cls.context_obj = AttribSafeContextObj() cls.registry.register(pylons.c, cls.context_obj) cls.app_globals_obj = app_globals.Globals() cls.registry.register(pylons.g, cls.app_globals_obj) cls.request_obj = Request(dict(HTTP_HOST="nohost")) cls.registry.register(pylons.request, cls.request_obj) cls.translator_obj = MockTranslator() cls.registry.register(pylons.translator, cls.translator_obj) cls.buffet = pylons.templating.Buffet('genshi', template_root='ckan.templates') cls.registry.register(pylons.buffet, cls.buffet) cls.registry.register(pylons.response, Response()) mapper = make_map() cls.registry.register(pylons.url, URLGenerator(mapper, {})) cls.registry.register(pylons.session, TestSession()) # Templates often want to find out the request's routes info, so put # some dummy values into the routes_dict, so the templates that do # this don't cause an exception. pylons.request.environ.update({ 'pylons.routes_dict': { 'action': 'test-action', 'controller': 'test-package::', } })
def create_request(path, environ=None): """Helper used in test cases to quickly setup a request obj. ``path`` The path will become PATH_INFO ``environ`` Additional environment Returns an instance of the `webob.Request` object. """ # setup the environ if environ is None: environ = {} environ.update(default_environ) # create a "blank" WebOb Request object # using Pylon's Request which is a webob Request plus # some compatibility methods req = Request.blank(path, environ) # setup a Registry reg = environ.setdefault('paste.registry', Registry()) reg.prepare() # setup pylons.request to point to our Registry reg.register(pylons.request, req) # setup tmpl context tmpl_context._push_object(ContextObj()) url._push_object(URLGenerator(default_map, environ)) return req
def register_translator(): # Workaround until core translation function defaults to Flask from paste.registry import Registry from ckan.lib.cli import MockTranslator from pylons import translator registry = Registry() registry.prepare() registry.register(translator, MockTranslator())
def setUp(self): super(AutoCompleterTest, self).setUp() # Création d'un environnement WSGI pour les tests. # Par défaut, l'utilisateur n'est pas identifé # et n'appartient donc à aucun groupe. environ = {'repoze.what.credentials': { 'groups': [], } } request = Request(environ) config = VigiloAppConfig('TurboGears') config.init_managers_predicate() registry = Registry() registry.prepare() registry.register(tg.request, request) registry.register(tg.config, config) self.ctrl = AutoCompleteController() # Par défaut, l'utilisateur n'est pas identifé. tg.request.identity = {'repoze.who.userid': None} self.accounts = ( (u'manager', u'managers'), # Accès à tout (manager) (u'nobody', u'nobody'), # Accès à rien. (u'direct', u'direct'), # Permissions directe sur l'objet. (u'indirect', u'indirect'), # Permissions indirectes sur l'objet. ) # Création des comptes utilisateurs. for username, group in self.accounts: functions.add_user(username, u'%s@test' % username, u'', u'', group) # Création de l'arborescence des objets. self.create_deps()
def _load_config(self): from paste.deploy import appconfig from ckan.config.environment import load_environment if not self.options.config: msg = 'No config file supplied' raise self.BadCommand(msg) self.filename = os.path.abspath(self.options.config) try: fileConfig(self.filename) except Exception: pass conf = appconfig('config:' + self.filename) load_environment(conf.global_conf, conf.local_conf) self.registry=Registry() self.registry.prepare() import pylons self.translator_obj = MockTranslator() self.registry.register(pylons.translator, self.translator_obj)
def _load_config(self): from paste.deploy import appconfig from ckan.config.environment import load_environment if not self.options.config: msg = 'No config file supplied' raise self.BadCommand(msg) self.filename = os.path.abspath(self.options.config) if not os.path.exists(self.filename): raise AssertionError('Config filename %r does not exist.' % self.filename) fileConfig(self.filename) conf = appconfig('config:' + self.filename) load_environment(conf.global_conf, conf.local_conf) self.registry=Registry() self.registry.prepare() import pylons self.translator_obj = MockTranslator() self.registry.register(pylons.translator, self.translator_obj)
def fake_request(pylons_config, server_name='mediacore.example', language='en', method='GET', request_uri='/', post_vars=None): app_globals = pylons_config['pylons.app_globals'] pylons.app_globals._push_object(app_globals) if post_vars and method.upper() != 'POST': raise ValueError('You must not specify post_vars for request method %r' % method) wsgi_environ = create_wsgi_environ('http://%s%s' % (server_name, request_uri), method.upper(), request_body=post_vars) request = Request(wsgi_environ, charset='utf-8') request.language = language request.settings = app_globals.settings pylons.request._push_object(request) response = Response(content_type='application/xml', charset='utf-8') pylons.response._push_object(response) session = SessionObject(wsgi_environ) pylons.session._push_object(session) routes_url = URLGenerator(pylons_config['routes.map'], wsgi_environ) pylons.url._push_object(routes_url) # Use ContextObj() when we get rid of 'pylons.strict_tmpl_context=False' in # mediacore.lib.environment tmpl_context = AttribSafeContextObj() tmpl_context.paginators = Bunch() pylons.tmpl_context._push_object(tmpl_context) # some parts of Pylons (e.g. Pylons.controllers.core.WSGIController) # use the '.c' alias instead. pylons.c = pylons.tmpl_context paste_registry = Registry() paste_registry.prepare() engines = create_tw_engine_manager(app_globals) host_framework = PylonsHostFramework(engines=engines) paste_registry.register(tw.framework, host_framework) setup_translator(language=language, registry=paste_registry, locale_dirs=pylons_config.get('locale_dirs')) wsgi_environ.update({ 'pylons.pylons': pylons, 'paste.registry': paste_registry, }) return request
def command(config_ini, nodepublisher_csv): config_ini_filepath = os.path.abspath(config_ini) load_config(config_ini_filepath) engine = engine_from_config(config,'sqlalchemy.') from ckan import model from ckan.lib.munge import munge_title_to_name logging.config.fileConfig(config_ini_filepath) log = logging.getLogger(os.path.basename(__file__)) global global_log global_log = log model.init_model(engine) # Register a translator in this thread so that # the _() functions in logic layer can work from ckan.lib.cli import MockTranslator registry=Registry() registry.prepare() translator_obj=MockTranslator() registry.register(translator, translator_obj) model.repo.new_revision() log.info('Reading %s', nodepublisher_csv) with open(nodepublisher_csv, 'rU') as f: reader = csv.reader( f) for row in reader: nid, title = row publishers[ int(nid) ] = munge_title_to_name(title) # Mappings where we are getting rid of duplicate publishers publishers[16268] = publishers[11408] # UKSA -> ONS publishers[11606] = publishers[11408] # ONS publishers[20054] = publishers[16248] # Met Office publishers[33036] = publishers[15255] # Windsor & Maidenhead publishers[32619] = publishers[33245] # Monmouthshire publishers[12662] = publishers[11567] # NHS update_datasets() generate_harvest_publishers() log.info('Warnings: %r', warnings)
def init_mediadrop(config_filename, here_dir=None, disable_logging=False): if not os.path.exists(config_filename): raise IOError('Config file %r does not exist.' % config_filename) if here_dir is None: here_dir = os.getcwd() if not disable_logging: logging_config.fileConfig(config_filename) conf = appconfig('config:%s' % config_filename, relative_to=here_dir) pylons_config = load_environment(conf.global_conf, conf.local_conf) paste_registry = Registry() paste_registry.prepare() pylons_config['paste.registry'] = paste_registry app_globals = pylons_config['pylons.app_globals'] register_instance(paste_registry, 'app_globals', app_globals) register_instance(paste_registry, 'config', pylons_config) fake_request(pylons_config, registry=paste_registry) setup_global_translator(registry=paste_registry)
def fake_request( pylons_config, server_name="mediadrop.example", language="en", method="GET", request_uri="/", post_vars=None ): app_globals = pylons_config["pylons.app_globals"] pylons.app_globals._push_object(app_globals) if post_vars and method.upper() != "POST": raise ValueError("You must not specify post_vars for request method %r" % method) wsgi_environ = create_wsgi_environ( "http://%s%s" % (server_name, request_uri), method.upper(), request_body=post_vars ) request = Request(wsgi_environ, charset="utf-8") request.language = language request.settings = app_globals.settings pylons.request._push_object(request) response = Response(content_type="application/xml", charset="utf-8") pylons.response._push_object(response) session = SessionObject(wsgi_environ) pylons.session._push_object(session) routes_url = URLGenerator(pylons_config["routes.map"], wsgi_environ) pylons.url._push_object(routes_url) # Use ContextObj() when we get rid of 'pylons.strict_tmpl_context=False' in # mediadrop.lib.environment tmpl_context = AttribSafeContextObj() tmpl_context.paginators = Bunch() pylons.tmpl_context._push_object(tmpl_context) # some parts of Pylons (e.g. Pylons.controllers.core.WSGIController) # use the '.c' alias instead. pylons.c = pylons.tmpl_context paste_registry = Registry() paste_registry.prepare() engines = create_tw_engine_manager(app_globals) host_framework = PylonsHostFramework(engines=engines) paste_registry.register(tw.framework, host_framework) setup_translator(language=language, registry=paste_registry, locale_dirs=pylons_config.get("locale_dirs")) wsgi_environ.update({"pylons.pylons": pylons, "paste.registry": paste_registry}) return request
def _load_config(self): from paste.deploy import appconfig if not self.options.config: msg = 'No config file supplied' raise self.BadCommand(msg) self.filename = os.path.abspath(self.options.config) if not os.path.exists(self.filename): raise AssertionError('Config filename %r does not exist.' % self.filename) fileConfig(self.filename) conf = appconfig('config:' + self.filename) assert 'ckan' not in dir() # otherwise loggers would be disabled # We have now loaded the config. Now we can import ckan for the # first time. from ckan.config.environment import load_environment load_environment(conf.global_conf, conf.local_conf) self.registry=Registry() self.registry.prepare() import pylons self.translator_obj = MockTranslator() self.registry.register(pylons.translator, self.translator_obj)
def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('config', help='path of configuration file') parser.add_argument('-v', '--verbose', action='store_true', help='increase output verbosity') args = parser.parse_args() # logging.basicConfig(level = logging.DEBUG if args.verbose else logging.WARNING, stream = sys.stdout) logging.basicConfig( level=logging.INFO if args.verbose else logging.WARNING, stream=sys.stdout) site_conf = appconfig('config:{}'.format(os.path.abspath(args.config))) load_environment(site_conf.global_conf, site_conf.local_conf) registry = Registry() registry.prepare() registry.register(pylons.translator, MockTranslator()) plugins.load('synchronous_search') # model.repo.new_revision() # for resource in model.Session.query(model.Resource).filter( # model.Resource.url.like('%etalab2.fr%'), # ): # url = resource.url # if url.startswith('http://ckan.etalab2.fr/'): # resource.url = resource.url.replace('http://ckan.etalab2.fr/', 'http://www.data.gouv.fr/fr/') # elif url.startswith('http://ckan-hetic.etalab2.fr/'): # resource.url = resource.url.replace('http://ckan-hetic.etalab2.fr/', 'http://www.data.gouv.fr/fr/') # elif url.startswith('http://www.etalab2.fr/'): # resource.url = resource.url.replace('http://www.etalab2.fr/', 'http://www.data.gouv.fr/') # else: # print resource.url # model.repo.commit_and_remove() # model.repo.new_revision() # for resource in model.Session.query(model.Resource).filter( # model.Resource.url.like('%www.data.gouv.fr%'), # ): # if resource.url.startswith('http://www.data.gouv.fr/') and not resource.url.startswith('http://www.data.gouv.fr/var'): # resource.url = resource.url.replace('http://www.data.gouv.fr/', 'http://new.data.gouv.fr/') # model.repo.commit_and_remove() model.repo.new_revision() for resource in model.Session.query(model.Resource).filter( model.Resource.url.like('http://new.data.gouv.fr/%'), ): resource.url = resource.url.replace('http://new.data.gouv.fr/', 'https://www.data.gouv.fr/') model.repo.commit_and_remove() model.repo.new_revision() for resource in model.Session.query(model.Resource).filter( model.Resource.url.like('https://new.data.gouv.fr/%'), ): resource.url = resource.url.replace('https://new.data.gouv.fr/', 'https://www.data.gouv.fr/') model.repo.commit_and_remove() return 0
def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('config', help='path of configuration file') parser.add_argument('-v', '--verbose', action='store_true', help='increase output verbosity') args = parser.parse_args() # logging.basicConfig(level = logging.DEBUG if args.verbose else logging.WARNING, stream = sys.stdout) logging.basicConfig( level=logging.INFO if args.verbose else logging.WARNING, stream=sys.stdout) site_conf = appconfig('config:{}'.format(os.path.abspath(args.config))) load_environment(site_conf.global_conf, site_conf.local_conf) registry = Registry() registry.prepare() registry.register(pylons.translator, MockTranslator()) plugins.load('synchronous_search') model.repo.new_revision() for package_extra in model.Session.query(model.PackageExtra).filter( model.PackageExtra.key == 'supplier_id', model.PackageExtra.package_id.in_( model.Session.query(model.Package.id).filter( model.Package.name.like('%-fork-%'))), ): model.Session.delete(package_extra) model.repo.commit_and_remove() return 0
def fake_request(pylons_config, server_name='mediadrop.example', language='en', method='GET', request_uri='/', post_vars=None): app_globals = pylons_config['pylons.app_globals'] pylons.app_globals._push_object(app_globals) if post_vars and method.upper() != 'POST': raise ValueError('You must not specify post_vars for request method %r' % method) wsgi_environ = create_wsgi_environ('http://%s%s' % (server_name, request_uri), method.upper(), request_body=post_vars) request = Request(wsgi_environ, charset='utf-8') request.language = language request.settings = app_globals.settings pylons.request._push_object(request) response = Response(content_type='application/xml', charset='utf-8') pylons.response._push_object(response) session = SessionObject(wsgi_environ) pylons.session._push_object(session) routes_url = URLGenerator(pylons_config['routes.map'], wsgi_environ) pylons.url._push_object(routes_url) # TODO: Use ContextObj() for Pylons 0.10 tmpl_context = AttribSafeContextObj() tmpl_context.paginators = Bunch() pylons.tmpl_context._push_object(tmpl_context) # some parts of Pylons (e.g. Pylons.controllers.core.WSGIController) # use the '.c' alias instead. pylons.c = pylons.tmpl_context paste_registry = Registry() paste_registry.prepare() engines = create_tw_engine_manager(app_globals) host_framework = PylonsHostFramework(engines=engines) paste_registry.register(tw.framework, host_framework) mediacore_i18n_path = os.path.join(os.path.dirname(mediacore.__file__), 'i18n') translator = Translator(language, dict(mediacore=mediacore_i18n_path)) # not sure why but sometimes pylons.translator is not a StackedObjectProxy # but just a regular Translator. if not hasattr(pylons.translator, '_push_object'): pylons.translator = StackedObjectProxy() paste_registry.replace(pylons.translator, translator) wsgi_environ.update({ 'pylons.pylons': pylons, 'paste.registry': paste_registry, }) return request
def main(): parser = argparse.ArgumentParser(description = __doc__) parser.add_argument('config', help = 'path of configuration file') parser.add_argument('-v', '--verbose', action = 'store_true', help = 'increase output verbosity') args = parser.parse_args() # logging.basicConfig(level = logging.DEBUG if args.verbose else logging.WARNING, stream = sys.stdout) logging.basicConfig(level = logging.INFO if args.verbose else logging.WARNING, stream = sys.stdout) site_conf = appconfig('config:{}'.format(os.path.abspath(args.config))) load_environment(site_conf.global_conf, site_conf.local_conf) registry = Registry() registry.prepare() registry.register(pylons.translator, MockTranslator()) plugins.load('synchronous_search') model.repo.new_revision() for organization_name, is_service_public in sorted(is_public_service_by_organization_name.iteritems()): if not is_service_public: continue organization = model.Session.query(model.Group).filter( model.Group.is_organization == True, model.Group.name == organization_name, ).first() if organization is None: log.warning(u'Unknown organization: {}'.format(organization_name)) continue if organization.certified_public_service is None: log.info(u'Certifying "{}" as public service'.format(organization_name)) certified_public_service = etalab_model.CertifiedPublicService() certified_public_service.organization_id = organization.id model.Session.add(certified_public_service) model.repo.commit_and_remove() return 0
def setup_class(cls): super(FunctionalTestBase, cls).setup_class() core_helpers.reset_db() # Fix ckanext-hierarchy "c" c = pylons.util.AttribSafeContextObj() registry = Registry() registry.prepare() registry.register(pylons.c, c)
def setup_class(cls): # Hack because the hierarchy extension uses c in some methods # that are called outside the context of a web request c = pylons.util.AttribSafeContextObj() registry = Registry() registry.prepare() registry.register(pylons.c, c) super(AuthTestBase, cls).setup_class()
def __init__(self, conf=None): self.config = load_config(conf) # package_update needs a translator defined i.e. _() from paste.registry import Registry import pylons registry = Registry() registry.prepare() registry.register(pylons.translator, MockTranslator()) self.app = make_app(self.config.global_conf, **self.config.local_conf)
def load_config(config, load_site_user=True): conf = _get_config(config) assert 'ckan' not in dir() # otherwise loggers would be disabled # We have now loaded the config. Now we can import ckan for the # first time. from ckan.config.environment import load_environment load_environment(conf) # Set this internal test request context with the configured environment so # it can be used when calling url_for from the CLI. global _cli_test_request_context app = make_app(conf) flask_app = app.apps['flask_app']._wsgi_app _cli_test_request_context = flask_app.test_request_context() registry = Registry() registry.prepare() import pylons registry.register(pylons.translator, MockTranslator()) site_user = None if model.user_table.exists() and load_site_user: # If the DB has already been initialized, create and register # a pylons context object, and add the site user to it, so the # auth works as in a normal web request c = pylons.util.AttribSafeContextObj() registry.register(pylons.c, c) site_user = logic.get_action('get_site_user')({ 'ignore_auth': True }, {}) pylons.c.user = site_user['name'] pylons.c.userobj = model.User.get(site_user['name']) ## give routes enough information to run url_for parsed = urlparse(conf.get('ckan.site_url', 'http://0.0.0.0')) request_config = routes.request_config() request_config.host = parsed.netloc + parsed.path request_config.protocol = parsed.scheme return site_user
def _load_config(self): from paste.deploy import appconfig if not self.options.config: msg = 'No config file supplied' raise self.BadCommand(msg) self.filename = os.path.abspath(self.options.config) if not os.path.exists(self.filename): raise AssertionError('Config filename %r does not exist.' % self.filename) fileConfig (self.filename) conf = appconfig('config:' + self.filename) # We have now loaded the config. Now we can import helloworld for the first time. from helloworld.config.environment import load_environment load_environment (conf.global_conf, conf.local_conf) self.registry = Registry() self.registry.prepare() return
def load_config(path): import paste.deploy conf = paste.deploy.appconfig('config:' + path) import ckan, pylons ckan.config.environment.load_environment(conf.global_conf, conf.local_conf) from ckan.lib.cli import MockTranslator from paste.registry import Registry registry = Registry() registry.prepare() translator_obj = MockTranslator() registry.register(pylons.translator, translator_obj)
class ModelTest(object): config = appconfig('config:test.ini', relative_to=conf_dir) root_path = os.path.join(conf_dir, 'r2') paths = { 'root': root_path, 'controllers': os.path.join(root_path, 'controllers'), 'templates': tmpl_dirs, 'static_files': os.path.join(root_path, 'public') } registry = Registry() registry.prepare() globals = Globals(config.global_conf, config.local_conf, paths) registry.register(pylons.g, globals) registry.register(pylons.translator, NullTranslations()) context_obj=ContextObj() registry.register(pylons.c, context_obj) r2.config.cache = globals.cache
def command(dry_run=False): from ckan import model from ckanext.dgu.lib.resource_formats import match from running_stats import StatsList # Register a translator in this thread so that # the _() functions in logic layer can work from ckan.lib.cli import MockTranslator registry = Registry() registry.prepare() translator_obj = MockTranslator() registry.register(translator, translator_obj) if not dry_run: model.repo.new_revision() # Add canonised formats to map for format_ in res_type_map.keys(): res_type_map[canonise(format_)] = res_type_map[format_] log.info('Tidying resource types') stats = StatsList() res_query = model.Session.query(model.Resource) log.info('Tidying formats. Resources=%i Canonised formats=%i', res_query.count(), len(set(res_type_map.values()))) for res in res_query: canonised_fmt = canonise(res.format or '') if canonised_fmt in res_type_map: improved_fmt = res_type_map[canonised_fmt] else: improved_fmt = tidy(res.format) match_ = match(improved_fmt) if match_: improved_fmt = match_ if (improved_fmt or '') != (res.format or ''): if not dry_run: res.format = improved_fmt stats.add(improved_fmt, res.format) else: stats.add('No change', res.format) if not dry_run: model.repo.commit_and_remove() log.info('Stats report: %r', stats.report()) print stats.report() log.info('Warnings (%i): %r', len(warnings), warnings)
def setup(self): self.registry=Registry() self.registry.prepare() self.context_obj=ContextObj() self.registry.register(pylons.c, self.context_obj) pylons.c.errors = None self.request_obj=Request(dict(HTTP_HOST="nohost")) self.registry.register(pylons.request, self.request_obj) self.translator_obj=MockTranslator() self.registry.register(pylons.translator, self.translator_obj) self.buffet = pylons.templating.Buffet('genshi', template_root='ckan.templates') self.registry.register(pylons.buffet, self.buffet) self.registry.register(pylons.response, Response()) self.registry.register(pylons.url, None)
def load_config(path): import paste.deploy from paste.registry import Registry from command import MockTranslator conf = paste.deploy.appconfig('config:' + os.path.abspath(path)) import ckan ckan.config.environment.load_environment(conf.global_conf, conf.local_conf) registry = Registry() registry.prepare() translator_obj = MockTranslator() registry.register(pylons.translator, translator_obj)
def fake_request(pylons_config, server_name='mediacore.example', language='en', method='GET', request_uri='/', post_vars=None): app_globals = pylons_config['pylons.app_globals'] pylons.app_globals._push_object(app_globals) if post_vars and method.upper() != 'POST': raise ValueError( 'You must not specify post_vars for request method %r' % method) wsgi_environ = create_wsgi_environ('http://%s%s' % (server_name, request_uri), method.upper(), request_body=post_vars) request = Request(wsgi_environ, charset='utf-8') request.language = language request.settings = app_globals.settings pylons.request._push_object(request) response = Response(content_type='application/xml', charset='utf-8') pylons.response._push_object(response) session = SessionObject(wsgi_environ) pylons.session._push_object(session) routes_url = URLGenerator(pylons_config['routes.map'], wsgi_environ) pylons.url._push_object(routes_url) # Use ContextObj() when we get rid of 'pylons.strict_tmpl_context=False' in # mediacore.lib.environment tmpl_context = AttribSafeContextObj() tmpl_context.paginators = Bunch() pylons.tmpl_context._push_object(tmpl_context) # some parts of Pylons (e.g. Pylons.controllers.core.WSGIController) # use the '.c' alias instead. pylons.c = pylons.tmpl_context paste_registry = Registry() paste_registry.prepare() engines = create_tw_engine_manager(app_globals) host_framework = PylonsHostFramework(engines=engines) paste_registry.register(tw.framework, host_framework) setup_translator(language=language, registry=paste_registry) wsgi_environ.update({ 'pylons.pylons': pylons, 'paste.registry': paste_registry, }) return request
def __init__(self, worker_id, heartbeat_interval=60000, action='update'): super(UpdateIntentsWorker, self).__init__(worker_id, heartbeat_interval) self.intents_worker = UserIntentsWorker(intents_extractor) self.action = action # Workaround until the core translation function defaults to the Flask one from paste.registry import Registry from ckan.lib.cli import MockTranslator registry = Registry() registry.prepare() from pylons import translator registry.register(translator, MockTranslator())
def __init__(self): super(MockedJobController, self).__init__() # Register into paste the fake request and response objects registry = Registry() registry.prepare() registry.register(request, MockRequest()) registry.register(response, MockResponse()) # This is expected to be set by Pylons, so we set it here self._py_object = type( "", (), {"__init__": (lambda s, **kwargs: s.__dict__.update(kwargs))})( request=request, response=response) # Inject fake proxy, so the submission works delegated = Credential(dlg_id='12345', dn='/DN=1234', proxy='', termination_time=datetime.utcnow() + timedelta(hours=5)) Session.merge(delegated) Session.commit()
def __init__(self): super(MockedJobController, self).__init__() # Register into paste the fake request and response objects registry = Registry() registry.prepare() registry.register(request, MockRequest()) registry.register(response, MockResponse()) # This is expected to be set by Pylons, so we set it here self._py_object = type("", (), { "__init__": (lambda s, **kwargs: s.__dict__.update(kwargs)) })(request=request, response=response) # Inject fake proxy, so the submission works delegated = Credential( dlg_id='12345', dn='/DN=1234', proxy='', termination_time=datetime.utcnow() + timedelta(hours=5) ) Session.merge(delegated) Session.commit()
def load_translations(lang): # Register a translator in this thread so that # the _() functions in logic layer can work from paste.registry import Registry from pylons import translator from pylons import request registry = Registry() registry.prepare() class FakePylons: translator = None fakepylons = FakePylons() class FakeRequest: # Stores details of the translator environ = {'pylons.pylons': fakepylons} registry.register(request, FakeRequest()) # create translator i18n.set_lang(lang) # pull out translator and register it registry.register(translator, fakepylons.translator)
def command(input_csv, config_ini, commit=False): config_ini_filepath = os.path.abspath(config_ini) load_config(config_ini_filepath) engine = engine_from_config(config, 'sqlalchemy.') logging.config.fileConfig(config_ini_filepath) log = logging.getLogger(os.path.basename(__file__)) from ckan import model from ckan.logic import get_action from ckan.lib.cli import MockTranslator model.init_model(engine) registry=Registry() registry.prepare() translator_obj=MockTranslator() registry.register(translator, translator_obj) ctx = { 'model': model, 'session': model.Session, 'user': get_action('get_site_user')({'model': model,'ignore_auth': True}, {})['name'] } if commit: rev = model.repo.new_revision() packages_to_check = set() reader = csv.reader(open(input_csv, 'r')) for row in reader: # For each URL in the csv, get the list of resources referencing # that URL resources = model.Session.query(model.Resource)\ .filter(model.Resource.state=='active')\ .filter(model.Resource.url==row[0]).all() for resource in resources: # For each resource, add the package to the list packages_to_check.add(resource.get_package_id()) # Delete the resource resource.state = 'deleted' model.Session.add(resource) if commit: model.Session.commit() print "Deleted resource: {0}".format(resource.id) stats.increment("Deleted resource") for pid in packages_to_check: # for each package we need to check, see if it has any # resources left, it not, delete it. pkg = model.Package.get(pid) if len(pkg.resources) == 0: pkg.state = 'deleted' model.Session.add(pkg) if commit: model.Session.commit() stats.increment('Deleted packages') print "Deleted package: {0}".format(pkg.name) if commit: model.repo.commit_and_remove() else: print "" print '*' * 60 print "DON'T PANIC, this was a dry run, nothing was committed" print '*' * 60 print '' print '*' * 60, 'Deletion Report' print stats.report(order_by_title=True)