def test_CldfDownload(self): from clld.web.adapters.cldf import CldfDownload tmp = self.tmp_path('dl.zip') dl = CldfDownload(Dataset, 'clld') dl.create(self.env['request'], verbose=False, outfile=tmp) ds = CldfDataset.from_zip(tmp) self.assertEqual(ds.name, 'dataset-contribution-contribution') self.assertEqual( 'http://localhost/values/{ID}', ds.table.schema.aboutUrl) self.assertEqual( 'http://localhost/languages/{Language_ID}', ds.table.schema.columns['Language_ID'].valueUrl) self.assertEqual(len(ds.rows), 3) self.assertIn('Language_glottocode', ds[0]) self.assertIn('10-20', ds['value2']['Source'])
def test_CldfDownload(env, tmppath, mocker, capsys): from clld.web.adapters.cldf import CldfDownload mocker.patch('clld.web.adapters.cldf.transaction') tmp = tmppath / 'dl.zip' dl = CldfDownload(Dataset, 'clld') dl.create(env['request'], outfile=tmp, verbose=True) out, err = capsys.readouterr() assert 'Value' in out outdir = tmppath / 'cldf' with ZipFile(tmp.as_posix()) as zip: assert 'Wordlist-metadata.json' in zip.namelist() zip.extractall(str(outdir)) ds = CldfDataset.from_metadata(outdir.joinpath('Wordlist-metadata.json')) assert ds.module == 'Wordlist' values = list(ds[ds.primary_table]) assert len(values) == 3 for v in values: list(ds.sources.expand_refs(v['Source']))
def includeme(config): """Upgrading: - register utilities "by hand", after config.include('clld.web.app') - add routes by hand (and remove these from the **kw passed to Configurator) :param config: :return: """ # # now we exploit the default package layout as created via the CLLD scaffold: # # note: the following exploits the import time side effect of modifying the webassets # environment! root_package = config.root_package.__name__ pkg_dir = Path(config.root_package.__file__).parent.resolve() maybe_import('%s.assets' % root_package, pkg_dir=pkg_dir) json_renderer = JSON() json_renderer.add_adapter(datetime.datetime, lambda obj, req: obj.isoformat()) json_renderer.add_adapter(datetime.date, lambda obj, req: obj.isoformat()) config.add_renderer('json', json_renderer) jsonp_renderer = JSONP(param_name='callback') jsonp_renderer.add_adapter(datetime.datetime, lambda obj, req: obj.isoformat()) jsonp_renderer.add_adapter(datetime.date, lambda obj, req: obj.isoformat()) config.add_renderer('jsonp', jsonp_renderer) config.set_request_factory(ClldRequest) config.registry.registerUtility(CtxFactoryQuery(), interfaces.ICtxFactoryQuery) config.registry.registerUtility(OlacConfig(), interfaces.IOlacConfig) config.registry.registerUtility(CldfConfig(), interfaces.ICldfConfig) # initialize the db connection engine = engine_from_config(config.registry.settings, 'sqlalchemy.') DBSession.configure(bind=engine) Base.metadata.bind = engine try: git_tag = git_describe(Path(pkg_dir).parent) except ValueError: # pragma: no cover git_tag = None config.add_settings({ 'pyramid.default_locale_name': 'en', 'clld.pkg': root_package, 'clld.git_tag': git_tag, 'clld.parameters': {} }) if 'clld.files' in config.registry.settings: # deployment-specific location of static data files abspath = Path(config.registry.settings['clld.files']).resolve() config.add_settings({'clld.files': abspath}) config.add_static_view('files', str(abspath)) # event subscribers: config.add_subscriber(add_localizer, events.NewRequest) config.add_subscriber(init_map, events.ContextFound) config.add_subscriber( partial(add_renderer_globals, maybe_import('%s.util' % root_package, pkg_dir=pkg_dir)), events.BeforeRender) # # make it easy to register custom functionality # for name, func in { 'register_utility': register_utility, 'register_datatable': partial(register_cls, interfaces.IDataTable), 'register_map': partial(register_cls, interfaces.IMap), 'register_menu': register_menu, 'register_resource': register_resource, 'register_adapter': register_adapter, 'register_adapters': register_adapters, 'register_download': register_download, 'register_staticresource': register_staticresource, 'add_route_and_view': add_route_and_view, 'add_settings_from_file': add_settings_from_file, 'add_301': add_301, 'add_410': add_410, 'add_page': add_page, 'register_resource_routes_and_views': register_resource_routes_and_views, }.items(): config.add_directive(name, func) # # routes and views # config.add_static_view('clld-static', 'clld:web/static') config.add_static_view('static', '%s:static' % root_package) config.add_route_and_view('_js', '/_js', js, http_cache=3600) # add some maintenance hatches config.add_route_and_view('_raise', '/_raise', _raise) config.add_route_and_view('_ping', '/_ping', _ping, renderer='json') # sitemap support: config.add_route_and_view('robots', '/robots.txt', robots) config.add_route_and_view('sitemapindex', '/sitemap.xml', sitemapindex) config.add_route_and_view('sitemap', '/sitemap.{rsc}.{n}.xml', sitemap) config.add_route('resourcemap', '/resourcemap.json') config.add_view(resourcemap, route_name='resourcemap', renderer='jsonp') config.add_route_and_view('select_combination', '/_select_combination', select_combination) config.add_route_and_view('unapi', '/unapi', unapi) config.add_route_and_view('olac', '/olac', olac) config.add_settings_from_file(pkg_dir.joinpath('appconf.ini')) if not config.registry.settings.get('mako.directories'): config.add_settings({'mako.directories': ['clld:web/templates']}) for rsc in RESOURCES: config.register_resource_routes_and_views(rsc) config.register_datatable( rsc.plural, getattr(datatables, rsc.plural.capitalize(), DataTable)) register_resource_adapters(config, rsc) # maps config.register_map('languages', Map) config.register_map('language', LanguageMap) config.register_map('parameter', ParameterMap) config.register_map('combination', CombinationMap) config.include('clld.web.adapters') for icon in ICONS: config.registry.registerUtility(icon, interfaces.IIcon, name=icon.name) config.registry.registerUtility(ORDERED_ICONS, interfaces.IIconList) config.registry.registerUtility(MapMarker(), interfaces.IMapMarker) # # inspect default locations for views and templates: # home_comp = OrderedDict() for name, template in [ ('introduction', False), ('about', False), ('terms', False), ('glossary', False), ('history', False), ('changes', False), ('credits', False), ('legal', True), ('download', True), ('contact', True), ('help', False), ]: home_comp[name] = template if pkg_dir.joinpath('templates').exists(): for p in pkg_dir.joinpath('templates').iterdir(): if p.stem in home_comp and p.suffix == '.mako': home_comp[p.stem] = True for name, template in home_comp.items(): if template: config.add_page(name) config.add_settings( {'home_comp': [k for k in home_comp.keys() if home_comp[k]]}) if 'clld.favicon' not in config.registry.settings: favicon = {'clld.favicon': 'clld:web/static/images/favicon.ico'} # hard to test (in particular on travis) and without too much consequence # (and the consequences faced are easy to spot). if pkg_dir.joinpath('static', 'favicon.ico').exists(): # pragma: no cover favicon['clld.favicon'] = root_package + ':static/favicon.ico' config.add_settings(favicon) config.add_settings({ 'clld.favicon_hash': md5(abspath_from_asset_spec(config.registry.settings['clld.favicon'])) }) translation_dirs = ['clld:locale'] if pkg_dir.joinpath('locale').exists(): translation_dirs.append('%s:locale' % root_package) # pragma: no cover config.add_translation_dirs(*translation_dirs) if pkg_dir.joinpath( 'static/publisher_logo.png').exists(): # pragma: no cover config.add_settings({ 'clld.publisher_logo': '%s:static/publisher_logo.png' % root_package }) if asbool(config.registry.settings.get('clld.pacific_centered_maps')): geojson.pacific_centered() v = maybe_import('%s.views' % root_package, pkg_dir=pkg_dir) if v: config.scan(v) # pragma: no cover menuitems = config.registry.settings.get( 'clld.menuitems_list', ['contributions', 'parameters', 'languages', 'contributors']) config.register_menu(('dataset', dict(label='Home')), *menuitems) config.include('pyramid_mako') for name in ['adapters', 'datatables', 'maps']: mod = maybe_import('%s.%s' % (root_package, name), pkg_dir=pkg_dir) if mod and hasattr(mod, 'includeme'): config.include(mod) config.register_download(CldfDownload(common.Dataset, root_package))
def testDownload2(self): from clld.web.adapters.download import CsvDump, N3Dump, RdfXmlDump from clld.web.adapters.cldf import CldfDownload tmp = mktemp() class Path(MagicMock, UnicodeMixin): @property def stem(self): return 'a' @property def parent(self): return Mock(exists=Mock(return_value=False)) def open(self, mode): return open(tmp, mode) with patch.multiple( 'clld.web.adapters.cldf', ZipFile=MagicMock(), Path=MagicMock(return_value=Path()), move=Mock(), remove=Mock(), ): with patch( 'clld.web.adapters.download.Path', new=MagicMock(return_value=Path()), ): dl = CldfDownload(Dataset, 'clld') dl.create(self.env['request'], verbose=False) with patch.multiple( 'clld.web.adapters.download', ZipFile=MagicMock(), Path=MagicMock(return_value=Path()), move=Mock(), remove=Mock(), ): dl = CsvDump(Language, 'clld') dl.create(self.env['request'], verbose=False) dl.create(self.env['request'], filename='name.n3', verbose=False) dl = N3Dump(Language, 'clld') dl.create(self.env['request'], verbose=False) if os.path.exists(tmp): os.remove(tmp) else: # pragma: no cover raise ValueError dl = RdfXmlDump(Language, 'clld') dl.create(self.env['request'], verbose=False) with closing(gzip.open(tmp, 'rb')) as fp: assert et.fromstring(fp.read()) if os.path.exists(tmp): os.remove(tmp) else: # pragma: no cover raise ValueError