def test_CldfDownload(self):
from clld.web.adapters.cldf import CldfDownload
tmp = self.tmp_path('dl.zip')
dl = CldfDownload(Dataset, 'clld')
dl.create(self.env['request'], verbose=False, outfile=tmp)
ds = CldfDataset.from_zip(tmp)
self.assertEqual(ds.name, 'dataset-contribution-contribution')
self.assertEqual(
'http://localhost/values/{ID}',
ds.table.schema.aboutUrl)
self.assertEqual(
'http://localhost/languages/{Language_ID}',
ds.table.schema.columns['Language_ID'].valueUrl)
self.assertEqual(len(ds.rows), 3)
self.assertIn('Language_glottocode', ds[0])
self.assertIn('10-20', ds['value2']['Source'])
def test_CldfDownload(env, tmppath, mocker, capsys):
from clld.web.adapters.cldf import CldfDownload
mocker.patch('clld.web.adapters.cldf.transaction')
tmp = tmppath / 'dl.zip'
dl = CldfDownload(Dataset, 'clld')
dl.create(env['request'], outfile=tmp, verbose=True)
out, err = capsys.readouterr()
assert 'Value' in out
outdir = tmppath / 'cldf'
with ZipFile(tmp.as_posix()) as zip:
assert 'Wordlist-metadata.json' in zip.namelist()
zip.extractall(str(outdir))
ds = CldfDataset.from_metadata(outdir.joinpath('Wordlist-metadata.json'))
assert ds.module == 'Wordlist'
values = list(ds[ds.primary_table])
assert len(values) == 3
for v in values:
list(ds.sources.expand_refs(v['Source']))
def test_CldfDownload(env, tmppath, mocker, capsys):
from clld.web.adapters.cldf import CldfDownload
mocker.patch('clld.web.adapters.cldf.transaction')
tmp = tmppath / 'dl.zip'
dl = CldfDownload(Dataset, 'clld')
dl.create(env['request'], outfile=tmp, verbose=True)
out, err = capsys.readouterr()
assert 'Value' in out
outdir = tmppath / 'cldf'
with ZipFile(tmp.as_posix()) as zip:
assert 'Wordlist-metadata.json' in zip.namelist()
zip.extractall(str(outdir))
ds = CldfDataset.from_metadata(outdir.joinpath('Wordlist-metadata.json'))
assert ds.module == 'Wordlist'
values = list(ds[ds.primary_table])
assert len(values) == 3
for v in values:
list(ds.sources.expand_refs(v['Source']))
def includeme(config):
"""Upgrading:
- register utilities "by hand", after config.include('clld.web.app')
- add routes by hand (and remove these from the **kw passed to Configurator)
:param config:
:return:
"""
#
# now we exploit the default package layout as created via the CLLD scaffold:
#
# note: the following exploits the import time side effect of modifying the webassets
# environment!
root_package = config.root_package.__name__
pkg_dir = Path(config.root_package.__file__).parent.resolve()
maybe_import('%s.assets' % root_package, pkg_dir=pkg_dir)
json_renderer = JSON()
json_renderer.add_adapter(datetime.datetime,
lambda obj, req: obj.isoformat())
json_renderer.add_adapter(datetime.date, lambda obj, req: obj.isoformat())
config.add_renderer('json', json_renderer)
jsonp_renderer = JSONP(param_name='callback')
jsonp_renderer.add_adapter(datetime.datetime,
lambda obj, req: obj.isoformat())
jsonp_renderer.add_adapter(datetime.date, lambda obj, req: obj.isoformat())
config.add_renderer('jsonp', jsonp_renderer)
config.set_request_factory(ClldRequest)
config.registry.registerUtility(CtxFactoryQuery(),
interfaces.ICtxFactoryQuery)
config.registry.registerUtility(OlacConfig(), interfaces.IOlacConfig)
config.registry.registerUtility(CldfConfig(), interfaces.ICldfConfig)
# initialize the db connection
engine = engine_from_config(config.registry.settings, 'sqlalchemy.')
DBSession.configure(bind=engine)
Base.metadata.bind = engine
try:
git_tag = git_describe(Path(pkg_dir).parent)
except ValueError: # pragma: no cover
git_tag = None
config.add_settings({
'pyramid.default_locale_name': 'en',
'clld.pkg': root_package,
'clld.git_tag': git_tag,
'clld.parameters': {}
})
if 'clld.files' in config.registry.settings:
# deployment-specific location of static data files
abspath = Path(config.registry.settings['clld.files']).resolve()
config.add_settings({'clld.files': abspath})
config.add_static_view('files', str(abspath))
# event subscribers:
config.add_subscriber(add_localizer, events.NewRequest)
config.add_subscriber(init_map, events.ContextFound)
config.add_subscriber(
partial(add_renderer_globals,
maybe_import('%s.util' % root_package, pkg_dir=pkg_dir)),
events.BeforeRender)
#
# make it easy to register custom functionality
#
for name, func in {
'register_utility': register_utility,
'register_datatable': partial(register_cls, interfaces.IDataTable),
'register_map': partial(register_cls, interfaces.IMap),
'register_menu': register_menu,
'register_resource': register_resource,
'register_adapter': register_adapter,
'register_adapters': register_adapters,
'register_download': register_download,
'register_staticresource': register_staticresource,
'add_route_and_view': add_route_and_view,
'add_settings_from_file': add_settings_from_file,
'add_301': add_301,
'add_410': add_410,
'add_page': add_page,
'register_resource_routes_and_views':
register_resource_routes_and_views,
}.items():
config.add_directive(name, func)
#
# routes and views
#
config.add_static_view('clld-static', 'clld:web/static')
config.add_static_view('static', '%s:static' % root_package)
config.add_route_and_view('_js', '/_js', js, http_cache=3600)
# add some maintenance hatches
config.add_route_and_view('_raise', '/_raise', _raise)
config.add_route_and_view('_ping', '/_ping', _ping, renderer='json')
# sitemap support:
config.add_route_and_view('robots', '/robots.txt', robots)
config.add_route_and_view('sitemapindex', '/sitemap.xml', sitemapindex)
config.add_route_and_view('sitemap', '/sitemap.{rsc}.{n}.xml', sitemap)
config.add_route('resourcemap', '/resourcemap.json')
config.add_view(resourcemap, route_name='resourcemap', renderer='jsonp')
config.add_route_and_view('select_combination', '/_select_combination',
select_combination)
config.add_route_and_view('unapi', '/unapi', unapi)
config.add_route_and_view('olac', '/olac', olac)
config.add_settings_from_file(pkg_dir.joinpath('appconf.ini'))
if not config.registry.settings.get('mako.directories'):
config.add_settings({'mako.directories': ['clld:web/templates']})
for rsc in RESOURCES:
config.register_resource_routes_and_views(rsc)
config.register_datatable(
rsc.plural, getattr(datatables, rsc.plural.capitalize(),
DataTable))
register_resource_adapters(config, rsc)
# maps
config.register_map('languages', Map)
config.register_map('language', LanguageMap)
config.register_map('parameter', ParameterMap)
config.register_map('combination', CombinationMap)
config.include('clld.web.adapters')
for icon in ICONS:
config.registry.registerUtility(icon, interfaces.IIcon, name=icon.name)
config.registry.registerUtility(ORDERED_ICONS, interfaces.IIconList)
config.registry.registerUtility(MapMarker(), interfaces.IMapMarker)
#
# inspect default locations for views and templates:
#
home_comp = OrderedDict()
for name, template in [
('introduction', False),
('about', False),
('terms', False),
('glossary', False),
('history', False),
('changes', False),
('credits', False),
('legal', True),
('download', True),
('contact', True),
('help', False),
]:
home_comp[name] = template
if pkg_dir.joinpath('templates').exists():
for p in pkg_dir.joinpath('templates').iterdir():
if p.stem in home_comp and p.suffix == '.mako':
home_comp[p.stem] = True
for name, template in home_comp.items():
if template:
config.add_page(name)
config.add_settings(
{'home_comp': [k for k in home_comp.keys() if home_comp[k]]})
if 'clld.favicon' not in config.registry.settings:
favicon = {'clld.favicon': 'clld:web/static/images/favicon.ico'}
# hard to test (in particular on travis) and without too much consequence
# (and the consequences faced are easy to spot).
if pkg_dir.joinpath('static',
'favicon.ico').exists(): # pragma: no cover
favicon['clld.favicon'] = root_package + ':static/favicon.ico'
config.add_settings(favicon)
config.add_settings({
'clld.favicon_hash':
md5(abspath_from_asset_spec(config.registry.settings['clld.favicon']))
})
translation_dirs = ['clld:locale']
if pkg_dir.joinpath('locale').exists():
translation_dirs.append('%s:locale' % root_package) # pragma: no cover
config.add_translation_dirs(*translation_dirs)
if pkg_dir.joinpath(
'static/publisher_logo.png').exists(): # pragma: no cover
config.add_settings({
'clld.publisher_logo':
'%s:static/publisher_logo.png' % root_package
})
if asbool(config.registry.settings.get('clld.pacific_centered_maps')):
geojson.pacific_centered()
v = maybe_import('%s.views' % root_package, pkg_dir=pkg_dir)
if v:
config.scan(v) # pragma: no cover
menuitems = config.registry.settings.get(
'clld.menuitems_list',
['contributions', 'parameters', 'languages', 'contributors'])
config.register_menu(('dataset', dict(label='Home')), *menuitems)
config.include('pyramid_mako')
for name in ['adapters', 'datatables', 'maps']:
mod = maybe_import('%s.%s' % (root_package, name), pkg_dir=pkg_dir)
if mod and hasattr(mod, 'includeme'):
config.include(mod)
config.register_download(CldfDownload(common.Dataset, root_package))
def testDownload2(self):
from clld.web.adapters.download import CsvDump, N3Dump, RdfXmlDump
from clld.web.adapters.cldf import CldfDownload
tmp = mktemp()
class Path(MagicMock, UnicodeMixin):
@property
def stem(self):
return 'a'
@property
def parent(self):
return Mock(exists=Mock(return_value=False))
def open(self, mode):
return open(tmp, mode)
with patch.multiple(
'clld.web.adapters.cldf',
ZipFile=MagicMock(),
Path=MagicMock(return_value=Path()),
move=Mock(),
remove=Mock(),
):
with patch(
'clld.web.adapters.download.Path',
new=MagicMock(return_value=Path()),
):
dl = CldfDownload(Dataset, 'clld')
dl.create(self.env['request'], verbose=False)
with patch.multiple(
'clld.web.adapters.download',
ZipFile=MagicMock(),
Path=MagicMock(return_value=Path()),
move=Mock(),
remove=Mock(),
):
dl = CsvDump(Language, 'clld')
dl.create(self.env['request'], verbose=False)
dl.create(self.env['request'], filename='name.n3', verbose=False)
dl = N3Dump(Language, 'clld')
dl.create(self.env['request'], verbose=False)
if os.path.exists(tmp):
os.remove(tmp)
else: # pragma: no cover
raise ValueError
dl = RdfXmlDump(Language, 'clld')
dl.create(self.env['request'], verbose=False)
with closing(gzip.open(tmp, 'rb')) as fp:
assert et.fromstring(fp.read())
if os.path.exists(tmp):
os.remove(tmp)
else: # pragma: no cover
raise ValueError
def testDownload2(self):
from clld.web.adapters.download import CsvDump, N3Dump, RdfXmlDump
from clld.web.adapters.cldf import CldfDownload
tmp = mktemp()
class Path(MagicMock, UnicodeMixin):
@property
def stem(self):
return 'a'
@property
def parent(self):
return Mock(exists=Mock(return_value=False))
def open(self, mode):
return open(tmp, mode)
with patch.multiple(
'clld.web.adapters.cldf',
ZipFile=MagicMock(),
Path=MagicMock(return_value=Path()),
move=Mock(),
remove=Mock(),
):
with patch(
'clld.web.adapters.download.Path',
new=MagicMock(return_value=Path()),
):
dl = CldfDownload(Dataset, 'clld')
dl.create(self.env['request'], verbose=False)
with patch.multiple(
'clld.web.adapters.download',
ZipFile=MagicMock(),
Path=MagicMock(return_value=Path()),
move=Mock(),
remove=Mock(),
):
dl = CsvDump(Language, 'clld')
dl.create(self.env['request'], verbose=False)
dl.create(self.env['request'], filename='name.n3', verbose=False)
dl = N3Dump(Language, 'clld')
dl.create(self.env['request'], verbose=False)
if os.path.exists(tmp):
os.remove(tmp)
else: # pragma: no cover
raise ValueError
dl = RdfXmlDump(Language, 'clld')
dl.create(self.env['request'], verbose=False)
with closing(gzip.open(tmp, 'rb')) as fp:
assert et.fromstring(fp.read())
if os.path.exists(tmp):
os.remove(tmp)
else: # pragma: no cover
raise ValueError
