def test_init_with_file(tmpdir):
extension_versions_file = tmpdir.join('extension_versions.csv')
extension_versions_file.write(extension_versions_data)
extensions_file = tmpdir.join('extensions.csv')
extensions_file.write(extensions_data)
obj = ExtensionRegistry(Path(extension_versions_file).as_uri(), Path(extensions_file).as_uri())
assert len(obj.versions) == 14
assert obj.versions[0].as_dict() == {
'id': 'charges',
'date': '',
'version': 'master',
'base_url': 'https://raw.githubusercontent.com/open-contracting-extensions/ocds_charges_extension/master/',
'download_url': 'https://github.com/open-contracting-extensions/ocds_charges_extension/archive/master.zip',
'category': 'ppp',
'core': False,
}
# Assume intermediate data is correctly parsed.
assert obj.versions[-1].as_dict() == {
'id': 'lots',
'date': '2018-01-30',
'version': 'v1.1.3',
'base_url': 'https://raw.githubusercontent.com/open-contracting-extensions/ocds_lots_extension/v1.1.3/',
'download_url': 'https://api.github.com/repos/open-contracting-extensions/ocds_lots_extension/zipball/v1.1.3',
'category': 'tender',
'core': True,
}
def download_extensions(ctx, path):
path = path.rstrip('/')
url = 'https://raw.githubusercontent.com/open-contracting/extension_registry/master/extension_versions.csv'
registry = ExtensionRegistry(url)
for extension in registry:
directory = os.path.join(path, extension.repository_name)
if not os.path.isdir(directory):
run('git clone {} {}'.format(extension.repository_url, directory))
def test_get():
obj = ExtensionRegistry(extension_versions_data)
result = obj.get(id='lots', version='v1.1.3')
assert result.as_dict() == {
'id': 'lots',
'date': '2018-01-30',
'version': 'v1.1.3',
'base_url': 'https://raw.githubusercontent.com/open-contracting-extensions/ocds_lots_extension/v1.1.3/',
'download_url': 'https://api.github.com/repos/open-contracting-extensions/ocds_lots_extension/zipball/v1.1.3',
}
def download_extensions(path):
"""
Download all registered extensions to a directory.
"""
path = path.rstrip('/')
registry = ExtensionRegistry(extension_versions_url)
for version in registry:
directory = os.path.join(path, version.repository_name)
if not os.path.isdir(directory):
os.system(f'git clone {version.repository_url} {directory}')
def set_topics():
"""
Add topics to repositories in the open-contracting-extensions organization.
- ocds-extension
- ocds-core-extension
- ocds-community-extension
- ocds-profile
- european-union
- public-private-partnerships
"""
format_string = 'https://raw.githubusercontent.com/open-contracting-extensions/{}/{}/docs/extension_versions.json'
profiles = defaultdict(list)
for profile, branch in (('european-union', 'latest'),
('public-private-partnerships', '1.0-dev')):
extension_versions = requests.get(format_string.format(
profile, branch)).json()
for extension_id in extension_versions.keys():
profiles[extension_id].append(profile)
registry = ExtensionRegistry(extension_versions_url, extensions_url)
repos = requests.get(
'https://api.github.com/orgs/open-contracting-extensions/repos?per_page=100'
).json()
for repo in repos:
topics = []
if repo['name'].endswith('_extension'):
topics.append('ocds-extension')
else:
topics.append('ocds-profile')
for version in registry:
if f"/{repo['full_name']}/" in version.base_url:
if version.core:
topics.append('ocds-core-extension')
else:
topics.append('ocds-community-extension')
topics.extend(profiles[version.id])
break
else:
if 'ocds-profile' not in topics:
click.echo(f"{repo['name']} is not registered")
requests.put(
f"https://api.github.com/repos/{repo['full_name']}/topics",
data=json.dumps({'names': topics}),
headers={
'accept': 'application/vnd.github.mercy-preview+json'
}).raise_for_status()
def test_init_with_versions_only():
obj = ExtensionRegistry(extension_versions_data)
assert len(obj.versions) == 14
assert obj.versions[0].as_dict() == {
'id': 'charges',
'date': '',
'version': 'master',
'base_url': 'https://raw.githubusercontent.com/open-contracting-extensions/ocds_charges_extension/master/',
'download_url': 'https://github.com/open-contracting-extensions/ocds_charges_extension/archive/master.zip',
}
# Assume intermediate data is correctly parsed.
assert obj.versions[-1].as_dict() == {
'id': 'lots',
'date': '2018-01-30',
'version': 'v1.1.3',
'base_url': 'https://raw.githubusercontent.com/open-contracting-extensions/ocds_lots_extension/v1.1.3/',
'download_url': 'https://api.github.com/repos/open-contracting-extensions/ocds_lots_extension/zipball/v1.1.3',
}
def __init__(self,
standard_tag,
extension_versions,
registry_base_url=None,
schema_base_url=None):
"""
Accepts an OCDS version and a dictionary of extension identifiers and versions, and initializes a reader of the
extension registry.
"""
self.standard_tag = standard_tag
self.extension_versions = extension_versions
self._file_cache = {}
self.schema_base_url = schema_base_url
# Allows setting the registry URL to e.g. a pull request, when working on a profile.
if not registry_base_url:
registry_base_url = 'https://raw.githubusercontent.com/open-contracting/extension_registry/master/'
self.registry = ExtensionRegistry(registry_base_url +
'extension_versions.csv')
def versions(self):
registry = ExtensionRegistry(self.args.extension_versions_url,
self.args.extensions_url)
versions = defaultdict(list)
for value in self.args.versions:
if '==' in value:
extension, version = value.split('==', 1)
versions[extension].append(version)
elif '=' in value:
# Help users with a common error.
raise CommandError(
f"Couldn't parse '{value}'. Use '==' not '='.")
else:
versions[value]
for version in registry:
if ((not self.args.versions or version.id in versions)
and (not versions[version.id]
or version.version in versions[version.id])):
yield version
def test_filter():
obj = ExtensionRegistry(extension_versions_data, extensions_data)
result = obj.filter(core=True, version='v1.1.3', category='tender')
assert len(result) == 2
assert result[0].as_dict() == {
'id': 'enquiries',
'date': '2018-02-01',
'version': 'v1.1.3',
'base_url': 'https://raw.githubusercontent.com/open-contracting-extensions/ocds_enquiry_extension/v1.1.3/',
'download_url': 'https://api.github.com/repos/open-contracting-extensions/ocds_enquiry_extension/zipball/v1.1.3', # noqa: E501
'category': 'tender',
'core': True,
}
assert result[1].as_dict() == {
'id': 'lots',
'date': '2018-01-30',
'version': 'v1.1.3',
'base_url': 'https://raw.githubusercontent.com/open-contracting-extensions/ocds_lots_extension/v1.1.3/',
'download_url': 'https://api.github.com/repos/open-contracting-extensions/ocds_lots_extension/zipball/v1.1.3',
'category': 'tender',
'core': True,
}
def run(self):
config = self.state.document.settings.env.config
extension_versions = config.extension_versions
language = config.overrides.get('language', 'en')
extension_list_name = self.options.pop('list', '')
set_classes(self.options)
admonition_node = nodes.admonition('', **self.options)
self.add_name(admonition_node)
title_text = self.arguments[0]
textnodes, _ = self.state.inline_text(title_text, self.lineno)
title = nodes.title(title_text, '', *textnodes)
title.line = 0
title.source = 'extension_list_' + extension_list_name
admonition_node += title
if 'classes' not in self.options:
admonition_node['classes'] += ['admonition', 'note']
admonition_node['classes'] += ['extension_list']
admonition_node['ids'] += ['extensionlist-' + extension_list_name]
definition_list = nodes.definition_list()
definition_list.line = 0
# Only list core extensions whose version matches the version specified in `conf.py` and whose category matches
# the category specified by the directive's `list` option.
registry = ExtensionRegistry(extension_versions_url, extensions_url)
num = 0
for identifier, version in extension_versions.items():
extension = registry.get(id=identifier, core=True, version=version)
if extension_list_name and extension.category != extension_list_name:
continue
# Avoid "403 Client Error: rate limit exceeded for url" on development branches.
try:
metadata = extension.metadata
except requests.exceptions.HTTPError:
if live_branch:
raise
metadata = {
'name': {
'en': identifier
},
'description': {
'en': identifier
}
}
name = metadata['name']['en']
description = metadata['description']['en']
some_term, _ = self.state.inline_text(name, self.lineno)
some_def, _ = self.state.inline_text(description, self.lineno)
link = nodes.reference(name, '', *some_term)
link['refuri'] = extension_explorer_template.format(
language, identifier, version)
link['translatable'] = True
link.source = 'extension_list_' + extension_list_name
link.line = num + 1
term = nodes.term(name, '', link)
definition_list += term
text = nodes.paragraph(description, '', *some_def)
text.source = 'extension_list_' + extension_list_name
text.line = num + 1
definition_list += nodes.definition(description, text)
if extension_list_name and not registry.filter(
category=extension_list_name):
raise self.warning(
f'No extensions have category {extension_list_name} in extensionlist directive'
)
admonition_node += definition_list
community = "The following are community extensions and are not maintained by Open Contracting Partnership."
community_text, _ = self.state.inline_text(community, self.lineno)
community_paragraph = nodes.paragraph(community, *community_text)
community_paragraph['classes'] += ['hide']
community_paragraph.source = 'extension_list_' + extension_list_name
community_paragraph.line = num + 2
admonition_node += community_paragraph
return [admonition_node]
def test_filter_invalid():
obj = ExtensionRegistry(extension_versions_data)
with pytest.raises(AttributeError) as excinfo:
obj.filter(invalid='invalid')
assert str(excinfo.value) == "'ExtensionVersion' object has no attribute 'invalid'"
def test_get_no_match():
obj = ExtensionRegistry(extension_versions_data)
with pytest.raises(DoesNotExist) as excinfo:
obj.get(id='nonexistent')
assert str(excinfo.value) == "Extension version matching {'id': 'nonexistent'} does not exist."
"extensions": ["{}extension.json"],
"releases": []
}}
```
This extension is maintained at <{}>
## Documentation
"""
extensions_path = os.path.join(docs_path, 'extensions')
extensions_url = 'https://raw.githubusercontent.com/open-contracting/extension_registry/master/extensions.csv'
extension_versions_url = 'https://raw.githubusercontent.com/open-contracting/extension_registry/master/extension_versions.csv' # noqa
extension_registry = ExtensionRegistry(extension_versions_url, extensions_url)
# At present, all core extensions are included in the standard's documentation.
for version in extension_registry.filter(core=True):
if version.id not in extension_versions:
raise Exception(
'{} is a core extension but is not included in the standard'.
format(version.id))
for identifier, version in extension_versions.items():
extension = extension_registry.get(id=identifier, version=version)
lines = extension.remote('README.md').split('\n')
heading = '\n'.join(lines[:1])
body = '\n'.join(lines[1:])
body = body.replace('\n##', '\n###')
text = heading + metadata.format(extension.base_url,
def load_extensions_info(self):
"""
Gets the core extensions from the extension registry.
If the language is not 'en', produces the translated versions for each core extension.
"""
extensions_dir = Path('extensions')
if extensions_dir.exists():
shutil.rmtree(str(extensions_dir))
extensions_dir.mkdir()
# download extension files according to version
registry = ExtensionRegistry(self.extension_versions_url, self.extensions_url)
for version in registry.filter(core=True, version=self.version):
if version.id not in self.exclusions:
zip_file = version.zipfile()
zip_file.extractall(path=str(extensions_dir))
# rename path to extension id
path = extensions_dir / zip_file.infolist()[0].filename
path.rename(extensions_dir / version.id)
if self.lang is 'en':
output_dir = extensions_dir
else:
# translate core extensions
translation_sources_dir = Path('ocds-extensions-translations-master')
if translation_sources_dir.exists():
shutil.rmtree(str(translation_sources_dir))
res = requests.get('https://github.com/open-contracting/ocds-extensions-translations/archive/master.zip')
res.raise_for_status()
content = BytesIO(res.content)
with ZipFile(content) as zipfile:
zipfile.extractall()
output_dir = Path('translations') / self.lang
if output_dir.exists():
shutil.rmtree(str(output_dir))
output_dir.mkdir(parents=True)
locale = str(translation_sources_dir / 'locale')
headers = ['Title', 'Description', 'Extension']
for dir in [x for x in extensions_dir.iterdir() if x.is_dir()]:
translate([
(glob(str(dir / 'extension.json')), output_dir / dir.parts[-1], dir.parts[-1] + '/' + self.version + '/schema'),
(glob(str(dir / 'release-schema.json')),
output_dir / dir.parts[-1],
dir.parts[-1] + '/' + self.version + '/schema')
], locale, self.lang, headers)
self.extension_urls = [path.resolve(strict=True).as_uri() for path in output_dir.iterdir() if path.is_dir()]
# get names and descriptions for each extension
for dir in [x for x in output_dir.iterdir() if x.is_dir()]:
path = dir.joinpath('extension.json')
info = {}
with path.open() as f:
info = json.load(f)
self.descriptions[info['name'][self.lang]] = info['description'][self.lang]
# mapping-schema looks for the name of the name extension in English
if self.lang is not 'en':
info['name']['en'] = info['name'][self.lang]
with path.open(mode='w') as f:
json.dump(info, f)
return self.extension_urls
def test_init_with_url():
obj = ExtensionRegistry(extension_versions_url, extensions_url)
assert len(obj.versions) > 50
def test_iter():
obj = ExtensionRegistry(extension_versions_data)
for i, version in enumerate(obj, 1):
pass
assert i == 14
def test_get_without_extensions():
obj = ExtensionRegistry(extension_versions_data)
with pytest.raises(MissingExtensionMetadata) as excinfo:
obj.get(category='tender')
assert str(excinfo.value) == 'ExtensionRegistry must be initialized with extensions data.'
def test_get_from_url_no_match():
obj = ExtensionRegistry(extension_versions_data)
with pytest.raises(DoesNotExist) as excinfo:
obj.get_from_url('http://example.com')
assert str(excinfo.value) == "Extension version matching {'base_url': 'http://example.com/'} does not exist."