def test_get():
obj = ExtensionRegistry(extension_versions_data)
result = obj.get(id='lots', version='v1.1.3')
assert result.as_dict() == {
'id': 'lots',
'date': '2018-01-30',
'version': 'v1.1.3',
'base_url': 'https://raw.githubusercontent.com/open-contracting-extensions/ocds_lots_extension/v1.1.3/',
'download_url': 'https://api.github.com/repos/open-contracting-extensions/ocds_lots_extension/zipball/v1.1.3',
}
def test_init_with_file(tmpdir):
extension_versions_file = tmpdir.join('extension_versions.csv')
extension_versions_file.write(extension_versions_data)
extensions_file = tmpdir.join('extensions.csv')
extensions_file.write(extensions_data)
obj = ExtensionRegistry(Path(extension_versions_file).as_uri(), Path(extensions_file).as_uri())
assert len(obj.versions) == 14
assert obj.versions[0].as_dict() == {
'id': 'charges',
'date': '',
'version': 'master',
'base_url': 'https://raw.githubusercontent.com/open-contracting-extensions/ocds_charges_extension/master/',
'download_url': 'https://github.com/open-contracting-extensions/ocds_charges_extension/archive/master.zip',
'category': 'ppp',
'core': False,
}
# Assume intermediate data is correctly parsed.
assert obj.versions[-1].as_dict() == {
'id': 'lots',
'date': '2018-01-30',
'version': 'v1.1.3',
'base_url': 'https://raw.githubusercontent.com/open-contracting-extensions/ocds_lots_extension/v1.1.3/',
'download_url': 'https://api.github.com/repos/open-contracting-extensions/ocds_lots_extension/zipball/v1.1.3',
'category': 'tender',
'core': True,
}
def download_extensions(ctx, path):
path = path.rstrip('/')
url = 'https://raw.githubusercontent.com/open-contracting/extension_registry/master/extension_versions.csv'
registry = ExtensionRegistry(url)
for extension in registry:
directory = os.path.join(path, extension.repository_name)
if not os.path.isdir(directory):
run('git clone {} {}'.format(extension.repository_url, directory))
def download_extensions(path):
"""
Download all registered extensions to a directory.
"""
path = path.rstrip('/')
registry = ExtensionRegistry(extension_versions_url)
for version in registry:
directory = os.path.join(path, version.repository_name)
if not os.path.isdir(directory):
os.system(f'git clone {version.repository_url} {directory}')
def __init__(self,
standard_tag,
extension_versions,
registry_base_url=None,
schema_base_url=None):
"""
Accepts an OCDS version and a dictionary of extension identifiers and versions, and initializes a reader of the
extension registry.
"""
self.standard_tag = standard_tag
self.extension_versions = extension_versions
self._file_cache = {}
self.schema_base_url = schema_base_url
# Allows setting the registry URL to e.g. a pull request, when working on a profile.
if not registry_base_url:
registry_base_url = 'https://raw.githubusercontent.com/open-contracting/extension_registry/master/'
self.registry = ExtensionRegistry(registry_base_url +
'extension_versions.csv')
def set_topics():
"""
Add topics to repositories in the open-contracting-extensions organization.
- ocds-extension
- ocds-core-extension
- ocds-community-extension
- ocds-profile
- european-union
- public-private-partnerships
"""
format_string = 'https://raw.githubusercontent.com/open-contracting-extensions/{}/{}/docs/extension_versions.json'
profiles = defaultdict(list)
for profile, branch in (('european-union', 'latest'),
('public-private-partnerships', '1.0-dev')):
extension_versions = requests.get(format_string.format(
profile, branch)).json()
for extension_id in extension_versions.keys():
profiles[extension_id].append(profile)
registry = ExtensionRegistry(extension_versions_url, extensions_url)
repos = requests.get(
'https://api.github.com/orgs/open-contracting-extensions/repos?per_page=100'
).json()
for repo in repos:
topics = []
if repo['name'].endswith('_extension'):
topics.append('ocds-extension')
else:
topics.append('ocds-profile')
for version in registry:
if f"/{repo['full_name']}/" in version.base_url:
if version.core:
topics.append('ocds-core-extension')
else:
topics.append('ocds-community-extension')
topics.extend(profiles[version.id])
break
else:
if 'ocds-profile' not in topics:
click.echo(f"{repo['name']} is not registered")
requests.put(
f"https://api.github.com/repos/{repo['full_name']}/topics",
data=json.dumps({'names': topics}),
headers={
'accept': 'application/vnd.github.mercy-preview+json'
}).raise_for_status()
def test_filter():
obj = ExtensionRegistry(extension_versions_data, extensions_data)
result = obj.filter(core=True, version='v1.1.3', category='tender')
assert len(result) == 2
assert result[0].as_dict() == {
'id': 'enquiries',
'date': '2018-02-01',
'version': 'v1.1.3',
'base_url': 'https://raw.githubusercontent.com/open-contracting-extensions/ocds_enquiry_extension/v1.1.3/',
'download_url': 'https://api.github.com/repos/open-contracting-extensions/ocds_enquiry_extension/zipball/v1.1.3', # noqa: E501
'category': 'tender',
'core': True,
}
assert result[1].as_dict() == {
'id': 'lots',
'date': '2018-01-30',
'version': 'v1.1.3',
'base_url': 'https://raw.githubusercontent.com/open-contracting-extensions/ocds_lots_extension/v1.1.3/',
'download_url': 'https://api.github.com/repos/open-contracting-extensions/ocds_lots_extension/zipball/v1.1.3',
'category': 'tender',
'core': True,
}
def test_init_with_versions_only():
obj = ExtensionRegistry(extension_versions_data)
assert len(obj.versions) == 14
assert obj.versions[0].as_dict() == {
'id': 'charges',
'date': '',
'version': 'master',
'base_url': 'https://raw.githubusercontent.com/open-contracting-extensions/ocds_charges_extension/master/',
'download_url': 'https://github.com/open-contracting-extensions/ocds_charges_extension/archive/master.zip',
}
# Assume intermediate data is correctly parsed.
assert obj.versions[-1].as_dict() == {
'id': 'lots',
'date': '2018-01-30',
'version': 'v1.1.3',
'base_url': 'https://raw.githubusercontent.com/open-contracting-extensions/ocds_lots_extension/v1.1.3/',
'download_url': 'https://api.github.com/repos/open-contracting-extensions/ocds_lots_extension/zipball/v1.1.3',
}
def versions(self):
registry = ExtensionRegistry(self.args.extension_versions_url,
self.args.extensions_url)
versions = defaultdict(list)
for value in self.args.versions:
if '==' in value:
extension, version = value.split('==', 1)
versions[extension].append(version)
elif '=' in value:
# Help users with a common error.
raise CommandError(
f"Couldn't parse '{value}'. Use '==' not '='.")
else:
versions[value]
for version in registry:
if ((not self.args.versions or version.id in versions)
and (not versions[version.id]
or version.version in versions[version.id])):
yield version
def run(self):
config = self.state.document.settings.env.config
extension_versions = config.extension_versions
language = config.overrides.get('language', 'en')
extension_list_name = self.options.pop('list', '')
set_classes(self.options)
admonition_node = nodes.admonition('', **self.options)
self.add_name(admonition_node)
title_text = self.arguments[0]
textnodes, _ = self.state.inline_text(title_text, self.lineno)
title = nodes.title(title_text, '', *textnodes)
title.line = 0
title.source = 'extension_list_' + extension_list_name
admonition_node += title
if 'classes' not in self.options:
admonition_node['classes'] += ['admonition', 'note']
admonition_node['classes'] += ['extension_list']
admonition_node['ids'] += ['extensionlist-' + extension_list_name]
definition_list = nodes.definition_list()
definition_list.line = 0
# Only list core extensions whose version matches the version specified in `conf.py` and whose category matches
# the category specified by the directive's `list` option.
registry = ExtensionRegistry(extension_versions_url, extensions_url)
num = 0
for identifier, version in extension_versions.items():
extension = registry.get(id=identifier, core=True, version=version)
if extension_list_name and extension.category != extension_list_name:
continue
# Avoid "403 Client Error: rate limit exceeded for url" on development branches.
try:
metadata = extension.metadata
except requests.exceptions.HTTPError:
if live_branch:
raise
metadata = {
'name': {
'en': identifier
},
'description': {
'en': identifier
}
}
name = metadata['name']['en']
description = metadata['description']['en']
some_term, _ = self.state.inline_text(name, self.lineno)
some_def, _ = self.state.inline_text(description, self.lineno)
link = nodes.reference(name, '', *some_term)
link['refuri'] = extension_explorer_template.format(
language, identifier, version)
link['translatable'] = True
link.source = 'extension_list_' + extension_list_name
link.line = num + 1
term = nodes.term(name, '', link)
definition_list += term
text = nodes.paragraph(description, '', *some_def)
text.source = 'extension_list_' + extension_list_name
text.line = num + 1
definition_list += nodes.definition(description, text)
if extension_list_name and not registry.filter(
category=extension_list_name):
raise self.warning(
f'No extensions have category {extension_list_name} in extensionlist directive'
)
admonition_node += definition_list
community = "The following are community extensions and are not maintained by Open Contracting Partnership."
community_text, _ = self.state.inline_text(community, self.lineno)
community_paragraph = nodes.paragraph(community, *community_text)
community_paragraph['classes'] += ['hide']
community_paragraph.source = 'extension_list_' + extension_list_name
community_paragraph.line = num + 2
admonition_node += community_paragraph
return [admonition_node]
def test_filter_invalid():
obj = ExtensionRegistry(extension_versions_data)
with pytest.raises(AttributeError) as excinfo:
obj.filter(invalid='invalid')
assert str(excinfo.value) == "'ExtensionVersion' object has no attribute 'invalid'"
def test_get_no_match():
obj = ExtensionRegistry(extension_versions_data)
with pytest.raises(DoesNotExist) as excinfo:
obj.get(id='nonexistent')
assert str(excinfo.value) == "Extension version matching {'id': 'nonexistent'} does not exist."
"extensions": ["{}extension.json"],
"releases": []
}}
```
This extension is maintained at <{}>
## Documentation
"""
extensions_path = os.path.join(docs_path, 'extensions')
extensions_url = 'https://raw.githubusercontent.com/open-contracting/extension_registry/master/extensions.csv'
extension_versions_url = 'https://raw.githubusercontent.com/open-contracting/extension_registry/master/extension_versions.csv' # noqa
extension_registry = ExtensionRegistry(extension_versions_url, extensions_url)
# At present, all core extensions are included in the standard's documentation.
for version in extension_registry.filter(core=True):
if version.id not in extension_versions:
raise Exception(
'{} is a core extension but is not included in the standard'.
format(version.id))
for identifier, version in extension_versions.items():
extension = extension_registry.get(id=identifier, version=version)
lines = extension.remote('README.md').split('\n')
heading = '\n'.join(lines[:1])
body = '\n'.join(lines[1:])
body = body.replace('\n##', '\n###')
text = heading + metadata.format(extension.base_url,
def load_extensions_info(self):
"""
Gets the core extensions from the extension registry.
If the language is not 'en', produces the translated versions for each core extension.
"""
extensions_dir = Path('extensions')
if extensions_dir.exists():
shutil.rmtree(str(extensions_dir))
extensions_dir.mkdir()
# download extension files according to version
registry = ExtensionRegistry(self.extension_versions_url, self.extensions_url)
for version in registry.filter(core=True, version=self.version):
if version.id not in self.exclusions:
zip_file = version.zipfile()
zip_file.extractall(path=str(extensions_dir))
# rename path to extension id
path = extensions_dir / zip_file.infolist()[0].filename
path.rename(extensions_dir / version.id)
if self.lang is 'en':
output_dir = extensions_dir
else:
# translate core extensions
translation_sources_dir = Path('ocds-extensions-translations-master')
if translation_sources_dir.exists():
shutil.rmtree(str(translation_sources_dir))
res = requests.get('https://github.com/open-contracting/ocds-extensions-translations/archive/master.zip')
res.raise_for_status()
content = BytesIO(res.content)
with ZipFile(content) as zipfile:
zipfile.extractall()
output_dir = Path('translations') / self.lang
if output_dir.exists():
shutil.rmtree(str(output_dir))
output_dir.mkdir(parents=True)
locale = str(translation_sources_dir / 'locale')
headers = ['Title', 'Description', 'Extension']
for dir in [x for x in extensions_dir.iterdir() if x.is_dir()]:
translate([
(glob(str(dir / 'extension.json')), output_dir / dir.parts[-1], dir.parts[-1] + '/' + self.version + '/schema'),
(glob(str(dir / 'release-schema.json')),
output_dir / dir.parts[-1],
dir.parts[-1] + '/' + self.version + '/schema')
], locale, self.lang, headers)
self.extension_urls = [path.resolve(strict=True).as_uri() for path in output_dir.iterdir() if path.is_dir()]
# get names and descriptions for each extension
for dir in [x for x in output_dir.iterdir() if x.is_dir()]:
path = dir.joinpath('extension.json')
info = {}
with path.open() as f:
info = json.load(f)
self.descriptions[info['name'][self.lang]] = info['description'][self.lang]
# mapping-schema looks for the name of the name extension in English
if self.lang is not 'en':
info['name']['en'] = info['name'][self.lang]
with path.open(mode='w') as f:
json.dump(info, f)
return self.extension_urls
def test_get_from_url_no_match():
obj = ExtensionRegistry(extension_versions_data)
with pytest.raises(DoesNotExist) as excinfo:
obj.get_from_url('http://example.com')
assert str(excinfo.value) == "Extension version matching {'base_url': 'http://example.com/'} does not exist."
class ProfileBuilder:
def __init__(self,
standard_tag,
extension_versions,
registry_base_url=None,
schema_base_url=None):
"""
Accepts an OCDS version and a dictionary of extension identifiers and versions, and initializes a reader of the
extension registry.
"""
self.standard_tag = standard_tag
self.extension_versions = extension_versions
self._file_cache = {}
self.schema_base_url = schema_base_url
# Allows setting the registry URL to e.g. a pull request, when working on a profile.
if not registry_base_url:
registry_base_url = 'https://raw.githubusercontent.com/open-contracting/extension_registry/master/'
self.registry = ExtensionRegistry(registry_base_url +
'extension_versions.csv')
def extensions(self):
"""
Returns the matching extension versions from the registry.
"""
for identifier, version in self.extension_versions.items():
yield self.registry.get(id=identifier, version=version)
def release_schema_patch(self):
"""
Returns the consolidated release schema patch.
"""
profile_patch = OrderedDict()
# Replaces `null` with sentinel values, to preserve the null'ing of fields by extensions in the final patch.
for extension in self.extensions():
data = re.sub(r':\s*null\b', ': "REPLACE_WITH_NULL"',
extension.remote('release-schema.json'))
json_merge_patch.merge(profile_patch, _json_loads(data))
return _json_loads(
json.dumps(profile_patch).replace('"REPLACE_WITH_NULL"', 'null'))
def patched_release_schema(self):
"""
Returns the patched release schema.
"""
content = self.get_standard_file_contents('release-schema.json')
patched = json_merge_patch.merge(_json_loads(content),
self.release_schema_patch())
if self.schema_base_url:
patched['id'] = urljoin(self.schema_base_url,
'release-schema.json')
return patched
def release_package_schema(self):
"""
Returns a release package schema. If `schema_base_url` was provided, updates schema URLs.
"""
data = _json_loads(
self.get_standard_file_contents('release-package-schema.json'))
if self.schema_base_url:
data['id'] = urljoin(self.schema_base_url,
'release-package-schema.json')
data['properties']['releases']['items']['$ref'] = urljoin(
self.schema_base_url, 'release-schema.json')
return data
def standard_codelists(self):
"""
Returns the standard's codelists as Codelist objects.
"""
codelists = OrderedDict()
# Populate the file cache.
self.get_standard_file_contents('release-schema.json')
# This method shouldn't need to know about `_file_cache`.
for path, content in self._file_cache.items():
name = os.path.basename(path)
if 'codelists' in path.split(os.sep) and name:
codelists[name] = Codelist(name)
codelists[name].extend(csv.DictReader(StringIO(content)),
'OCDS Core')
return list(codelists.values())
def extension_codelists(self):
"""
Returns the extensions' codelists as Codelist objects.
The extensions' codelists may be new, or may add codes to (+name.csv), remove codes from (-name.csv) or replace
(name.csv) the codelists of the standard or other extensions.
Codelist additions and removals are merged across extensions. If new codelists or codelist replacements differ
across extensions, an error is raised.
"""
codelists = OrderedDict()
# Keep the original content of codelists, to compare across extensions.
originals = {}
for extension in self.extensions():
# We use the "codelists" field in extension.json (which standard-maintenance-scripts validates). An
# extension is not guaranteed to offer a download URL, which is the only other way to get codelists.
for name in extension.metadata.get('codelists', []):
content = extension.remote('codelists/' + name)
if name not in codelists:
codelists[name] = Codelist(name)
originals[name] = content
elif not codelists[name].patch:
assert originals[
name] == content, 'codelist {} differs across extensions'.format(
name)
continue
codelists[name].extend(csv.DictReader(StringIO(content)),
extension.metadata['name']['en'])
# If a codelist replacement (name.csv) is consistent with additions (+name.csv) and removals (-name.csv), the
# latter should be removed. In other words, the expectations are that:
#
# * A codelist replacement shouldn't omit added codes.
# * A codelist replacement shouldn't include removed codes.
# * If codes are added after a codelist is replaced, this should result in duplicate codes.
# * If codes are removed after a codelist is replaced, this should result in no change.
#
# If these expectations are not met, an error is raised. As such, profile authors only have to handle cases
# where codelist modifications are inconsistent across extensions.
for codelist in list(codelists.values()):
basename = codelist.basename
if codelist.patch and basename in codelists:
name = codelist.name
codes = codelists[basename].codes
if codelist.addend:
for row in codelist:
code = row['Code']
assert code in codes, '{} added by {}, but not in {}'.format(
code, name, basename)
logger.info(
'{0} has the codes added by {1} - ignoring {1}'.format(
basename, name))
else:
for row in codelist:
code = row['Code']
assert code not in codes, '{} removed by {}, but in {}'.format(
code, name, basename)
logger.info(
'{0} has no codes removed by {1} - ignoring {1}'.
format(basename, name))
del codelists[name]
return list(codelists.values())
def patched_codelists(self):
"""
Returns patched and new codelists as Codelist objects.
"""
codelists = OrderedDict()
for codelist in self.standard_codelists():
codelists[codelist.name] = codelist
for codelist in self.extension_codelists():
if codelist.patch:
basename = codelist.basename
if codelist.addend:
# Add the rows.
codelists[basename].rows.extend(codelist.rows)
# Note that the rows may not all have the same columns, but DictWriter can handle this.
else:
# Remove the codes. Multiple extensions can remove the same codes.
removed = codelist.codes
codelists[basename].rows = [
row for row in codelists[basename]
if row['Code'] not in removed
]
else:
# Set or replace the rows.
codelists[codelist.name] = codelist
return list(codelists.values())
def get_standard_file_contents(self, basename):
"""
Returns the contents of the file within the standard.
Downloads the given version of the standard, and caches the contents of files in the schema/ directory.
"""
if not self._file_cache:
url = 'https://codeload.github.com/open-contracting/standard/zip/' + self.standard_tag
response = requests.get(url)
response.raise_for_status()
zipfile = ZipFile(BytesIO(response.content))
names = zipfile.namelist()
path = 'standard/schema/'
start = len(names[0] + path)
for name in names[1:]:
if path in name:
self._file_cache[name[start:]] = zipfile.read(name).decode(
'utf-8')
return self._file_cache[basename]
def test_init_with_url():
obj = ExtensionRegistry(extension_versions_url, extensions_url)
assert len(obj.versions) > 50
def test_iter():
obj = ExtensionRegistry(extension_versions_data)
for i, version in enumerate(obj, 1):
pass
assert i == 14
def test_get_without_extensions():
obj = ExtensionRegistry(extension_versions_data)
with pytest.raises(MissingExtensionMetadata) as excinfo:
obj.get(category='tender')
assert str(excinfo.value) == 'ExtensionRegistry must be initialized with extensions data.'