def test_patched_release_schema_with_release_schema_patch_url():
url = 'https://raw.githubusercontent.com/open-contracting-extensions/ocds_coveredBy_extension/master/release-schema.json' # noqa: E501
builder = ProfileBuilder('1__1__4', [url])
result = builder.patched_release_schema()
assert '$schema' in result
assert 'coveredBy' in result['definitions']['Tender']['properties']
def test_patched_release_schema_with_absolute_path():
url = Path(path('ocds_coveredBy_extension')).resolve().as_uri()
builder = ProfileBuilder('1__1__4', [url])
result = builder.patched_release_schema()
assert '$schema' in result
assert 'coveredBy' in result['definitions']['Tender']['properties']
def test_patched_release_schema_with_download_url():
url = 'https://github.com/open-contracting-extensions/ocds_coveredBy_extension/archive/master.zip'
builder = ProfileBuilder('1__1__4', [url])
result = builder.patched_release_schema()
assert '$schema' in result
assert 'coveredBy' in result['definitions']['Tender']['properties']
def test_patched_release_schema_with_schema_base_url():
schema_base_url = 'https://standard.open-contracting.org/profiles/ppp/schema/1__0__0__beta/'
builder = ProfileBuilder('1__1__4', {}, schema_base_url=schema_base_url)
result = builder.patched_release_schema()
# Changes `id`.
assert result[
'id'] == 'https://standard.open-contracting.org/profiles/ppp/schema/1__0__0__beta/release-schema.json' # noqa: E501
def test_merge_with_schema():
builder = ProfileBuilder('1__1__4', {'additionalContactPoint': 'master'})
schema = builder.patched_release_schema()
data = json.loads(
read('release-package_additional-contact-points.json'))['releases']
compiled_release = list(merge(data, schema=schema))[0]
assert compiled_release == json.loads(read('compile_extensions.json'))
def test_patched_release_schema_with_extension_field():
builder = ProfileBuilder('1__1__4', {'location': 'v1.1.4'})
result = builder.patched_release_schema(extension_field='extension')
definition = result['definitions']['Location']
assert definition['extension'] == 'Location'
assert definition['properties']['geometry']['extension'] == 'Location'
assert definition['properties']['geometry']['properties']['type'][
'extension'] == 'Location'
def get_patched_schema(self):
schema_response = requests.get(self.schema_url)
schema = schema_response.json()
builder = ProfileBuilder(None, self.extensions_info.extension_urls)
schema = builder.patched_release_schema(
schema=schema, extension_field=self.extension_field)
schema = jsonref.JsonRef.replace_refs(schema)
with open('release-schema.json', 'w') as f:
jsonref.dump(schema, f)
return schema
def test_patched_release_schema_with_extension_field_and_language():
builder = ProfileBuilder('1__1__4', [
'https://extensions.open-contracting.org/en/extensions/location/master/'
])
result = builder.patched_release_schema(extension_field='extension',
language='es')
definition = result['definitions']['Location']
assert definition['extension'] == 'Ubicación'
assert definition['properties']['geometry']['extension'] == 'Ubicación'
assert definition['properties']['geometry']['properties']['type'][
'extension'] == 'Ubicación'
def test_patched_release_schema():
# Use the ppp extension to test null values.
builder = ProfileBuilder(
'1__1__5',
{
'https://raw.githubusercontent.com/open-contracting-extensions/ocds_ppp_extension/70c5cb759d4739d1eca5db832e723afb69bbdae0/', # noqa: E501
'https://github.com/open-contracting-extensions/ocds_location_extension/archive/v1.1.5.zip',
})
result = builder.patched_release_schema()
# Patches core.
assert '$schema' in result
assert 'Location' in result['definitions']
# Removes null'ed fields.
assert 'buyer' not in result['properties']
def handle(self):
with open(self.args.file) as f:
schema = json.load(f)
if self.args.extension:
builder = ProfileBuilder(None, self.args.extension)
schema = builder.patched_release_schema(
schema=schema, extension_field=self.args.extension_field)
base_uri = pathlib.Path(os.path.realpath(self.args.file)).as_uri()
if not self.args.no_replace_refs:
schema = jsonref.JsonRef.replace_refs(schema, base_uri=base_uri)
try:
mapping_sheet(schema,
sys.stdout,
order_by=self.args.order_by,
infer_required=self.args.infer_required,
extension_field=self.args.extension_field,
include_deprecated=not self.args.no_deprecated,
include_definitions=self.args.no_replace_refs)
except MissingColumnError as e:
raise CommandError(str(e)) from e
def merge(data,
uri='',
publisher=None,
published_date='',
version=DEFAULT_VERSION,
schema=None,
return_versioned_release=False,
return_package=False,
use_linked_releases=False,
streaming=False):
"""
Merges release packages and individual releases.
By default, yields compiled releases. If ``return_versioned_release`` is ``True``, yields versioned releases. If
``return_package`` is ``True``, wraps the compiled releases (and versioned releases if ``return_versioned_release``
is ``True``) in a record package.
If ``return_package`` is set and ``publisher`` isn't set, the output record package will have the same publisher as
the last input release package.
:param data: an iterable of release packages and individual releases
:param str uri: if ``return_package`` is ``True``, the record package's ``uri``
:param dict publisher: if ``return_package`` is ``True``, the record package's ``publisher``
:param str published_date: if ``return_package`` is ``True``, the record package's ``publishedDate``
:param str version: if ``return_package`` is ``True``, the record package's ``version``
:param dict schema: the URL, path or dict of the patched release schema to use
:param bool return_package: wrap the compiled releases in a record package
:param bool use_linked_releases: if ``return_package`` is ``True``, use linked releases instead of full releases,
if the input is a release package
:param bool return_versioned_release: if ``return_package`` is ``True``, include versioned releases in the record
package; otherwise, yield versioned releases instead of compiled releases
:param bool streaming: if ``return_package`` is ``True``, set the package's records to a generator (this only works
if the calling code exhausts the generator before ``merge`` returns)
:raises InconsistentVersionError: if the versions are inconsistent across packages to merge
:raises MissingOcidKeyError: if the release is missing an ``ocid`` field
"""
with Packager() as packager:
packager.add(data)
if not schema and packager.version:
prefix = packager.version.replace('.', '__') + '__'
tag = next(tag for tag in reversed(get_tags())
if tag.startswith(prefix))
schema = get_release_schema_url(tag)
if packager.package['extensions']:
builder = ProfileBuilder(tag,
list(packager.package['extensions']))
schema = builder.patched_release_schema()
merger = Merger(schema)
if return_package:
packager.package['uri'] = uri
packager.package['publishedDate'] = published_date
packager.package['version'] = version
if publisher:
packager.package['publisher'] = publisher
yield from packager.output_package(
merger,
return_versioned_release=return_versioned_release,
use_linked_releases=use_linked_releases,
streaming=streaming)
else:
yield from packager.output_releases(
merger, return_versioned_release=return_versioned_release)
def get_extended_mapping_sheet(extensions, version):
builder = ProfileBuilder(version, extensions)
schema = jsonref.JsonRef.replace_refs(builder.patched_release_schema())
return _get_mapping_sheet(schema)
def update(ppp_base_url):
"""
Aligns OC4IDS with OCDS. It uses OCDS for PPPs as a basis, as it includes most definitions and codelists needed in
OC4IDS. It copies definitions and codelists across, making modifications as required.
Run this command for every release of OCDS for PPPs, review any changes to schemas or codelists, and update the
command as needed.
Some OC4IDS-specific definitions have fields with the same names as in OCDS-specific definitions, notably:
- procurementMethod
- procurementMethodDetails
- tenderers
The descriptions of most other such fields have diverged. As such, the command makes no effort to copy the
descriptions of such fields, and instead leaves this up to the editor.
"""
def copy_def(definition, replacements=None):
value = deepcopy(ppp_schema['definitions'][definition])
schema['definitions'][definition] = value
if replacements:
for keys, replacement in replacements.items():
leaf = keys[-1]
for key in keys[:-1]:
value = value[key]
value[leaf] = replacement(value[leaf])
ocds_base_url = 'https://standard.open-contracting.org/1.1/en/'
builder = ProfileBuilder('1__1__5', {'budget': 'master'})
ppp_schema = get(f'{ppp_base_url}release-schema.json').json(
object_pairs_hook=OrderedDict)
ppp_schema = builder.patched_release_schema(schema=ppp_schema)
schema_dir = basedir / 'schema' / 'project-level'
codelists_dir = schema_dir / 'codelists'
with (schema_dir / 'project-schema.json').open() as f:
schema = json.load(f, object_pairs_hook=OrderedDict)
infra_codelists = {
'contractingProcessStatus.csv',
'contractNature.csv',
'metricID.csv',
'modificationType.csv',
'projectSector.csv',
'projectStatus.csv',
'projectType.csv',
'relatedProjectScheme.csv',
'relatedProject.csv',
}
ocds_codelists = {
'currency.csv',
'documentType.csv',
'geometryType.csv',
'locationGazetteers.csv',
'method.csv',
'partyRole.csv',
'releaseTag.csv',
'unitClassificationScheme.csv',
}
compare([path.name for path in codelists_dir.iterdir()], infra_codelists,
ocds_codelists, 'schema/project-level/codelists', 'codelists')
infra_definitions = {
'ContractingProcess',
'ContractingProcessSummary', # Similar to individual release in OCDS
'LinkedRelease', # Similar to linked release in OCDS
'Modification',
'RelatedProject', # Similar to relatedProcess in OCDS
'Person',
}
ocds_definitions = {
'Period',
'Classification',
'Location',
'Value',
'Organization',
'OrganizationReference',
'Address',
'ContactPoint',
'BudgetBreakdown',
'Document',
'Identifier',
'Metric',
'Observation',
'Transaction',
}
compare(schema['definitions'], infra_definitions, ocds_definitions,
'schema/project-level/project-schema.json#/definitions',
'definitions')
# Originally from https://docs.google.com/spreadsheets/d/1ttXgMmmLvqBlPRi_4jAJhIobjnCiwMv13YwGfFOnoJk/edit#gid=0
ignore = {
# https://github.com/open-contracting/infrastructure/issues/269
'finalAudit',
# https://github.com/open-contracting/standard/issues/870
'contractSchedule',
# PPP-specific code or description
'needsAssessment',
'projectAdditionality',
'financeAdditionality',
'pppModeRationale',
'riskComparison',
'discountRate',
'equityTransferCaps',
'financeArrangements',
'guaranteeReports',
'grants',
'servicePayments',
'landTransfer',
'assetTransfer',
'revenueShare',
'otherGovernmentSupport',
'tariffMethod',
'tariffReview',
'tariffs',
'tariffIllustration',
'handover',
'financialStatement',
}
# Copy the OCDS codelists.
for basename in ocds_codelists:
path = schema_dir / 'codelists' / basename
if basename in ('documentType.csv', 'partyRole.csv'):
with open(path) as f:
reader = csv.DictReader(f)
fieldnames = reader.fieldnames
oc4ids_rows = []
oc4ids_codes = []
for row in reader:
if row['Source'] == 'OC4IDS':
oc4ids_rows.append(row)
oc4ids_codes.append(row['Code'])
with open(path, 'w') as f:
if basename == 'documentType.csv':
io = StringIO()
writer = csv.DictWriter(io,
fieldnames,
lineterminator='\n',
extrasaction='ignore')
writer.writeheader()
seen = []
# Add codes from OCDS for PPPs.
reader = csv_reader(f'{ppp_base_url}codelists/{basename}')
for row in reader:
if row['Code'] not in ignore:
seen.append(row['Code'])
# These codes' descriptions are entirely new.
if row['Code'] in ('environmentalImpact', ):
row = next(oc4ids_row for oc4ids_row in oc4ids_rows
if oc4ids_row['Code'] == row['Code'])
else:
edit_code(row, oc4ids_codes, 'OCDS for PPPs')
writer.writerow(row)
# Add codes from OCDS.
reader = csv_reader(
f'{ocds_base_url}codelists/documentType.csv')
for row in reader:
if row['Code'] not in seen and row['Code'] not in ignore:
seen.append(row['Code'])
edit_code(row, oc4ids_codes, 'OCDS')
writer.writerow(row)
# Add pre-existing codes from OC4IDS.
writer.writerows(row for row in oc4ids_rows
if row['Code'] not in seen)
text = io.getvalue()
elif basename == 'partyRole.csv':
io = StringIO()
writer = csv.DictWriter(io,
fieldnames,
lineterminator='\n',
extrasaction='ignore')
writer.writeheader()
seen = []
# Add codes from OCDS.
reader = csv_reader(f'{ocds_base_url}codelists/partyRole.csv')
for row in reader:
if row['Code'] not in seen:
seen.append(row['Code'])
edit_code(row, oc4ids_codes, 'OCDS')
writer.writerow(row)
# Add pre-existing codes from OC4IDS.
writer.writerows(row for row in oc4ids_rows
if row['Code'] not in seen)
text = io.getvalue()
else:
text = get(f'{ppp_base_url}codelists/{basename}').text
f.write(text)
# The following definitions follow the same order as in project-schema.json.
copy_def(
'Period',
{
# Refer to project.
(
'description', ):
lambda s: s.replace('contracting process',
'project or contracting process'),
})
copy_def(
'Classification',
{
# Remove line item classifications from the definition.
('properties', 'scheme', 'description'):
lambda s: s[:s.index(' For line item classifications,')],
})
# Remove the `itemClassificationScheme.csv` codelist.
del (schema['definitions']['Classification']['properties']['scheme']
['codelist'])
del (schema['definitions']['Classification']['properties']['scheme']
['openCodelist'])
copy_def('Location')
# noqa: Original from ocds_location_extension: "The location where activity related to this tender, contract or license will be delivered, or will take place. A location can be described by either a geometry (point location, line or polygon), or a gazetteer entry, or both."
schema['definitions']['Location'][
'description'] = "The location where activity related to this project will be delivered, or will take place. A location may be described using a geometry (point location, line or polygon), a gazetteer entry, an address, or a combination of these." # noqa: E501
# Add id to Location.
schema['definitions']['Location']['properties']['id'] = {
'title': 'Identifier',
'description':
'A local identifier for this location, unique within the array this location appears in.',
'type': 'string',
'minLength': 1,
}
# Add address to Location.
schema['definitions']['Location']['properties']['address'] = {
'title': 'Address',
'description': 'A physical address where works will take place.',
'$ref': '#/definitions/Address',
}
schema['definitions']['Location']['properties'].move_to_end('id',
last=False)
schema['definitions']['Location']['required'] = ['id']
# Set stricter validation on gazetteer identifiers
schema['definitions']['Location']['properties']['gazetteer']['properties'][
'identifiers']['uniqueItems'] = True
copy_def('Value')
copy_def(
'Organization',
{
# Refer to project instead of contracting process, link to infrastructure codelist instead of PPP codelist.
('properties', 'roles', 'description'):
lambda s: s.replace('contracting process', 'project').replace(
'profiles/ppp/latest/en/',
'infrastructure/{{version}}/{{lang}}/') # noqa: E501
})
# Remove unneeded extensions and details from Organization.
del (schema['definitions']['Organization']['properties']['shareholders'])
del (schema['definitions']['Organization']['properties']
['beneficialOwnership'])
del (schema['definitions']['Organization']['properties']['details'])
# Set stricter validation on party roles
schema['definitions']['Organization']['properties']['roles'][
'uniqueItems'] = True
# Add `people` property to OrganizationReference
schema['definitions']['Organization']['properties']['people'] = {
"title": "People",
"description":
"People associated with, representing, or working on behalf of this organization in respect of this project.", # noqa: E501
"type": "array",
"items": {
"$ref": "#/definitions/Person"
},
"uniqueItems": True
}
copy_def('OrganizationReference')
copy_def('Address')
copy_def(
'ContactPoint',
{
# Refer to project instead of contracting process.
('properties', 'name', 'description'):
lambda s: s.replace('contracting process', 'project'),
})
copy_def('BudgetBreakdown')
copy_def(
'Document',
{
# Link to infrastructure codelist instead of PPP codelist
('properties', 'documentType', 'description'):
lambda s: s.
replace('profiles/ppp/latest/en/',
'infrastructure/{{version}}/{{lang}}/'), # noqa: E501
})
# noqa: Original from standard: "A short description of the document. We recommend descriptions do not exceed 250 words. In the event the document is not accessible online, the description field can be used to describe arrangements for obtaining a copy of the document.",
schema['definitions']['Document']['properties']['description'][
'description'] = "Where a link to a full document is provided, the description should provide a 1 - 3 paragraph summary of the information the document contains, and the `pageStart` field should be used to make sure readers can find the correct section of the document containing more information. Where there is no linked document available, the description field may contain all the information required by the current `documentType`. \n\nLine breaks in text (represented in JSON using `\\n\\n`) must be respected by systems displaying this information, and systems may also support basic HTML tags (H1-H6, B, I, U, strong, A and optionally IMG) or [markdown syntax](https://github.com/adam-p/markdown-here/wiki/Markdown-Cheatsheet) for formatting. " # noqa: E501
# noqa: Original from standard: " direct link to the document or attachment. The server providing access to this document should be configured to correctly report the document mime type."
schema['definitions']['Document']['properties']['url'][
'description'] = "This should be a direct link to the document or web page where the information described by the current documentType exists." # noqa: E501
copy_def('Identifier')
copy_def(
'Metric', {
('properties', 'id', 'description'):
lambda s: s.replace('contracting process',
'contracting process or project')
}), # noqa: E501
schema['definitions']['Metric'][
'description'] = "Metrics are used to set out forecast and actual metrics targets for a project: for example, planned and actual physical and financial progress over time." # noqa: E501
# noqa: Original from standard: "Metrics are used to set out targets and results from a contracting process. During the planning and tender sections, a metric indicates the anticipated results. In award and contract sections it indicates the awarded/contracted results. In the implementation section it is used to provide updates on actually delivered results, also known as outputs."
copy_def('Observation')
# Remove the `relatedImplementationMilestone` property
del (schema['definitions']['Observation']['properties']
['relatedImplementationMilestone'])
copy_def('Transaction')
remove_null_and_pattern_properties(schema)
remove_integer_identifier_types(schema)
remove_deprecated_properties(schema)
add_validation_properties(schema)
with (schema_dir / 'project-schema.json').open('w') as f:
json.dump(schema, f, ensure_ascii=False, indent=2)
f.write('\n')
import json
import sys
import requests
from ocdsextensionregistry import ProfileBuilder
url = 'https://raw.githubusercontent.com/open-contracting-extensions/european-union/latest/docs/extension_versions.json' # noqa: E501
builder = ProfileBuilder('1__1__5', requests.get(url).json())
schema = builder.patched_release_schema(extension_field='extension')
json.dump(schema, sys.stdout, ensure_ascii=False, indent=2)