def upgrade_10_11(data):
"""
Upgrades a record package, release package, record or release from 1.0 to 1.1.
Retains the deprecated Amendment.changes, Budget.source and Milestone.documents fields.
``data`` must be an ``OrderedDict``. If you have only the parsed JSON, re-parse it with:
``upgrade_10_11(json.loads(json.dumps(data), object_pairs_hook=OrderedDict))``
"""
version = get_ocds_minor_version(data)
if version != '1.0':
return data
if is_package(data):
data['version'] = '1.1'
_move_to_top(data, ('uri', 'version'))
if is_record_package(data):
for record in data['records']:
upgrade_record_10_11(record)
elif is_release_package(data):
for release in data['releases']:
upgrade_release_10_11(release)
elif is_record(data):
upgrade_record_10_11(data)
else: # release
upgrade_release_10_11(data)
return data
def upgrade_10_11(data):
"""
Upgrades a record package, release package or release from 1.0 to 1.1.
Retains the deprecated Amendment.changes, Budget.source and Milestone.documents fields.
"""
version = get_ocds_minor_version(data)
if version != '1.0':
return
if is_package(data):
data['version'] = '1.1'
_move_to_top(data, ('uri', 'version'))
if is_record_package(data):
for record in data['records']:
if 'releases' in record:
for release in record['releases']:
upgrade_release_10_11(release)
if 'compiledRelease' in record:
upgrade_release_10_11(record['compiledRelease'])
elif is_release_package(data):
for release in data['releases']:
upgrade_release_10_11(release)
else: # release
upgrade_release_10_11(data)
def add(self, data):
"""
Adds release packages and/or individual releases to be merged.
:param data: an iterable of release packages and individual releases
:raises InconsistentVersionError: if the versions are inconsistent across packages to merge
"""
for i, item in enumerate(data):
version = get_ocds_minor_version(item)
if self.version:
if version != self.version:
# OCDS 1.1 and OCDS 1.0 have different merge rules for `awards.suppliers`. Also, mixing new and
# deprecated fields can lead to inconsistencies (e.g. transaction `amount` and `value`).
# https://standard.open-contracting.org/latest/en/schema/changelog/#advisories
raise InconsistentVersionError(
'item {}: version error: this item uses version {}, but earlier '
'items used version {}'.format(i, version,
self.version),
self.version, version)
else:
self.version = version
if is_release(item):
self.backend.add_release(item, '')
else: # release package
uri = item.get('uri', '')
_update_package_metadata(self.package, item)
# Note: If there are millions of packages to merge, we should use SQLite to store the packages instead.
if uri:
self.package['packages'].append(uri)
for release in item['releases']:
self.backend.add_release(release, uri)
self.backend.flush()
def compile_release_packages(stream,
uri='',
publisher=None,
published_date='',
schema=None,
return_versioned_release=False,
return_package=False,
use_linked_releases=False):
"""
Reads release packages from the stream, merges the releases by OCID, and yields the compiled releases.
If ``return_versioned_release`` is ``True``, yields the versioned release. If ``return_package`` is ``True``, wraps
the compiled releases (and versioned releases if ``return_versioned_release`` is ``True``) in a record package.
If ``return_package`` is set and ``publisher`` isn't set, the output record package will have the same publisher as
the last input release package.
:param str uri: if ``return_package`` is ``True``, the record package's ``uri``
:param dict publisher: if ``return_package`` is ``True``, the record package's ``publisher``
:param str published_date: if ``return_package`` is ``True``, the record package's ``publishedDate``
:param dict schema: the URL or path of the release schema to use
:param bool return_package: wrap the compiled releases in a record package
:param bool use_linked_releases: if ``return_package`` is ``True``, use linked releases instead of full releases
:param bool return_versioned_release: if ``return_package`` is ``True``, include versioned releases in the record
package; otherwise, yield versioned releases instead of compiled releases
"""
if return_package:
output = OrderedDict([
('uri', uri),
('publisher', publisher),
('publishedDate', published_date),
('license', None),
('publicationPolicy', None),
('version', None),
('extensions', OrderedDict()),
('packages', []),
('records', []),
])
version = None
releases_by_ocid = defaultdict(list)
linked_releases = []
for i, line in enumerate(stream):
package = json_loads(line)
if not version:
version = get_ocds_minor_version(package)
else:
v = get_ocds_minor_version(package)
if v != version:
raise InconsistentVersionError(
'item {}: version error: this package uses version {}, but earlier '
'packages used version {}'.format(i, v, version), version,
v)
if not schema:
prefix = version.replace('.', '__') + '__'
tag = next(tag for tag in reversed(get_tags())
if tag.startswith(prefix))
schema = get_release_schema_url(tag)
for release in package['releases']:
releases_by_ocid[release['ocid']].append(release)
if return_package and use_linked_releases:
linked_releases.append(
OrderedDict([
('url', package['uri'] + '#' + release['id']),
('date', release['date']),
('tag', release['tag']),
]))
if return_package:
_update_package_metadata(output, package, publisher)
output['packages'].append(package['uri'])
if return_package:
for ocid, releases in releases_by_ocid.items():
record = OrderedDict([
('ocid', ocid),
('releases', []),
('compiledRelease', merge(releases, schema)),
])
if use_linked_releases:
record['releases'] = linked_releases
else:
record['releases'] = releases
if return_versioned_release:
record['versionedRelease'] = merge_versioned(releases, schema)
output['records'].append(record)
_set_extensions_metadata(output)
_remove_empty_optional_metadata(output)
yield output
else:
for releases in releases_by_ocid.values():
if return_versioned_release:
merge_method = merge_versioned
else:
merge_method = merge
merged_release = merge_method(releases, schema)
yield merged_release
def test_get_ocds_minor_version(data, expected):
assert get_ocds_minor_version(data) == expected