def upgrade_10_11(data): """ Upgrades a record package, release package, record or release from 1.0 to 1.1. Retains the deprecated Amendment.changes, Budget.source and Milestone.documents fields. ``data`` must be an ``OrderedDict``. If you have only the parsed JSON, re-parse it with: ``upgrade_10_11(json.loads(json.dumps(data), object_pairs_hook=OrderedDict))`` """ version = get_ocds_minor_version(data) if version != '1.0': return data if is_package(data): data['version'] = '1.1' _move_to_top(data, ('uri', 'version')) if is_record_package(data): for record in data['records']: upgrade_record_10_11(record) elif is_release_package(data): for release in data['releases']: upgrade_release_10_11(release) elif is_record(data): upgrade_record_10_11(data) else: # release upgrade_release_10_11(data) return data
def upgrade_10_11(data): """ Upgrades a record package, release package or release from 1.0 to 1.1. Retains the deprecated Amendment.changes, Budget.source and Milestone.documents fields. """ version = get_ocds_minor_version(data) if version != '1.0': return if is_package(data): data['version'] = '1.1' _move_to_top(data, ('uri', 'version')) if is_record_package(data): for record in data['records']: if 'releases' in record: for release in record['releases']: upgrade_release_10_11(release) if 'compiledRelease' in record: upgrade_release_10_11(record['compiledRelease']) elif is_release_package(data): for release in data['releases']: upgrade_release_10_11(release) else: # release upgrade_release_10_11(data)
def add(self, data): """ Adds release packages and/or individual releases to be merged. :param data: an iterable of release packages and individual releases :raises InconsistentVersionError: if the versions are inconsistent across packages to merge """ for i, item in enumerate(data): version = get_ocds_minor_version(item) if self.version: if version != self.version: # OCDS 1.1 and OCDS 1.0 have different merge rules for `awards.suppliers`. Also, mixing new and # deprecated fields can lead to inconsistencies (e.g. transaction `amount` and `value`). # https://standard.open-contracting.org/latest/en/schema/changelog/#advisories raise InconsistentVersionError( 'item {}: version error: this item uses version {}, but earlier ' 'items used version {}'.format(i, version, self.version), self.version, version) else: self.version = version if is_release(item): self.backend.add_release(item, '') else: # release package uri = item.get('uri', '') _update_package_metadata(self.package, item) # Note: If there are millions of packages to merge, we should use SQLite to store the packages instead. if uri: self.package['packages'].append(uri) for release in item['releases']: self.backend.add_release(release, uri) self.backend.flush()
def compile_release_packages(stream, uri='', publisher=None, published_date='', schema=None, return_versioned_release=False, return_package=False, use_linked_releases=False): """ Reads release packages from the stream, merges the releases by OCID, and yields the compiled releases. If ``return_versioned_release`` is ``True``, yields the versioned release. If ``return_package`` is ``True``, wraps the compiled releases (and versioned releases if ``return_versioned_release`` is ``True``) in a record package. If ``return_package`` is set and ``publisher`` isn't set, the output record package will have the same publisher as the last input release package. :param str uri: if ``return_package`` is ``True``, the record package's ``uri`` :param dict publisher: if ``return_package`` is ``True``, the record package's ``publisher`` :param str published_date: if ``return_package`` is ``True``, the record package's ``publishedDate`` :param dict schema: the URL or path of the release schema to use :param bool return_package: wrap the compiled releases in a record package :param bool use_linked_releases: if ``return_package`` is ``True``, use linked releases instead of full releases :param bool return_versioned_release: if ``return_package`` is ``True``, include versioned releases in the record package; otherwise, yield versioned releases instead of compiled releases """ if return_package: output = OrderedDict([ ('uri', uri), ('publisher', publisher), ('publishedDate', published_date), ('license', None), ('publicationPolicy', None), ('version', None), ('extensions', OrderedDict()), ('packages', []), ('records', []), ]) version = None releases_by_ocid = defaultdict(list) linked_releases = [] for i, line in enumerate(stream): package = json_loads(line) if not version: version = get_ocds_minor_version(package) else: v = get_ocds_minor_version(package) if v != version: raise InconsistentVersionError( 'item {}: version error: this package uses version {}, but earlier ' 'packages used version {}'.format(i, v, version), version, v) if not schema: prefix = version.replace('.', '__') + '__' tag = next(tag for tag in reversed(get_tags()) if tag.startswith(prefix)) schema = get_release_schema_url(tag) for release in package['releases']: releases_by_ocid[release['ocid']].append(release) if return_package and use_linked_releases: linked_releases.append( OrderedDict([ ('url', package['uri'] + '#' + release['id']), ('date', release['date']), ('tag', release['tag']), ])) if return_package: _update_package_metadata(output, package, publisher) output['packages'].append(package['uri']) if return_package: for ocid, releases in releases_by_ocid.items(): record = OrderedDict([ ('ocid', ocid), ('releases', []), ('compiledRelease', merge(releases, schema)), ]) if use_linked_releases: record['releases'] = linked_releases else: record['releases'] = releases if return_versioned_release: record['versionedRelease'] = merge_versioned(releases, schema) output['records'].append(record) _set_extensions_metadata(output) _remove_empty_optional_metadata(output) yield output else: for releases in releases_by_ocid.values(): if return_versioned_release: merge_method = merge_versioned else: merge_method = merge merged_release = merge_method(releases, schema) yield merged_release
def test_get_ocds_minor_version(data, expected): assert get_ocds_minor_version(data) == expected