def handle(self, *args, **kwargs):
"""Add dummy example data to database for demo."""
# Create combined package
cf_ocds_path = os.path.join(DATA_DIR, "contracts_finder", "ocds")
package_path = os.path.join(DATA_DIR, "contracts_finder",
"package.json")
def generate_cf():
for root, dirs, files in os.walk(cf_ocds_path):
for f in files:
if not f.endswith(".json"):
continue
f_path = os.path.join(root, f)
try:
ocds_json = json.load(open(f_path))
yield ocds_json
except:
logger.exception("Failed to insert file %s", f_path)
rp = merge(generate_cf(), return_package=True)
with open(package_path, 'w') as out:
for r in rp:
out.write(json.dumps(r, indent=4))
def upsert_ocds_data(self, ocds_json_path_or_string, supplied_data=None, process_json=None):
"""
Takes a path to an OCDS Package or a string containing OCDS JSON data
Upserts all data to the Bluetail database
"""
if os.path.exists(ocds_json_path_or_string):
ocds_json = json.load(open(ocds_json_path_or_string))
filename = os.path.split(ocds_json_path_or_string)[1]
else:
ocds_json = json.loads(ocds_json_path_or_string)
filename = "package.json"
if process_json:
ocds_json = process_json(ocds_json)
if not supplied_data:
# Create SuppliedData entry
supplied_data = SuppliedData()
supplied_data.current_app = "bluetail"
supplied_data.original_file.save(filename, ContentFile(json.dumps(ocds_json)))
supplied_data.save()
if ocds_json.get("records"):
# We have a record package
self.upload_record_package(ocds_json, supplied_data=supplied_data)
if ocds_json.get("releases"):
# We have a release package
# First we use OCDSkit merge to create a record package
rp = merge([ocds_json], published_date=ocds_json.get("publishedDate"), return_package=True)
# Then upload the package
for r in rp:
self.upload_record_package(r, supplied_data=supplied_data)
def perform_compile(request, published_date='', warnings=None):
packages = [file.json() for file in get_files_from_session(request)]
return_versioned_release = request.GET.get('includeVersioned') == 'true'
return json_response({
'result.json': next(merge(packages, return_package=True, published_date=published_date,
return_versioned_release=return_versioned_release)),
}, warnings)
def __init__(self,
releases_or_release_packages,
project_id=None,
records=None,
output=None):
# coerce generator into list as we iterate over it twice.
releases_or_release_packages = list(releases_or_release_packages)
all_releases = []
for releases_or_release_package in releases_or_release_packages:
if is_package(releases_or_release_package):
all_releases.extend(
check_type(releases_or_release_package.get("releases"),
list))
else:
all_releases.append(releases_or_release_package)
self.releases = sorted_releases(all_releases)
self.releases_by_ocid = defaultdict(list)
for release in self.releases:
ocid = cast_string(release.get("ocid"))
if ocid:
self.releases_by_ocid[ocid].append(release)
self.project_id = project_id
if not records:
record_package = next(
merge(
releases_or_release_packages,
return_package=True,
use_linked_releases=True,
))
records = check_type(record_package.get("records"), list)
compiled_releases = []
for record in records:
# projects only have linked releases 'uri' is a good proxy for that.
record_releases = check_type(record.get("releases"), list)
compiled_release = check_type(record.get("compiledRelease"), dict)
compiled_release["releases"] = [
release for release in record_releases if release.get("url")
]
compiled_release["embeddedReleases"] = [
release for release in record_releases
if not release.get("url")
]
compiled_releases.append(compiled_release)
self.compiled_releases = compiled_releases
self.output = output or {}
if project_id and "id" not in self.output:
self.output["id"] = project_id
self.party_analysis()
self.generate_document_ids = False
def test_merge_with_schema():
builder = ProfileBuilder('1__1__4', {'additionalContactPoint': 'master'})
schema = builder.patched_release_schema()
data = json.loads(
read('release-package_additional-contact-points.json'))['releases']
compiled_release = list(merge(data, schema=schema))[0]
assert compiled_release == json.loads(read('compile_extensions.json'))
def test_merge_without_schema():
data = json.loads(
read('release-package_additional-contact-points.json'))['releases']
compiled_release = list(merge(data))[0]
assert compiled_release == json.loads(read('compile_no-extensions.json'))
def test_merge_empty():
compiled_releases = list(merge([]))
assert compiled_releases == []