def upsert_ocds_data(self, ocds_json_path_or_string, supplied_data=None, process_json=None):
"""
Takes a path to an OCDS Package or a string containing OCDS JSON data
Upserts all data to the Bluetail database
"""
if os.path.exists(ocds_json_path_or_string):
ocds_json = json.load(open(ocds_json_path_or_string))
filename = os.path.split(ocds_json_path_or_string)[1]
else:
ocds_json = json.loads(ocds_json_path_or_string)
filename = "package.json"
if process_json:
ocds_json = process_json(ocds_json)
if not supplied_data:
# Create SuppliedData entry
supplied_data = SuppliedData()
supplied_data.current_app = "bluetail"
supplied_data.original_file.save(filename, ContentFile(json.dumps(ocds_json)))
supplied_data.save()
if ocds_json.get("records"):
# We have a record package
self.upload_record_package(ocds_json, supplied_data=supplied_data)
if ocds_json.get("releases"):
# We have a release package
# First we use OCDSkit merge to create a record package
rp = merge([ocds_json], published_date=ocds_json.get("publishedDate"), return_package=True)
# Then upload the package
for r in rp:
self.upload_record_package(r, supplied_data=supplied_data)
def upload_record_package(self, package_json, supplied_data=None):
"""
Upload a record package
creates a SuppliedData object if not given
creates a OCDSPackageDataJSON object
"""
if not supplied_data:
supplied_data = SuppliedData()
supplied_data.current_app = "bluetail"
supplied_data.save()
package_data = deepcopy(package_json)
records = package_data.pop("records")
package, created = OCDSPackageDataJSON.objects.update_or_create(
supplied_data=supplied_data,
package_data=package_data
)
for record in records:
ocid = record.get("ocid")
record_json, created = OCDSRecordJSON.objects.update_or_create(
ocid=ocid,
defaults={
"record_json": record,
"package_data": package,
}
)
def handle(self, *args, **kwargs):
"""Add dummy example data to database for demo."""
s3_storage = get_storage_class(settings.S3_FILE_STORAGE)()
upsert_helper = UpsertDataHelpers()
# Loop through directories in /media/ on S3 bucket
directories, filenames = s3_storage.listdir(name=".")
for id in directories:
logger.info(id)
if SuppliedData.objects.filter(id=id):
logger.info("SuppliedData object already exists.")
continue
directories, filenames = s3_storage.listdir(name=id)
for filename in filenames:
original_file_path = os.path.join(id, filename)
logger.info(f"Downloading {original_file_path}")
filename_root = os.path.splitext(filename)[0]
created = datetime.strptime(
filename_root,
"%Y%m%dT%H%M%SZ").replace(tzinfo=timezone.utc)
supplied_data = SuppliedData(id=id,
original_file=original_file_path,
created=created)
supplied_data.current_app = "bluetail"
supplied_data.save()
package_json = s3_storage._open(original_file_path).read()
upsert_helper.upsert_ocds_data(package_json,
supplied_data=supplied_data)
def data_input(request, form_classes=default_form_classes, text_file_name='test.json'):
forms = {form_name: form_class() for form_name, form_class in form_classes.items()}
request_data = None
if "source_url" in request.GET:
request_data = request.GET
if request.POST:
request_data = request.POST
if request_data:
if 'source_url' in request_data:
form_name = 'url_form'
elif 'paste' in request_data:
form_name = 'text_form'
else:
form_name = 'upload_form'
form = form_classes[form_name](request_data, request.FILES)
forms[form_name] = form
if form.is_valid():
if form_name == 'text_form':
data = SuppliedData()
else:
data = form.save(commit=False)
data.current_app = request.current_app
data.form_name = form_name
data.save()
if form_name == 'url_form':
try:
data.download()
except requests.ConnectionError as err:
return render(request, 'error.html', context={
'sub_title': _("Sorry we got a ConnectionError whilst trying to download that file"),
'link': 'index',
'link_text': _('Try Again'),
'msg': _(str(err) + '\n\n Common reasons for this error include supplying a local '
'development url that our servers can\'t access, or misconfigured SSL certificates.')
})
except requests.HTTPError as err:
return render(request, 'error.html', context={
'sub_title': _("Sorry we got a HTTP Error whilst trying to download that file"),
'link': 'index',
'link_text': _('Try Again'),
'msg': _(str(err) + '\n\n If you can access the file through a browser then the problem '
'may be related to permissions, or you may be blocking certain user agents.')
})
elif form_name == 'text_form':
data.original_file.save(text_file_name, ContentFile(form['paste'].value()))
if 'raw' in request_data:
return redirect(data.get_raw_url())
else:
return redirect(data.get_absolute_url())
return render(request, 'input/input.html', {'forms': forms})
def input(request):
form_classes = {
'upload_form': UploadForm,
'url_form': UrlForm,
'text_form': TextForm,
}
forms = {form_name: form_class() for form_name, form_class in form_classes.items()}
request_data = None
if "source_url" in request.GET:
request_data = request.GET
if request.POST:
request_data = request.POST
if request_data:
if 'source_url' in request_data:
form_name = 'url_form'
elif 'paste' in request_data:
form_name = 'text_form'
else:
form_name = 'upload_form'
form = form_classes[form_name](request_data, request.FILES)
forms[form_name] = form
if form.is_valid():
if form_name == 'text_form':
data = SuppliedData()
else:
data = form.save(commit=False)
data.current_app = request.current_app
data.form_name = form_name
data.save()
if form_name == 'url_form':
data.download()
elif form_name == 'text_form':
data.original_file.save('test.json', ContentFile(form['paste'].value()))
return redirect(data.get_absolute_url())
return render(request, 'datainput/input.html', {'forms': forms})
def handle(self, *args, file_name, **options):
data = SuppliedData()
head, tail = split(file_name)
data.original_file.save(tail, File(open(file_name)))
data.save()
return data.get_absolute_url()