def upload_file(self, metadata, engine, releases):
conn = engine.connect()
tabulated = {}
for release in releases:
if not is_linked_release(release):
tabulated = self.process_object(tuple(), '', tuple(), tabulated, release, None)
for table, rows in tabulated.items():
if not table:
table_name = 'releases'
else:
table_name = '_'.join(table)
try:
table = metadata.tables[table_name]
except KeyError:
print('table {} does not exist'.format(table_name))
continue
for row in rows:
for key in table.c.keys():
if key not in row.keys():
row[key] = None
extras = {}
for key in list(row.keys()):
if key not in table.c:
extras[key] = row.pop(key)
if 'postgresql' in engine.name:
row['extras'] = extras
else:
row['extras'] = json_dumps(extras)
conn.execute(table.insert(), rows)
def _process_record(self, ocid, record, warnings=None):
if not warnings:
warnings = []
releases = record.get('releases', [])
releases_with_date, releases_without_date = self._check_dates_in_releases(releases)
# Can we compile ourselves?
releases_linked = [r for r in releases_with_date if is_linked_release(r)]
if releases_with_date and not releases_linked:
# We have releases with date fields and none are linked (have URL's).
# We can compile them ourselves.
# (Checking releases_with_date here and not releases means that a record with
# a compiledRelease and releases with no dates will be processed by using the compiledRelease,
# so we still have some data)
if releases_without_date:
warnings.append('This OCID had some releases without a date element. ' +
'We have compiled all other releases.')
self._compile_releases_by_ocdsmerge(ocid, releases_with_date, warnings=warnings)
return
# Whatever happens now, users will appreciate a warning about the bad data
if releases_without_date:
warnings.append('This OCID had some releases without a date element.')
# Is there a compiledRelease?
compiled_release = record.get('compiledRelease')
if compiled_release:
warnings.append('This already had a compiledRelease in the record! ' +
'It was passed through this transform unchanged.')
self._store_result(ocid, compiled_release, warnings=warnings)
return
# Is there a release tagged 'compiled'?
releases_compiled = \
[x for x in releases if 'tag' in x and isinstance(x['tag'], list) and 'compiled' in x['tag']]
if len(releases_compiled) > 1:
# If more than one, pick one at random. and log that.
warnings.append('This already has multiple compiled releases in the releases array! ' +
'The compiled release to pass through was selected arbitrarily.')
self._store_result(ocid, releases_compiled[0], warnings=warnings)
elif len(releases_compiled) == 1:
# There is just one compiled release - pass it through unchanged, and log that.
warnings.append('This already has one compiled release in the releases array! ' +
'It was passed through this transform unchanged.')
self._store_result(ocid, releases_compiled[0], warnings=warnings)
else:
# We can't process this ocid. Warn of that.
self.database.add_collection_note(
self.destination_collection.database_id,
'OCID ' + ocid + ' could not be compiled because at least one release in the releases array is a ' +
'linked release or there are no releases with dates, ' +
'and the record has neither a compileRelease nor a release with a tag of "compiled".'
)
def _process_record(self, ocid, record, warnings=None):
if not warnings:
warnings = []
releases = record.get('releases', [])
releases_linked = [r for r in releases if is_linked_release(r)]
if releases and not releases_linked:
# We have releases and none are linked (have URL's).
# We can compile them ourselves.
try:
merger = ocdsmerge.Merger()
out = merger.create_compiled_release(releases)
self._store_result(ocid, out, warnings=warnings)
except ocdsmerge.exceptions.OCDSMergeError as error:
self.database.add_collection_note(
self.destination_collection.database_id, 'OCID ' + ocid +
' could not be compiled because merge library threw an error: '
+ error.__class__.__name__ + ' ' + str(error))
return
compiled_release = record.get('compiledRelease')
if compiled_release:
warnings.append(
'This already had a compiledRelease in the record! ' +
'It was passed through this transform unchanged.')
self._store_result(ocid, compiled_release, warnings=warnings)
return
releases_compiled = \
[x for x in releases if 'tag' in x and isinstance(x['tag'], list) and 'compiled' in x['tag']]
if len(releases_compiled) > 1:
# If more than one, pick one at random. and log that.
warnings.append(
'This already has multiple compiled releases in the releases array! '
+
'The compiled release to pass through was selected arbitrarily.'
)
self._store_result(ocid, releases_compiled[0], warnings=warnings)
elif len(releases_compiled) == 1:
# There is just one compiled release - pass it through unchanged, and log that.
warnings.append(
'This already has one compiled release in the releases array! '
+ 'It was passed through this transform unchanged.')
self._store_result(ocid, releases_compiled[0], warnings=warnings)
else:
# We can't process this ocid. Warn of that.
self.database.add_collection_note(
self.destination_collection.database_id, 'OCID ' + ocid +
' could not be compiled because at least one release in the releases array is a '
+
'linked release, and the record has neither a compileRelease nor a release with a tag of "compiled".'
)
def test_is_linked_release(data, expected):
assert is_linked_release(data) == expected