def save_qa_result(resource, qa_result):
"""
Saves the results of the QA check to the qa table.
"""
import ckan.model as model
from ckanext.qa.model import QA
now = datetime.datetime.now()
qa = QA.get_for_resource(resource.id)
if not qa:
qa = QA.create(resource.id)
model.Session.add(qa)
else:
log.info(u'QA from before: %r', qa)
for key in ('openness_score', 'openness_score_reason', 'format'):
setattr(qa, key, qa_result[key])
qa.archival_timestamp = qa_result['archival_timestamp']
qa.updated = now
model.Session.commit()
log.info('QA results updated ok')
return qa # for tests
def save_qa_result(resource, qa_result):
"""
Saves the results of the QA check to the qa table.
"""
import ckan.model as model
from ckanext.qa.model import QA
now = datetime.datetime.now()
qa = QA.get_for_resource(resource.id)
if not qa:
qa = QA.create(resource.id)
model.Session.add(qa)
else:
log.info(u'QA from before: %r', qa)
for key in ('openness_score', 'openness_score_reason', 'format'):
setattr(qa, key, qa_result[key])
qa.archival_timestamp = qa_result['archival_timestamp']
qa.updated = now
model.Session.commit()
log.info('QA results updated ok')
return qa # for tests
def migrate(options):
from ckan import model
from ckanext.archiver.model import Archival
from ckanext.qa.model import QA
resources = common.get_resources(state='active',
publisher_ref=options.publisher,
resource_id=options.resource,
dataset_name=options.dataset)
stats = StatsList()
widgets = ['Resources: ', Percentage(), ' ', Bar(), ' ', ETA()]
progress = ProgressBar(widgets=widgets)
for res in progress(resources):
# Gather the details of QA from TaskStatus
# to fill all properties of QA apart from:
# * package_id
# * resource_id
fields = {}
qa_task_status = model.Session.query(model.TaskStatus)\
.filter_by(entity_id=res.id)\
.filter_by(task_type='qa')\
.filter_by(key='status')\
.first()
if not qa_task_status:
add_stat('No QA data', res, stats)
continue
qa_error = json.loads(qa_task_status.error)
fields['openness_score'] = int(qa_task_status.value)
fields['openness_score_reason'] = qa_error['reason']
fields['format'] = qa_error['format']
qa_date = qa_task_status.last_updated
# NB qa_task_status.last_updated appears to be 1hr ahead of the revision
# time, so some timezone nonesense going on. Can't do much.
archival = Archival.get_for_resource(res.id)
if not archival:
print add_stat('QA but no Archival data', res, stats)
continue
archival_date = archival.updated
# the state of the resource was as it was archived on the date of
# the QA update but we only know when the latest archival was. So
# if it was archived before the QA update thenwe know that was the
# archival, otherwise we don't know when the relevant archival was.
if archival_date and qa_date >= archival_date:
fields['archival_timestamp'] = archival_date
fields['updated'] = archival_date
fields['created'] = archival_date
# Assume the resource URL archived was the one when the
# archival was done (it may not be if the URL was queued and
# there was significant delay before it was archived)
get_resource_as_at = archival_date
else:
# This is common for when a resource is created and qa runs just
# before archiver and you get:
# "This file had not been downloaded at the time of scoring it."
# Just put sensible datetimes since we don't really know the exact
# ones
fields['archival_timestamp'] = qa_date
fields['updated'] = qa_date
fields['created'] = qa_date
get_resource_as_at = qa_date
res_rev = model.Session.query(model.ResourceRevision).\
filter_by(id=res.id).\
filter(model.ResourceRevision.revision_timestamp < get_resource_as_at).\
order_by(model.ResourceRevision.revision_timestamp.desc()).\
first()
fields['resource_timestamp'] = res_rev.revision_timestamp
# Compare with any existing data in the Archival table
qa = QA.get_for_resource(res.id)
if qa:
changed = None
for field, value in fields.items():
if getattr(qa, field) != value:
if options.write:
setattr(qa, field, value)
changed = True
if not changed:
add_stat('Already exists correctly in QA table', res, stats)
continue
add_stat('Updated in QA table', res, stats)
else:
qa = QA.create(res.id)
if options.write:
for field, value in fields.items():
setattr(qa, field, value)
model.Session.add(qa)
add_stat('Added to QA table', res, stats)
print 'Summary\n', stats.report()
if options.write:
model.repo.commit_and_remove()
print 'Written'
package = toolkit.get_action('package_show')(context_, {'id': package_id})
package_index.index_package(package, defer_commit=False)
log.info('Search indexed %s', package['name'])
def save_qa_result(resource, qa_result, log):
"""
Saves the results of the QA check to the qa table.
"""
import ckan.model as model
from ckanext.qa.model import QA
now = datetime.datetime.now()
qa = QA.get_for_resource(resource.id)
if not qa:
qa = QA.create(resource.id)
model.Session.add(qa)
else:
log.info('QA from before: %r', qa)
for key in ('openness_score', 'openness_score_reason', 'format'):
setattr(qa, key, qa_result[key])
qa.archival_timestamp = qa_result['archival_timestamp']
qa.updated = now
model.Session.commit()
log.info('QA results updated ok')
return qa # for tests
