def check_data(runtime_id, package_id, test_functions, codelists, data):
def get_result_hierarchy(activity):
hierarchy = activity.get('hierarchy', default=0)
if hierarchy is "":
return 0
return hierarchy
def run_test_activity(organisation_id, activity):
result_hierarchy = get_result_hierarchy(activity)
test_activity(runtime_id, package_id, activity,
result_hierarchy,
test_functions,
codelists, organisation_id)
def run_test_organisation(organisation_id,
org_organisation_data):
run_info_results(package_id, runtime_id, org_organisation_data,
test_level.ORGANISATION, organisation_id)
organisation_data = etree.tostring(org_organisation_data)
test_organisation(runtime_id, package_id,
organisation_data, test_functions,
codelists, organisation_id)
def get_activities(organisation):
xp = organisation['activities_xpath']
try:
return data.xpath(xp)
except etree.XPathEvalError:
raise InvalidXPath(xp)
def run_tests_for_organisation(organisation):
org_activities = get_activities(organisation)
org_id = organisation['organisation_id']
[ run_test_activity(org_id, activity)
for activity in org_activities ]
if len(org_activities)>0:
run_info_results(package_id, runtime_id, org_activities,
test_level.ACTIVITY, org_id)
org_organisations_data = data.xpath('//iati-organisation')
[ run_test_organisation(org_id, org_organisation_data) for
org_organisation_data in org_organisations_data]
organisations = dqpackages.get_organisations_for_testing(package_id)
assert len(organisations) > 0
for organisation in organisations:
run_tests_for_organisation(organisation)
dqprocessing.aggregate_results(runtime_id, package_id)
dqfunctions.add_test_status(package_id, package_status.TESTED)
def download_packages(runtime):
# Check registry for packages list
registry_packages = [ (pkg["name"], pkg["metadata_modified"])
for pkg in packages_from_registry_with_offset(REGISTRY_URL) ]
print "Found", len(registry_packages),"packages on the IATI Registry"
print "Checking for updates, calculating and queuing packages;"
print "this may take a moment..."
registry_packages=dict(registry_packages)
testing_packages=[]
packages = models.Package.query.filter_by(active=True).all()
print registry_packages
for package in packages:
name = package.package_name
print name
print package.package_metadata_modified
# Handle automatically retrieved packages (from Registry)
if package.man_auto == 'auto':
try:
if package.package_metadata_modified != registry_packages[name]:
print "Need to update package", name
# need to add status here, because otherwise the status could
# be written to DB after the package has finished testing
add_test_status(package.id, package_status.NEW)
testing_packages.append(package.id)
enqueue_download(package, runtime.id)
else:
print "Packages have the same ID"
print "Metadata modified is", package.package_metadata_modified
print "Registry package name is", registry_packages[name]
except KeyError, e:
if name not in registry_packages:
print "name wasn't there"
# TODO: handle deleted packages; for now just pass
pass
else:
raise Exception, e
# Handle manually added packages
else:
if ((package.package_metadata_modified == "") or (package.package_metadata_modified == None)):
print "Need to update manual package", name
# need to add status here, because otherwise the status could
# be written to DB after the package has finished testing
add_test_status(package.id, package_status.NEW)
testing_packages.append(package.id)
enqueue_download(package, runtime.id)
else:
print "Not testing manual package, because it has not been set to refresh"
def download_package(runtime, package_name):
package = models.Package.query.filter_by(
package_name=package_name).first()
add_test_status(package.id, package_status.NEW)
registry = ckanclient.CkanClient(base_location=CKANurl)
pkg = registry.package_entity_get(package.package_name)
try:
if pkg['metadata_modified'] == package.package_metadata_modified:
add_test_status(package.id, package_status.UP_TO_DATE)
else:
get_package(pkg, package, runtime.id)
except IndexError:
print "Package has no resources"
pass
except KeyError:
print "Package resource has no package_metadata_modified"
pass
def get_package(pkg, package, runtime_id):
new_package = False
update_package = False
# Check if package already exists; if it has not been updated more
# recently than the database, then download it again
check_package = package
if ((package.package_metadata_modified) != (pkg['metadata_modified'])):
# if the package has been updated,
# then download it and update the package data
update_package = True
print "Updating package", pkg['name']
if (update_package or new_package):
resources = pkg.get('resources', [])
assert len(resources) <= 1
if resources == []:
return
resource = resources[0]
enqueue_download(package, runtime_id)
else:
print "Package", pkg["name"], "is already the latest version"
add_test_status(package.id, package_status.UP_TO_DATE)
def load_package(package):
dqfunctions.add_test_status(package.id, package_status.TO_DOWNLOAD)
filename = os.path.join(path, package.package_name + ".xml")
# Run tests on file -> send to queue
enqueue_download(filename, runtime, package.id, None)
output.append(package.package_name)
