def test_fetch_package_list(self, iatikit_mock):
data_mock = iatikit_mock.return_value
data_mock.last_updated = datetime.datetime.utcnow()
data_mock.datasets = [
iatikit.Dataset("tst-a.xml"),
iatikit.Dataset("tst-b.xml")
]
datasets = crawler.fetch_dataset_list()
self.assertIn("tst-a", [ds.name for ds in datasets])
self.assertIn("tst-b", [ds.name for ds in datasets])
def test_update_deletes_datasets(self, iatikit_mock):
data_mock = iatikit_mock.return_value
data_mock.last_updated = datetime.datetime.utcnow()
data_mock.datasets = [
iatikit.Dataset("tst-a.xml"),
iatikit.Dataset("tst-b.xml")
]
datasets = crawler.fetch_dataset_list()
self.assertEquals(2, datasets.count())
data_mock.datasets = [iatikit.Dataset("tst-a.xml")]
datasets = crawler.fetch_dataset_list()
self.assertEquals(1, datasets.count())
def validate(uuid):
"""Show the validation results for a supplied dataset."""
supplied_data = SuppliedData.query.get_or_404(str(uuid))
filepath = join(current_app.config['MEDIA_FOLDER'],
supplied_data.original_file)
if not exists(filepath):
flash('Error: That dataset is no longer available', 'danger')
return redirect(url_for('public.home'))
dataset = iatikit.Dataset(filepath)
if supplied_data.validated:
errors = {
'xml_errors': supplied_data.xml_errors,
'iati_errors': supplied_data.iati_errors,
'codelist_errors': supplied_data.codelist_errors,
}
else:
errors = {
'xml_errors': [],
'iati_errors': [],
'codelist_errors': [],
}
valid_xml = dataset.validate_xml()
if valid_xml:
dataset.unminify_xml()
valid_iati = dataset.validate_iati()
for error, count in valid_iati.error_summary:
iati_error = ValidationError('iati_error', error, count,
supplied_data)
errors['iati_errors'].append(iati_error)
db.session.add(iati_error)
valid_codelists = dataset.validate_codelists()
for error, count in valid_codelists.error_summary:
codelist_error = ValidationError('codelist_error', error,
count, supplied_data)
errors['codelist_errors'].append(codelist_error)
db.session.add(codelist_error)
else:
for error, count in valid_xml.error_summary:
xml_error = ValidationError('xml_error', error, count,
supplied_data)
errors['xml_errors'].append(xml_error)
db.session.add(xml_error)
success = all([e == [] for e in errors.values()])
supplied_data.validated = True
db.session.add(supplied_data)
db.session.commit()
return render_template('public/validate.html',
data=supplied_data,
dataset=dataset,
errors=errors,
success=success)
def badge():
"""Show the validation status of a dataset as an SVG badge."""
source_url = request.args.get('url')
if source_url is None:
svg_file = join('static', 'badges', 'no-url.svg')
return send_file(svg_file, mimetype='image/svg+xml')
supplied_data = SuppliedData(source_url, None, None, 'url_form')
filepath = join(current_app.config['MEDIA_FOLDER'],
supplied_data.original_file)
dataset = iatikit.Dataset(filepath)
if dataset.validate_xml() and dataset.validate_iati() \
and dataset.validate_codelists():
svg_file = join('static', 'badges', 'passing.svg')
else:
svg_file = join('static', 'badges', 'failing.svg')
return send_file(svg_file, mimetype='image/svg+xml')
def show(uuid):
"""Show a validation error in its XML context."""
validation_error = ValidationError.query.get_or_404(str(uuid))
filepath = join(current_app.config['MEDIA_FOLDER'],
validation_error.supplied_data.original_file)
if not exists(filepath):
flash('That dataset is no longer available', 'danger')
return redirect(url_for('public.home'))
match = re.search(r'/iati-(?:activity|organisation)\[(\d+)\]',
validation_error.path)
act_num = int(match.group(1)) - 1
dataset = iatikit.Dataset(filepath)
dataset.unminify_xml()
activity = dataset.activities[act_num]
start_line = activity.etree.sourceline
line = validation_error.line - start_line + 1
highlighted_xml = highlight(
activity.xml, XmlLexer(),
HtmlFormatter(linenos='inline',
lineanchors='L',
hl_lines=[line],
linenostart=start_line))
return render_template('public/show_error.html', code=highlighted_xml)