def test_fetch_package_list(self, iatikit_mock): data_mock = iatikit_mock.return_value data_mock.last_updated = datetime.datetime.utcnow() data_mock.datasets = [ iatikit.Dataset("tst-a.xml"), iatikit.Dataset("tst-b.xml") ] datasets = crawler.fetch_dataset_list() self.assertIn("tst-a", [ds.name for ds in datasets]) self.assertIn("tst-b", [ds.name for ds in datasets])
def test_update_deletes_datasets(self, iatikit_mock): data_mock = iatikit_mock.return_value data_mock.last_updated = datetime.datetime.utcnow() data_mock.datasets = [ iatikit.Dataset("tst-a.xml"), iatikit.Dataset("tst-b.xml") ] datasets = crawler.fetch_dataset_list() self.assertEquals(2, datasets.count()) data_mock.datasets = [iatikit.Dataset("tst-a.xml")] datasets = crawler.fetch_dataset_list() self.assertEquals(1, datasets.count())
def validate(uuid): """Show the validation results for a supplied dataset.""" supplied_data = SuppliedData.query.get_or_404(str(uuid)) filepath = join(current_app.config['MEDIA_FOLDER'], supplied_data.original_file) if not exists(filepath): flash('Error: That dataset is no longer available', 'danger') return redirect(url_for('public.home')) dataset = iatikit.Dataset(filepath) if supplied_data.validated: errors = { 'xml_errors': supplied_data.xml_errors, 'iati_errors': supplied_data.iati_errors, 'codelist_errors': supplied_data.codelist_errors, } else: errors = { 'xml_errors': [], 'iati_errors': [], 'codelist_errors': [], } valid_xml = dataset.validate_xml() if valid_xml: dataset.unminify_xml() valid_iati = dataset.validate_iati() for error, count in valid_iati.error_summary: iati_error = ValidationError('iati_error', error, count, supplied_data) errors['iati_errors'].append(iati_error) db.session.add(iati_error) valid_codelists = dataset.validate_codelists() for error, count in valid_codelists.error_summary: codelist_error = ValidationError('codelist_error', error, count, supplied_data) errors['codelist_errors'].append(codelist_error) db.session.add(codelist_error) else: for error, count in valid_xml.error_summary: xml_error = ValidationError('xml_error', error, count, supplied_data) errors['xml_errors'].append(xml_error) db.session.add(xml_error) success = all([e == [] for e in errors.values()]) supplied_data.validated = True db.session.add(supplied_data) db.session.commit() return render_template('public/validate.html', data=supplied_data, dataset=dataset, errors=errors, success=success)
def badge(): """Show the validation status of a dataset as an SVG badge.""" source_url = request.args.get('url') if source_url is None: svg_file = join('static', 'badges', 'no-url.svg') return send_file(svg_file, mimetype='image/svg+xml') supplied_data = SuppliedData(source_url, None, None, 'url_form') filepath = join(current_app.config['MEDIA_FOLDER'], supplied_data.original_file) dataset = iatikit.Dataset(filepath) if dataset.validate_xml() and dataset.validate_iati() \ and dataset.validate_codelists(): svg_file = join('static', 'badges', 'passing.svg') else: svg_file = join('static', 'badges', 'failing.svg') return send_file(svg_file, mimetype='image/svg+xml')
def show(uuid): """Show a validation error in its XML context.""" validation_error = ValidationError.query.get_or_404(str(uuid)) filepath = join(current_app.config['MEDIA_FOLDER'], validation_error.supplied_data.original_file) if not exists(filepath): flash('That dataset is no longer available', 'danger') return redirect(url_for('public.home')) match = re.search(r'/iati-(?:activity|organisation)\[(\d+)\]', validation_error.path) act_num = int(match.group(1)) - 1 dataset = iatikit.Dataset(filepath) dataset.unminify_xml() activity = dataset.activities[act_num] start_line = activity.etree.sourceline line = validation_error.line - start_line + 1 highlighted_xml = highlight( activity.xml, XmlLexer(), HtmlFormatter(linenos='inline', lineanchors='L', hl_lines=[line], linenostart=start_line)) return render_template('public/show_error.html', code=highlighted_xml)