def test_harvester(self):
di = logging.getLogger('ckanext.ddi.harvesters.ddiharvester')
bs = logging.getLogger('ckanext.harvest.harvesters.base')
di.setLevel(logging.DEBUG)
bs.setLevel(logging.DEBUG)
sout = logging.StreamHandler(sys.stdout)
sout.setLevel(logging.DEBUG)
bs.addHandler(sout)
di.addHandler(sout)
cli = VocabularyCommands('vocabulary')
cli.cmd_import_agrovoc(_get_path('agrovoc_excerpt.nt'))
cli.cmd_load('datatype', _get_path('faociok.datatype.csv'))
cli.cmd_import_m49(_get_path('M49_Codes.xlsx'))
h = self._create_harvest_obj('http://test/sourc/a',
source_type='fao-nada')
hobj = HarvestObject.get(h['id'])
with gzip.open(_get_path('harvest_object_content.gz'), 'rb') as f:
hobj.content = f.read()
harv = FaoNadaHarvester()
out = harv.import_stage(hobj)
self.assertTrue(isinstance(out, dict), [(herr.message, herr.stage, herr.line) for herr in hobj.errors])
self.assertEqual(out.get('fao_datatype'), 'microdata', out.get('fao_datatype'))
# 188 - Costa Rica
self.assertEqual(out.get('fao_m49_regions'), '{188}', out.get('fao_m49_regions'))
self.assertTrue(out.get('fao_agrovoc') in ('{}', []), out.get('fao_agrovoc'))
def test_vocabulary_term_rename(self):
"""
Test vocabulary rename_term command
"""
cli = VocabularyCommands('vocabulary')
cli.cmd_import_m49(_get_path('M49_Codes.xlsx'))
_load_vocabulary('datatype', 'faociok.datatype.csv')
cli.cmd_import_agrovoc(_get_path('agrovoc_excerpt.nt'))
dataset = {'title': 'some title',
'id': 'sometitle',
'name': 'sometitle',
'fao_datatype': 'other',
'fao_m49_regions': ['9', '8'],
'fao_agrovoc': ['c_432', 'c_7020'],
'resources': [],
}
pkg = self._create_dataset(dataset)
self.assertRaises(ValueError, cli.cmd_rename_term, 'invalid-vocabulary', None, None)
self.assertRaises(ValueError, cli.cmd_rename_term, 'datatype', 'missing', 'other')
self.assertRaises(ValueError, cli.cmd_rename_term, 'datatype', 'monitoring', 'otherrr')
cli.cmd_rename_term('datatype', 'other', 'microdata')
# we're changing db in line above, need to flush
Session.commit()
q = Session.query(PackageExtra.package_id).join(Package, Package.id==PackageExtra.package_id)\
.filter(PackageExtra.key=='fao_datatype',
PackageExtra.value=='microdata',
Package.state=='active')\
.scalar()
self.assertEqual(q, pkg['id'])
package_show = t.get_action('package_show')
p = package_show({'ignore_auth': True,
'use_cache': False},
{'name_or_id': 'sometitle'})
self.assertEqual(p['fao_datatype'], 'microdata', p)
cli.cmd_rename_term('m49_regions', '9', '21')
Session.commit()
p = package_show({'ignore_auth': True,
'use_cache': False},
{'name_or_id': 'sometitle'})
self.assertEqual(set(p['fao_m49_regions']), set(['8', '21']), p.get('fao_m49_regions'))
def test_m49_validator(self):
"""
Test m49 regions validator
"""
cli = VocabularyCommands('vocabulary')
cli.cmd_load('datatype', _get_path('faociok.datatype.csv'))
cli.cmd_import_m49(_get_path('M49_Codes.xlsx'))
test_values = (('region', False, None,),
([], True, [],),
# oceania, albania
('{9,8}', True, ['9', '8'],),
# south america, not in vocabulary
('{5}', False, None,),
('{9,8,5}', False, None,),
)
_run_validator_checks(test_values, t.get_validator('fao_m49_regions'))
def test_autocomplete_view(self):
cli = VocabularyCommands('vocabulary')
cli.cmd_load('datatype', _get_path('faociok.datatype.csv'))
cli.cmd_import_m49(_get_path('M49_Codes.xlsx'))
# hack on stacked db session. cli will import data to one db session
# webapp view may use different session/transaction to retrive it
# so we make sure session is commited
model.Session.commit()
expected_id = '380'
app = self._get_test_app()
resp = app.get('/api/util/fao/autocomplete/m49_regions?incomplete=ita&lang=en')
found_it = False
for result in resp.json['ResultSet']['Result']:
if result['term'] == expected_id:
found_it = result
break
self.assertTrue(isinstance(found_it, dict), resp.json)
self.assertEqual(found_it['label'], 'Italy', found_it)
resp = app.get('/api/util/fao/autocomplete/m49_regions?incomplete=ita&lang=fr')
self.assertTrue(len(resp.json['ResultSet']['Result'])> 0, resp.json)
found_it = False
for result in resp.json['ResultSet']['Result']:
if result['term'] == expected_id:
found_it = result
break
self.assertTrue(isinstance(found_it, dict), resp.json)
self.assertTrue(found_it['label'].endswith(' [FR]'))
resp = app.get('/api/util/fao/autocomplete/datatype?incomplete=oth&lang=fr')
self.assertTrue(len(resp.json['ResultSet']['Result']) == 1, resp.json)
found_it = resp.json['ResultSet']['Result'][0]
self.assertTrue(isinstance(found_it, dict), resp.json)
self.assertEqual(found_it['label'].strip(), 'other FR', found_it)
def test_agrovoc_validator(self):
"""
Test Agrovoc validator
"""
cli = VocabularyCommands('vocabulary')
cli.cmd_import_agrovoc(_get_path('agrovoc_excerpt.nt'))
test_values = (('region', False, None,),
([], True, [],),
('{c_100,c_200}', False, None,),
('{c_432,c_7020,c_100}', False, None,),
('{c_432,c_7020}', True, ['c_432', 'c_7020'],),
)
_run_validator_checks(test_values, t.get_validator('fao_agrovoc'))