def get_localized_datatype(self, datatype):
out = {'fao_datatype': datatype}
v = VocabularyTerm.get(Vocabulary.VOCABULARY_DATATYPE, datatype)
for l in v.labels:
lname = 'fao_datatype_{}'.format(l.lang)
out[lname] = l.label
return out
def get_fao_agrovoc_term(name):
lang = get_lang()
term = VocabularyTerm.get(Vocabulary.VOCABULARY_AGROVOC, name)
if term:
label = term.get_label(lang) or term.get_label(DEFAULT_LANG) or term.get_label('en')
return label.label
return name
def get_fao_m49_region(name):
lang = get_lang()
term = VocabularyTerm.get(Vocabulary.VOCABULARY_M49_REGIONS, name)
if term:
label = term.get_label(lang) or term.get_label(DEFAULT_LANG) or term.get_label('en')
return label.label
return name
def get_url_for_location(location_code):
lang = get_lang()
term = VocabularyTerm.get(Vocabulary.VOCABULARY_M49_REGIONS, location_code)
if not term:
return h.url_for('search')
label = term.get_label(lang).label or term.get_label('en').label
qdict = {'fao_m49_regions_l{}_{}'.format(term.depth, lang): label}
return h.url_for('search', **qdict)
def get_vocabulary_items_annotated(vocabulary_name,
is_multiple=False,
filters=None):
return VocabularyTerm.get_terms(vocabulary_name,
lang=get_lang(),
include_dataset_count=True,
is_multiple=is_multiple,
filters=filters)
def get_vocabulary_items(vocabulary_name,
is_multiple=False,
filters=None,
order_by=None):
return VocabularyTerm.get_terms(vocabulary_name,
lang=get_lang(),
is_multiple=is_multiple,
filters=filters,
order_by=order_by)
def get_localized_regions(self, regions):
out = {'fao_m49_regions': regions}
for reg in regions:
v = VocabularyTerm.get(Vocabulary.VOCABULARY_M49_REGIONS, reg)
parent = v
while parent:
for label in parent.labels:
lname = 'fao_m49_regions_l{}_{}'.format(parent.depth, label.lang)
try:
out[lname].add(label.label)
except KeyError:
out[lname] = set([label.label])
parent = parent.parent
for k,v in out.items():
if isinstance(v, set):
out[k] = list(v)
return out
def get_localized_agrovoc(self, terms):
if not isinstance(terms, list):
terms = v._deserialize_from_array(terms)
out = {'fao_agrovoc': terms}
for term in terms:
term = VocabularyTerm.get(Vocabulary.VOCABULARY_AGROVOC, term)
if not term:
continue
for label in term.labels:
lname = 'fao_agrovoc_{}'.format(label.lang)
try:
out[lname].add(label.label)
except KeyError:
out[lname] = set([label.label])
for k,val in out.items():
if isinstance(val, set):
out[k] = list(val)
return out
def get_field_data(data, field, lang=None):
if field.get('element') == 'agrovoc':
values = v._deserialize_from_array(data)
out = []
lang = lang or get_lang()
for val in values:
if not val:
continue
term = VocabularyTerm.get(Vocabulary.VOCABULARY_AGROVOC, val)
if not term:
out.append(u'{}|{}'.format(val, val))
continue
label = term.get_label(lang)
if not label:
label = term.get_label('en')
out.append(u'{}|{}'.format(val, label.label or val))
print('out', out)
return out
elif field.get('multiple'):
return v._deserialize_from_array(data)
else:
return data
def import_stage(self, harvest_object):
content = harvest_object.content
# when xml is unicode and has charset declaration in <?xml.. part
# this will raise exception in lxml
# ValueError: Unicode strings with encoding declaration are not supported.
# Please use bytes input or XML fragments without declaration.
# that's why we encode to str
if isinstance(content, unicode):
try:
content = content.encode('utf-8')
except UnicodeEncodeError:
pass
harvest_object.content = content
ret = super(FaoNadaHarvester, self).import_stage(harvest_object)
if not ret:
return ret
# datatype
pkg_id = harvest_object.package_id
context = self._get_context()
pget = toolkit.get_action('package_show')
pupd = toolkit.get_action('package_update')
ret = pget(context, {'name_or_id': pkg_id})
# get locations/m49
ckan_metadata = DdiCkanMetadata()
_pkg_dict = ckan_metadata.load(harvest_object.content)
region_name = _pkg_dict.get('country')
region = None
if region_name:
region = VocabularyTerm.get_term(Vocabulary.VOCABULARY_M49_REGIONS,
region_name).first()
if region:
region = region[0]
extras = ret.get('extras') or []
# force microdata datatype for imported packages
ret['fao_datatype'] = 'microdata'
# agrovoc is local, so we need to preserve
found_in_extras = -1
for idx, ex in enumerate(extras):
if ex['key'] == 'fao_agrovoc':
found_in_extras = idx
break
if found_in_extras > -1:
orig_agrovoc = extras[found_in_extras]
# might be Missing instance
if isinstance(orig_agrovoc['value'], (
str,
unicode,
)):
ret['fao_agrovoc'] = orig_agrovoc['value']
else:
ret['fao_agrovoc'] = '{}'
else:
ret['fao_agrovoc'] = '{}'
# remove duplicated extras
ret['extras'] = [
ex for ex in extras if not ex['key'].startswith('fao_')
]
ret['fao_m49_regions'] = '{{{}}}'.format(
region.name) if region else '{}'
ret['metadata_modified'] = None
out = pupd(self._get_context(), ret)
return ret
def get_fao_datatype(name):
lang = get_lang()
term = VocabularyTerm.get(Vocabulary.VOCABULARY_DATATYPE, name)
label = term.get_label(lang) or term.get_label(
DEFAULT_LANG) or term.get_label('en')
return (label.label or term.name) if term else None
def get_locations_featured():
lang = get_lang()
return VocabularyTerm.get_most_frequent_parent(
Vocabulary.VOCABULARY_M49_REGIONS, lang=lang, multiple=True, limit=4)