class RFA(clcsv.SheetWrapper):
EMPTY_STRING_AS_NONE = True
COERCE = {
"Constituency Name": dataobj.to_unicode(),
"Region": dataobj.to_unicode(),
"Country": dataobj.to_unicode(),
"Constituency ID": dataobj.to_unicode(),
"Constituency Type": dataobj.to_unicode()
}
DEFAULT_COERCE = [dataobj.to_int()]
def add_identifier(self, id, type):
if id is None or type is None:
return
uc = dataobj.to_unicode()
obj = {"id" : self._coerce(id, uc), "type" : self._coerce(type, uc)}
self._delete_from_list("metadata.identifier", matchsub=obj, prune=False)
self._add_to_list("metadata.identifier", obj)
def add_identifier(self, id, type):
if id is None or type is None:
return
uc = dataobj.to_unicode()
obj = {"id" : self._coerce(id, uc), "type" : self._coerce(type, uc)}
self._delete_from_list("metadata.identifier", matchsub=obj, prune=False)
self._add_to_list("metadata.identifier", obj)
class WDI(clcsv.SheetWrapper):
EMPTY_STRING_AS_NONE = True
HEADERS = {
"Country Name": "country",
"Country Code": "cc",
"Indicator Name": "indicator",
"Indicator Code": "ic"
}
COERCE = {
"country": dataobj.to_unicode(),
"cc": dataobj.to_unicode(),
"indicator": dataobj.to_unicode(),
"ic": dataobj.to_unicode()
}
DEFAULT_COERCE = [dataobj.to_float()]
def license(self, obj):
# validate the object structure quickly
allowed = ["title", "type", "url", "version"]
for k in obj.keys():
if k not in allowed:
raise dataobj.DataSchemaException("License object must only contain the following keys: {x}".format(x=", ".join(allowed)))
# coerce the values of the keys
uc = dataobj.to_unicode()
for k in allowed:
if k in obj:
obj[k] = self._coerce(obj[k], uc)
# finally write it
self._set_single("metadata.license_ref", obj)
def license(self, obj):
# validate the object structure quickly
allowed = ["title", "type", "url", "version"]
for k in obj.keys():
if k not in allowed:
raise dataobj.DataSchemaException("License object must only contain the following keys: {x}".format(x=", ".join(allowed)))
# coerce the values of the keys
uc = dataobj.to_unicode()
for k in allowed:
if k in obj:
obj[k] = self._coerce(obj[k], uc)
# finally write it
self._set_single("metadata.license_ref", obj)
def projects(self, objlist):
# validate the object structure quickly
allowed = ["name", "grant_number", "identifier"]
for obj in objlist:
for k in obj.keys():
if k not in allowed:
raise dataobj.DataSchemaException("Project object must only contain the following keys: {x}".format(x=", ".join(allowed)))
# coerce the values of some of the keys
uc = dataobj.to_unicode()
for k in ["name", "grant_number"]:
if k in obj:
obj[k] = self._coerce(obj[k], uc)
# finally write it
self._set_list("metadata.project", objlist)
def projects(self, objlist):
# validate the object structure quickly
allowed = ["name", "grant_number", "identifier"]
for obj in objlist:
for k in obj.keys():
if k not in allowed:
raise dataobj.DataSchemaException("Project object must only contain the following keys: {x}".format(x=", ".join(allowed)))
# coerce the values of some of the keys
uc = dataobj.to_unicode()
for k in ["name", "grant_number"]:
if k in obj:
obj[k] = self._coerce(obj[k], uc)
# finally write it
self._set_list("metadata.project", objlist)
class SpeciesSheet(clcsv.SheetWrapper):
HEADERS = {
# main identifying field
u'Species ID': u'species_id',
# fields in the species sheet
u'Kingdom': u'kingdom',
u'Phylum': u'phylum',
u'Class': u'species_class',
u'Order': u'order',
u'Family': u'family',
u'Genus': u'genus',
u'Species': u'species',
u'Authority': u'authority',
u'Infraspecific rank': u'infraspecific_rank',
u'Infraspecific name': u'infraspecific_name',
u'Infraspecific authority': u'infraspecific_authority',
u'Stock/subpopulation': u'subpopulation',
u'Synonyms': u'synonyms',
u'Common names (Eng)': u'common_names_eng',
u'Common names (Fre)': u'common_names_fre',
u'Common names (Spa)': u'common_names_spa',
u'Red List status': u'red_list_status_code',
u'Red List criteria': u'red_list_criteria',
u'Red List criteria version': u'red_list_criteria_version',
u'Year assessed': u'year_assessed',
u'Population trend': u'population_trend',
u'Petitioned': u'petitioned',
}
COERCE = {
"synonyms": synonym,
"common_names_eng": common_name,
"red_list_criteria_version": dataobj.to_float(),
"year_assessed": dataobj.to_int(),
"petitioned": petition,
}
DEFAULT_COERCE = [dataobj.to_unicode()]
IGNORE_VALUES = {}
EMPTY_STRING_AS_NONE = True
def __init__(self, path):
super(SpeciesSheet, self).__init__(path)
def source_name(self, val):
self._set_single("metadata.source.name", val, coerce=dataobj.to_unicode())
def set_license(self, type, url):
uc = dataobj.to_unicode()
type = self._coerce(type, uc)
url = self._coerce(url, uc)
obj = {"title" : type, "type" : type, "url" : url}
self._set_single("metadata.license_ref", obj)
def id(self):
return self._get_single("id", coerce=dataobj.to_unicode())
def publisher(self):
return self._get_single("metadata.publisher", coerce=dataobj.to_unicode())
def add_subject(self, kw):
self._add_to_list("metadata.subject", kw, coerce=dataobj.to_unicode(), unique=True)
def publisher(self, val):
self._set_single("metadata.publisher", val, coerce=dataobj.to_unicode(), allow_none=False, ignore_none=True)
def source_name(self, val):
self._set_single("metadata.source.name", val, coerce=dataobj.to_unicode())
def packaging_format(self):
return self._get_single("content.packaging_format", dataobj.to_unicode())
def test_02_construct(self):
struct = {
"fields" : {
"one" : {"coerce" : "unicode"}
},
"objects" : [
"three", "four", "five"
],
"lists" : {
"six" : {"contains" : "whatever"},
"seven" : {"contains" : "field", "coerce" : "integer"},
"eight" : {"contains" : "object"},
"nine" : {"contains" : "object"}
},
"required" : ["one"],
"structs" : {
# Note there's no structure for three, so it can be anything
"four" : {
"fields" : {
"alpha" : {"coerce" : "integer"}
}
},
"nine" : {
"fields" : {
"beta" : {"coerce" : "integer"}
}
}
}
}
coerce = {
"unicode" : dataobj.to_unicode(),
"integer" : dataobj.to_int()
}
obj = { "one" : "hello" }
new = dataobj.construct(obj, struct, coerce)
assert new["one"] == u"hello"
assert isinstance(new["one"], unicode)
# try adding a disallowed field
obj["two"] = "world"
with self.assertRaises(dataobj.DataStructureException):
try:
new = dataobj.construct(obj, struct, coerce)
except dataobj.DataStructureException as e:
print e.message
raise e
# try not providing a required field
obj = {}
with self.assertRaises(dataobj.DataStructureException):
try:
new = dataobj.construct(obj, struct, coerce)
except dataobj.DataStructureException as e:
print e.message
raise e
# try an unvalidated sub-object
obj = {"one" : "hello"}
obj["three"] = {"something" : "here"}
new = dataobj.construct(obj, struct, coerce)
assert new["three"]["something"] == "here"
# and a validated sub-object
obj["four"] = {"alpha" : "4"}
new = dataobj.construct(obj, struct, coerce)
assert new["four"]["alpha"] == 4
# and a field that should be an object but isn't
obj["five"] = "something here"
with self.assertRaises(dataobj.DataStructureException):
try:
new = dataobj.construct(obj, struct, coerce)
except dataobj.DataStructureException as e:
print e.message
raise e
# a list where the list contains directive is broken
del obj["five"]
obj["six"] = ["6"]
with self.assertRaises(dataobj.DataStructureException):
try:
new = dataobj.construct(obj, struct, coerce)
except dataobj.DataStructureException as e:
print e.message
raise e
# successfully coerce a list of field values
del obj["six"]
obj["seven"] = ["1", "1", "2", "3"]
new = dataobj.construct(obj, struct, coerce)
assert new["seven"] == [1, 1, 2, 3]
# faile to coerce a list of field values
obj["seven"] = ["a", "b", "walton-upon-thames"]
with self.assertRaises(dataobj.DataStructureException):
try:
new = dataobj.construct(obj, struct, coerce)
except dataobj.DataStructureException as e:
print e.message
raise e
# list is supposed to contain an object but doesn't
del obj["seven"]
obj["eight"] = ["not an object"]
with self.assertRaises(dataobj.DataStructureException):
try:
new = dataobj.construct(obj, struct, coerce)
except dataobj.DataStructureException as e:
print e.message
raise e
# list contains an object but no validation required on its structure
obj["eight"] = [{"an" : "object"}]
new = dataobj.construct(obj, struct, coerce)
assert new["eight"][0]["an"] == "object"
# list contains an object which fails to validate
obj["nine"] = [{"beta" : "whatever"}]
with self.assertRaises(dataobj.DataStructureException):
try:
new = dataobj.construct(obj, struct, coerce)
except dataobj.DataStructureException as e:
print e.message
raise e
# list contains an object that does validate
obj["nine"] = [{"beta" : 9}]
new = dataobj.construct(obj, struct, coerce)
assert new["nine"][0]["beta"] == 9
def packaging_format(self, val):
self._set_single("content.packaging_format", val, coerce=dataobj.to_unicode())
def id(self):
return self._get_single("id", coerce=dataobj.to_unicode())
def set_license(self, type, url):
uc = dataobj.to_unicode()
type = self._coerce(type, uc)
url = self._coerce(url, uc)
obj = {"title" : type, "type" : type, "url" : url}
self._set_single("metadata.license_ref", obj)
def source_name(self):
return self._get_single("metadata.source.name", coerce=dataobj.to_unicode())
def add_subject(self, kw):
self._add_to_list("metadata.subject", kw, coerce=dataobj.to_unicode(), unique=True)
def type(self, val):
self._set_single("metadata.type", val, coerce=dataobj.to_unicode(), allow_none=False, ignore_none=True)
def my_title(self, val):
self._set_single("title", val, coerce=dataobj.to_unicode())
def publisher(self):
return self._get_single("metadata.publisher", coerce=dataobj.to_unicode())
def publisher(self, val):
self._set_single("metadata.publisher", val, coerce=dataobj.to_unicode(), allow_none=False, ignore_none=True)
def source_name(self):
return self._get_single("metadata.source.name", coerce=dataobj.to_unicode())
def version(self, val):
self._set_single("metadata.version", val, coerce=dataobj.to_unicode())
def language(self):
# Note that in this case we don't coerce to iso language, as it's a slightly costly operation, and all incoming
# data should already be coerced
return self._get_single("metadata.language", coerce=dataobj.to_unicode())
def test_02_construct(self):
struct = {
"fields": {
"one": {
"coerce": "unicode"
}
},
"objects": ["three", "four", "five"],
"lists": {
"six": {
"contains": "whatever"
},
"seven": {
"contains": "field",
"coerce": "integer"
},
"eight": {
"contains": "object"
},
"nine": {
"contains": "object"
}
},
"required": ["one"],
"structs": {
# Note there's no structure for three, so it can be anything
"four": {
"fields": {
"alpha": {
"coerce": "integer"
}
}
},
"nine": {
"fields": {
"beta": {
"coerce": "integer"
}
}
}
}
}
coerce = {"unicode": dataobj.to_unicode(), "integer": dataobj.to_int()}
obj = {"one": "hello"}
new = dataobj.construct(obj, struct, coerce)
assert new["one"] == u"hello"
assert isinstance(new["one"], unicode)
# try adding a disallowed field
obj["two"] = "world"
with self.assertRaises(dataobj.DataStructureException):
try:
new = dataobj.construct(obj, struct, coerce)
except dataobj.DataStructureException as e:
print e.message
raise e
# try not providing a required field
obj = {}
with self.assertRaises(dataobj.DataStructureException):
try:
new = dataobj.construct(obj, struct, coerce)
except dataobj.DataStructureException as e:
print e.message
raise e
# try an unvalidated sub-object
obj = {"one": "hello"}
obj["three"] = {"something": "here"}
new = dataobj.construct(obj, struct, coerce)
assert new["three"]["something"] == "here"
# and a validated sub-object
obj["four"] = {"alpha": "4"}
new = dataobj.construct(obj, struct, coerce)
assert new["four"]["alpha"] == 4
# and a field that should be an object but isn't
obj["five"] = "something here"
with self.assertRaises(dataobj.DataStructureException):
try:
new = dataobj.construct(obj, struct, coerce)
except dataobj.DataStructureException as e:
print e.message
raise e
# a list where the list contains directive is broken
del obj["five"]
obj["six"] = ["6"]
with self.assertRaises(dataobj.DataStructureException):
try:
new = dataobj.construct(obj, struct, coerce)
except dataobj.DataStructureException as e:
print e.message
raise e
# successfully coerce a list of field values
del obj["six"]
obj["seven"] = ["1", "1", "2", "3"]
new = dataobj.construct(obj, struct, coerce)
assert new["seven"] == [1, 1, 2, 3]
# faile to coerce a list of field values
obj["seven"] = ["a", "b", "walton-upon-thames"]
with self.assertRaises(dataobj.DataStructureException):
try:
new = dataobj.construct(obj, struct, coerce)
except dataobj.DataStructureException as e:
print e.message
raise e
# list is supposed to contain an object but doesn't
del obj["seven"]
obj["eight"] = ["not an object"]
with self.assertRaises(dataobj.DataStructureException):
try:
new = dataobj.construct(obj, struct, coerce)
except dataobj.DataStructureException as e:
print e.message
raise e
# list contains an object but no validation required on its structure
obj["eight"] = [{"an": "object"}]
new = dataobj.construct(obj, struct, coerce)
assert new["eight"][0]["an"] == "object"
# list contains an object which fails to validate
obj["nine"] = [{"beta": "whatever"}]
with self.assertRaises(dataobj.DataStructureException):
try:
new = dataobj.construct(obj, struct, coerce)
except dataobj.DataStructureException as e:
print e.message
raise e
# list contains an object that does validate
obj["nine"] = [{"beta": 9}]
new = dataobj.construct(obj, struct, coerce)
assert new["nine"][0]["beta"] == 9
def type(self, val):
self._set_single("metadata.type", val, coerce=dataobj.to_unicode(), allow_none=False, ignore_none=True)
def version(self):
return self._get_single("metadata.version", coerce=dataobj.to_unicode())
def type(self):
return self._get_single("metadata.type", coerce=dataobj.to_unicode())
def version(self, val):
self._set_single("metadata.version", val, coerce=dataobj.to_unicode())
def packaging_format(self, val):
self._set_single("content.packaging_format", val, coerce=dataobj.to_unicode())
def type(self):
return self._get_single("metadata.type", coerce=dataobj.to_unicode())
def packaging_format(self):
return self._get_single("content.packaging_format", dataobj.to_unicode())
def language(self):
# Note that in this case we don't coerce to iso language, as it's a slightly costly operation, and all incoming
# data should already be coerced
return self._get_single("metadata.language", coerce=dataobj.to_unicode())
def my_title(self, val):
self._set_single("title", val, coerce=dataobj.to_unicode())
def version(self):
return self._get_single("metadata.version", coerce=dataobj.to_unicode())
