def openness_score(pkg):
score = 0
fmt = schema_description.resource_field_by_id['format']['choices_by_key']
for r in pkg['resources']:
if r['resource_type'] != 'file' and r['resource_type'] != 'api':
continue
resource_score = fmt[r['format']]['openness_score']
if boolean_validator(r.get('data_includes_uris', ''), {}):
resource_score = 4
if boolean_validator(r.get('data_includes_links', ''), {}):
resource_score = 5
score = max(score, resource_score)
return score
def openness_score(pkg):
score = 0
fmt = schema_description.resource_field_by_id['format']['choices_by_key']
for r in pkg['resources']:
if r['resource_type'] != 'file' and r['resource_type'] != 'api':
continue
resource_score = fmt[r['format']]['openness_score']
if boolean_validator(r.get('data_includes_uris', ''), {}):
resource_score = 4
if boolean_validator(r.get('data_includes_links', ''), {}):
resource_score = 5
score = max(score, resource_score)
return score
def _trim_package(pkg):
"""
remove keys from pkg that we don't care about when comparing
or updating/creating packages. Also try to convert types and
create missing fields that will be present in package_show.
"""
# XXX full of custom hacks and deep knowledge of our schema :-(
if not pkg:
return
for k in ['extras', 'metadata_modified', 'metadata_created',
'revision_id', 'revision_timestamp', 'organization',
'version', 'tracking_summary',
'tags', # just because we don't use them
'num_tags', 'num_resources', 'maintainer',
'isopen', 'relationships_as_object', 'license_title',
'license_title_fra', 'license_url_fra', 'license_url',
'author',
'groups', # just because we don't use them
'relationships_as_subject', 'department_number',
# FIXME: remove these when we can:
'resource_type',
# new in 2.3:
'creator_user_id',
]:
if k in pkg:
del pkg[k]
for r in pkg['resources']:
for k in ['package_id', 'revision_id',
'revision_timestamp', 'cache_last_updated',
'webstore_last_updated', 'state', 'hash',
'description', 'tracking_summary', 'mimetype_inner',
'mimetype', 'cache_url', 'created', 'webstore_url',
'last_modified', 'position']:
if k in r:
del r[k]
for k in ['name', 'size']:
if k not in r:
r[k] = None
for k in ['private']:
pkg[k] = boolean_validator(unicode(pkg.get(k, '')), None)
if 'name' not in pkg:
pkg['name'] = pkg['id']
if 'type' not in pkg:
pkg['type'] = 'dataset'
if 'state' not in pkg:
pkg['state'] = 'active'
for k in ['url']:
if k not in pkg:
pkg[k] = ''
for name, lang, field in schema_description.dataset_field_iter():
if field['type'] == 'date':
try:
pkg[name] = str(isodate(pkg[name], None)) if pkg.get(name) else ''
except Invalid:
pass # not for us to fail validation
elif field['type'] == 'url':
if not pkg.get(name): # be consistent about what an empty url is
pkg[name] = ""
elif field['type'] == 'fixed' and name in pkg:
del pkg[name]
def test_missing(self):
assert_equals(validators.boolean_validator('', None), False)
def test_bool_false(self):
assert_equals(validators.boolean_validator(False, None), False)
def test_bool_true(self):
assert_equals(validators.boolean_validator(True, None), True)
def test_string_false(self):
assert_equals(validators.boolean_validator('f', None), False)
def test_string_true(self):
assert_equals(validators.boolean_validator('true', None), True)
assert_equals(validators.boolean_validator('yes', None), True)
assert_equals(validators.boolean_validator('t', None), True)
assert_equals(validators.boolean_validator('y', None), True)
assert_equals(validators.boolean_validator('1', None), True)
def test_bool_false():
assert not validators.boolean_validator(False, None)
def test_string_false(self):
assert_equals(validators.boolean_validator('f', None), False)
def test_bool_false(self):
assert_equals(validators.boolean_validator(False, None), False)
def test_bool_true(self):
assert_equals(validators.boolean_validator(True, None), True)
def _trim_package(pkg):
"""
remove keys from pkg that we don't care about when comparing
or updating/creating packages. Also try to convert types and
create missing fields that will be present in package_show.
"""
# XXX full of custom hacks and deep knowledge of our schema :-(
if not pkg:
return
for k in ['extras', 'metadata_modified', 'metadata_created',
'revision_id', 'revision_timestamp', 'organization',
'version', 'tracking_summary',
'tags', # just because we don't use them
'num_tags', 'num_resources', 'maintainer',
'isopen', 'relationships_as_object', 'license_title',
'license_title_fra', 'license_url_fra', 'license_url',
'maintainer_email', 'author',
'groups', # just because we don't use them
'relationships_as_subject', 'department_number',
# FIXME: remove these when we can:
'resource_type',
]:
if k in pkg:
del pkg[k]
for r in pkg['resources']:
for k in ['resource_group_id', 'revision_id',
'revision_timestamp', 'cache_last_updated',
'webstore_last_updated', 'id', 'state', 'hash',
'description', 'tracking_summary', 'mimetype_inner',
'mimetype', 'cache_url', 'created', 'webstore_url',
'last_modified', 'position', ]:
if k in r:
del r[k]
for k in ['name', 'size']:
if k not in r:
r[k] = None
for k in ['ready_to_publish', 'private']:
pkg[k] = boolean_validator(unicode(pkg.get(k, '')), None)
if 'name' not in pkg:
pkg['name'] = pkg['id']
if 'type' not in pkg:
pkg['type'] = 'dataset'
if 'state' not in pkg:
pkg['state'] = 'active'
for k in ['url']:
if k not in pkg:
pkg[k] = ''
for name, lang, field in schema_description.dataset_field_iter():
if field['type'] == 'date':
try:
pkg[name] = str(isodate(pkg[name], None)) if pkg.get(name) else ''
except Invalid:
pass # not for us to fail validation
elif field['type'] == 'tag_vocabulary' and not isinstance(
pkg.get(name), list):
pkg[name] = convert_pilot_uuid_list(field)(pkg.get(name, []))
elif field['type'] == 'url':
if not pkg.get(name): # be consistent about what an empty url is
pkg[name] = ""
elif field['type'] == 'fixed' and name in pkg:
del pkg[name]
def test_string_false():
assert not validators.boolean_validator("f", None)
def test_string_true():
assert validators.boolean_validator("true", None)
assert validators.boolean_validator("yes", None)
assert validators.boolean_validator("t", None)
assert validators.boolean_validator("y", None)
assert validators.boolean_validator("1", None)
def test_none():
assert not validators.boolean_validator(None, None)
def test_missing():
assert not validators.boolean_validator("", None)
def test_none(self):
assert_equals(validators.boolean_validator(None, None), False)
def test_string_true(self):
assert_equals(validators.boolean_validator('true', None), True)
assert_equals(validators.boolean_validator('yes', None), True)
assert_equals(validators.boolean_validator('t', None), True)
assert_equals(validators.boolean_validator('y', None), True)
assert_equals(validators.boolean_validator('1', None), True)
def test_none(self):
assert_equals(validators.boolean_validator(None, None), False)
def test_missing(self):
assert_equals(validators.boolean_validator('', None), False)
def test_bool_true():
assert validators.boolean_validator(True, None)
