def _is_clean_date(self,db_string):
if not self.regex_year.match(db_string) \
and not self.regex_year_month.match(db_string) \
and not self.regex_year_month_day.match(db_string):
return False
try:
parsed = DateType.parse_timedate(db_string,'db')
assert parsed['readable_format'] in ['YYYY','YYYY-MM','YYYY-MM-DD']
if parsed['readable_format'] in ['YYYY-MM','YYYY-MM-DD']:
assert parsed['month'] in range(1,13)
if parsed['readable_format'] == 'YYYY-MM-DD':
assert parsed['day'] in range(1,32)
assert parsed['year'] in range(1900,2100)
return True
except (DateConvertError,AssertionError), e:
return False
def _is_clean_date(self, db_string):
if not self.regex_year.match(db_string) \
and not self.regex_year_month.match(db_string) \
and not self.regex_year_month_day.match(db_string):
return False
try:
parsed = DateType.parse_timedate(db_string, 'db')
assert parsed['readable_format'] in [
'YYYY', 'YYYY-MM', 'YYYY-MM-DD'
]
if parsed['readable_format'] in ['YYYY-MM', 'YYYY-MM-DD']:
assert parsed['month'] in range(1, 13)
if parsed['readable_format'] == 'YYYY-MM-DD':
assert parsed['day'] in range(1, 32)
assert parsed['year'] in range(1900, 2100)
return True
except (DateConvertError, AssertionError), e:
return False
def test_submit_full_package_edit_form_valid(self):
package = self.get_package_by_name(self.package_name)
data = {
'name': self.package_name_alt,
'title': 'test title',
'url': 'http://someurl.com/',
'notes': 'test notes',
'tags': 'sheep goat fish',
'resources-0-url': 'http://someurl.com/download.csv',
'resources-0-format': 'CSV',
'resources-0-description': 'A csv file',
'author': 'Brian',
'author_email': '*****@*****.**',
'license_id': 'cc-zero',
'published_by': 'Department for Education [3]',
'published_via': 'Ealing PCT [2]',
'temporal_coverage-to': '6/2009',
'temporal_coverage-from': '12:30 24/6/2008',
'date_updated': '12:30 30/7/2009',
'date_released': '30/7/2009',
'date_update_future': '1/7/2009',
'mandate': 'Law 1996',
}
res = self.post_package_edit_form(package.id, **data)
self.assert_blank_response(res)
assert not self.get_package_by_name(self.package_name)
pkg = self.get_package_by_name(self.package_name_alt)
assert pkg
for key in data.keys():
if key.startswith('resources'):
subkey = key.split('-')[-1]
pkg_value = getattr(pkg.resources[0], subkey)
elif key == 'tags':
pkg_value = set([tag.name for tag in pkg.tags])
data[key] = set(data[key].split())
elif key in self.core_package_fields:
pkg_value = getattr(pkg, key)
else:
# an extra
pkg_value = pkg.extras[key]
if '-' in pkg_value:
pkg_value = DateType.db_to_form(pkg_value)
assert pkg_value == data[key], '%r should be %r but is %r' % (
key, data[key], pkg_value)
def test_submit_full_package_edit_form_valid(self):
package = self.get_package_by_name(self.package_name)
data = {
'name':self.package_name_alt,
'title':'test title',
'url':'http://someurl.com/',
'notes':'test notes',
'tags':'sheep goat fish',
'resources-0-url':'http://someurl.com/download.csv',
'resources-0-format':'CSV',
'resources-0-description':'A csv file',
'author':'Brian',
'author_email':'*****@*****.**',
'license_id':'cc-zero',
'published_by':'Department for Education [3]',
'published_via':'Ealing PCT [2]',
'temporal_coverage-to':'6/2009',
'temporal_coverage-from':'12:30 24/6/2008',
'date_updated':'12:30 30/7/2009',
'date_released':'30/7/2009',
'date_update_future':'1/7/2009',
'mandate':'Law 1996',
}
res = self.post_package_edit_form(package.id, **data)
self.assert_blank_response(res)
assert not self.get_package_by_name(self.package_name)
pkg = self.get_package_by_name(self.package_name_alt)
assert pkg
for key in data.keys():
if key.startswith('resources'):
subkey = key.split('-')[-1]
pkg_value = getattr(pkg.resources[0], subkey)
elif key == 'tags':
pkg_value = set([tag.name for tag in pkg.tags])
data[key] = set(data[key].split())
elif key in self.core_package_fields:
pkg_value = getattr(pkg, key)
else:
# an extra
pkg_value = pkg.extras[key]
if '-' in pkg_value:
pkg_value = DateType.db_to_form(pkg_value)
assert pkg_value == data[key], '%r should be %r but is %r' % (key, data[key], pkg_value)
def date_to_form(value, context):
try:
value = DateType.db_to_form(value)
except DateConvertError, e:
raise Invalid(str(e))
def date_to_form(value, context):
try:
value = DateType.db_to_form(value)
except DateConvertError, e:
raise Invalid(str(e))
try:
title = row[0].value.encode('utf-8')
except Exception, e:
raise Exception('Error with encoding of Title: %s' % e)
try:
description = row[1].value.encode('utf-8')
except Exception, e:
raise Exception('Error with encoding of Description: %s' % e)
publisher_name = row[2].value
publish_date = row[3].value
release_notes = row[4].value
if isinstance(publish_date, basestring) and publish_date.strip():
# e.g. CSV containing "1/2/14" -> "14/02/01"
try:
publish_date = DateType.form_to_db(publish_date)
except DateConvertError:
# Lots of text went into this field but have decided to not allow from now
# and it never gets displayed.
msg = 'Could not parse date: "%s" Must be: DD/MM/YY' % publish_date
log.error(msg)
raise Exception(msg)
if isinstance(publish_date, datetime.datetime):
# e.g. Excel -> "14/02/01"
publish_date = DateType.date_to_db(publish_date)
if not isinstance(publish_date, basestring):
# e.g. Excel int
msg = 'Could not parse date: "%s" Must be: DD/MM/YY' % publish_date
log.error(msg)
raise Exception(msg)
def date_to_form(value, context):
try:
value = DateType.db_to_form(value)
except DateConvertError, e:
raise Invalid(unicode(e).encode('ascii', 'ignore'))
def test_a_full_timeseries_dataset(self):
"""
Tests the submission of a fully-completed timeseries dataset.
"""
package_data = _EXAMPLE_TIMESERIES_DATA
package_name = package_data['name']
CreateTestData.flag_for_deletion(package_name)
assert not self.get_package_by_name(package_name),\
'Package "%s" already exists' % package_name
# create package via form
response = self._form_client.post_form(package_data)
# ensure it's correct
pkg = self.get_package_by_name(package_name)
assert pkg, response.body
assert_equal(package_data['title'], pkg.title)
assert_equal(package_data['notes'], pkg.notes)
publisher = pkg.as_dict()['groups'][0]
assert_equal(package_data['groups__0__name'], publisher)
# Extra data
# Timeseries data
expected_timeseries_keys = filter(lambda k: k.startswith('timeseries_resources'),
package_data.keys())
timeseries_resources = ckanext.dgu.lib.helpers.timeseries_resources(pkg.as_dict())
assert_equal(len(timeseries_resources), 4)
for key in expected_timeseries_keys:
index, field = key.split('__')[1:]
index = int(index)
assert_equal(package_data[key],
timeseries_resources[index][field])
# Publisher / contact details
# The contact-email should not be an extra-field on the dataset as it's the
# same as the publisher group's contact-email. ie - it hasn't been overridden.
# The resof the information should be in the extras fields
assert_not_in('contact-email', pkg.extras)
assert_equal(package_data['contact-name'], pkg.extras['contact-name'])
assert_equal(package_data['contact-phone'], pkg.extras['contact-phone'])
assert_equal(package_data['foi-name'], pkg.extras['foi-name'])
assert_equal(package_data['foi-email'], pkg.extras['foi-email'])
assert_equal(package_data['foi-phone'], pkg.extras['foi-phone'])
# Themes and tags
assert_equal(package_data['theme-primary'], pkg.extras['theme-primary'])
assert_equal(set(package_data['theme-secondary']),
set(pkg.extras['theme-secondary']))
# Health and Education are from the primary and secondary themes, which
# end up in the tags
assert_equal(set(['tag1', 'tag2', 'a multi word tag', 'Health', 'Education']),
set(tag.name for tag in pkg.get_tags()))
# Additional resources
expected_additional_keys = filter(lambda k: k.startswith('additional_resources'),
package_data.keys())
additional_resources = ckanext.dgu.lib.helpers.additional_resources(pkg.as_dict())
assert_equal(len(additional_resources), 2)
for key in expected_additional_keys:
index, field = key.split('__')[1:]
index = int(index)
assert_equal(package_data[key],
additional_resources[index][field])
assert_equal(package_data['mandate'], pkg.extras['mandate'])
assert_equal(package_data['access_constraints'], pkg.license_id)
assert_equal(package_data['temporal_coverage-from'], DateType.db_to_form(pkg.extras['temporal_coverage-from']))
assert_equal(package_data['temporal_coverage-to'], DateType.db_to_form(pkg.extras['temporal_coverage-to']))
assert_in('England', pkg.extras['geographic_coverage'])
def test_edit_form_form_has_all_fields(self):
"""
Asserts that edit-form of a package has the fields prefilled correctly.
"""
package = self.fixtures.pkgs[0]
offset = url_for(controller='package', action='edit', id=package['name'])
response = self.app.get(offset, extra_environ={'REMOTE_USER': self.admin})
# form field name -> expected form field value
expected_field_values = {}
# populate expected_field_values with the simple fields first
for field_name in self._expected_fields:
try:
expected_value = package[field_name]
if isinstance(expected_value, basestring):
expected_field_values[field_name] = expected_value
except KeyError:
pass
# populate expected_field_values for tag_string and license_id
# by hand, as the field names in the package dict don't follow the
# same naming scheme as the form fields.
expected_field_values['license_id'] = package['license']
# tags may be re-ordered, so test them manually
expected_tags = set(map(lambda s: s.strip(), package['tags'].split()))
tag_string_form_value = re.finditer(r'<input [^>]*id="tag_string" name="tag_string" [^>]*value="([^"]+)" />', response.body).next().group(1)
actual_tags = set(map(lambda s: s.strip(), tag_string_form_value.split(',')))
assert_equal(expected_tags, actual_tags)
# Promote the key-value pairs stored in 'extras' to form fields.
expected_field_values.update((k,v) for (k,v) in package['extras'].items()\
if k not in self._unexpected_fields)
# Some of the data isn't in the format as rendered on the form, so
# overwrite it by hand for now.
expected_field_values['geographic_coverage'] = 'england'
date_fields = ('date_update_future',
'date_released',
'date_updated',
'temporal_coverage-from',
'temporal_coverage-to',
)
for field_name in date_fields:
expected_field_values[field_name] = DateType.db_to_form(expected_field_values[field_name])
# TODO: fix these fields
del expected_field_values['published_by']
del expected_field_values['published_via']
# Ensure the resources have been un-merged correctly.
for resource_type in 'additional timeseries individual'.split():
resource_type += '_resources'
fields = []
for field_name in [f for f in self._expected_fields if f.startswith(resource_type)]:
fields.append(field_name.split('__')[-1])
resources = getattr(ckanext.dgu.lib.helpers, resource_type)(package)
for index, resource in enumerate(resources):
for field in fields:
# eg. additional_resources__0__url
full_field_name = '__'.join([resource_type,
str(index),
field])
try:
expected_field_values[full_field_name] = resource[field]
except KeyError:
expected_field_values[full_field_name] = resource['extras'][field]
for field_name, expected_value in expected_field_values.items():
if field_name not in self._deprecated_fields or expected_value:
self.check_named_element(response.body,
'(input|textarea|select)',
'name="%s"' % field_name,
expected_value)
try:
title = row[0].value.encode('utf-8')
except Exception, e:
raise Exception('Error with encoding of Title: %s' % e)
try:
description = row[1].value.encode('utf-8')
except Exception, e:
raise Exception('Error with encoding of Description: %s' % e)
publisher_name = row[2].value
publish_date = row[3].value
release_notes = row[4].value
if isinstance(publish_date, basestring) and publish_date.strip():
# e.g. CSV containing "1/2/14" -> "14/02/01"
try:
publish_date = DateType.form_to_db(publish_date)
except DateConvertError:
# Lots of text went into this field but have decided to not allow from now
# and it never gets displayed.
msg = 'Could not parse date: "%s" Must be: DD/MM/YY' % publish_date
log.error(msg)
raise Exception(msg)
if isinstance(publish_date, datetime.datetime):
# e.g. Excel -> "14/02/01"
publish_date = DateType.date_to_db(publish_date)
if not isinstance(publish_date, basestring):
# e.g. Excel int
msg = 'Could not parse date: "%s" Must be: DD/MM/YY' % publish_date
log.error(msg)
raise Exception(msg)
def test_a_full_timeseries_dataset(self):
"""
Tests the submission of a fully-completed timeseries dataset.
"""
package_data = _EXAMPLE_TIMESERIES_DATA
package_name = package_data["name"]
CreateTestData.flag_for_deletion(package_name)
assert not self.get_package_by_name(package_name), 'Package "%s" already exists' % package_name
# create package via form
response = self._form_client.post_form(package_data)
# ensure it's correct
pkg = self.get_package_by_name(package_name)
assert pkg, response.body
assert_equal(package_data["title"], pkg.title)
assert_equal(package_data["notes"], pkg.notes)
publisher = pkg.as_dict()["groups"][0]
assert_equal(package_data["groups__0__name"], publisher)
# Extra data
# Timeseries data
expected_timeseries_keys = filter(lambda k: k.startswith("timeseries_resources"), package_data.keys())
timeseries_resources = ckanext.dgu.lib.helpers.timeseries_resources(pkg.as_dict())
assert_equal(len(timeseries_resources), 4)
for key in expected_timeseries_keys:
index, field = key.split("__")[1:]
index = int(index)
assert_equal(package_data[key], timeseries_resources[index][field])
# Publisher / contact details
# The contact-email should not be an extra-field on the dataset as it's the
# same as the publisher group's contact-email. ie - it hasn't been overridden.
# The resof the information should be in the extras fields
assert_not_in("contact-email", pkg.extras)
assert_equal(package_data["contact-name"], pkg.extras["contact-name"])
assert_equal(package_data["contact-phone"], pkg.extras["contact-phone"])
assert_equal(package_data["foi-name"], pkg.extras["foi-name"])
assert_equal(package_data["foi-email"], pkg.extras["foi-email"])
assert_equal(package_data["foi-phone"], pkg.extras["foi-phone"])
# Themes and tags
assert_equal(package_data["theme-primary"], pkg.extras["theme-primary"])
assert_equal(set(package_data["theme-secondary"]), set(pkg.extras["theme-secondary"]))
# Health and Education are from the primary and secondary themes, which
# end up in the tags
assert_equal(
set(["tag1", "tag2", "a multi word tag", "Health", "Education"]), set(tag.name for tag in pkg.get_tags())
)
# Additional resources
expected_additional_keys = filter(lambda k: k.startswith("additional_resources"), package_data.keys())
additional_resources = ckanext.dgu.lib.helpers.additional_resources(pkg.as_dict())
assert_equal(len(additional_resources), 2)
for key in expected_additional_keys:
index, field = key.split("__")[1:]
index = int(index)
assert_equal(package_data[key], additional_resources[index][field])
assert_equal(package_data["mandate"], pkg.extras["mandate"])
assert_equal(package_data["access_constraints"], pkg.license_id)
assert_equal(package_data["temporal_coverage-from"], DateType.db_to_form(pkg.extras["temporal_coverage-from"]))
assert_equal(package_data["temporal_coverage-to"], DateType.db_to_form(pkg.extras["temporal_coverage-to"]))
assert_in("England", pkg.extras["geographic_coverage"])
def test_edit_form_form_has_all_fields(self):
"""
Asserts that edit-form of a package has the fields prefilled correctly.
"""
package = self.fixtures.pkgs[0]
offset = url_for(controller="package", action="edit", id=package["name"])
response = self.app.get(offset, extra_environ={"REMOTE_USER": self.admin})
# form field name -> expected form field value
expected_field_values = {}
# populate expected_field_values with the simple fields first
for field_name in self._expected_fields:
try:
expected_value = package[field_name]
if isinstance(expected_value, basestring):
expected_field_values[field_name] = expected_value
except KeyError:
pass
# populate expected_field_values for tag_string and license_id
# by hand, as the field names in the package dict don't follow the
# same naming scheme as the form fields.
expected_field_values["license_id"] = package["license"]
# tags may be re-ordered, so test them manually
expected_tags = set(map(lambda s: s.strip(), package["tags"].split()))
tag_string_form_value = (
re.finditer(r'<input [^>]*id="tag_string" name="tag_string" [^>]*value="([^"]+)" />', response.body)
.next()
.group(1)
)
actual_tags = set(map(lambda s: s.strip(), tag_string_form_value.split(",")))
assert_equal(expected_tags, actual_tags)
# Promote the key-value pairs stored in 'extras' to form fields.
expected_field_values.update((k, v) for (k, v) in package["extras"].items() if k not in self._unexpected_fields)
# Some of the data isn't in the format as rendered on the form, so
# overwrite it by hand for now.
expected_field_values["geographic_coverage"] = "england"
date_fields = (
"date_update_future",
"date_released",
"date_updated",
"temporal_coverage-from",
"temporal_coverage-to",
)
for field_name in date_fields:
expected_field_values[field_name] = DateType.db_to_form(expected_field_values[field_name])
# TODO: fix these fields
del expected_field_values["published_by"]
del expected_field_values["published_via"]
# Ensure the resources have been un-merged correctly.
for resource_type in "additional timeseries individual".split():
resource_type += "_resources"
fields = []
for field_name in [f for f in self._expected_fields if f.startswith(resource_type)]:
fields.append(field_name.split("__")[-1])
resources = getattr(ckanext.dgu.lib.helpers, resource_type)(package)
for index, resource in enumerate(resources):
for field in fields:
# eg. additional_resources__0__url
full_field_name = "__".join([resource_type, str(index), field])
try:
expected_field_values[full_field_name] = resource[field]
except KeyError:
expected_field_values[full_field_name] = resource["extras"][field]
for field_name, expected_value in expected_field_values.items():
if field_name not in self._deprecated_fields or expected_value:
self.check_named_element(
response.body, "(input|textarea|select)", 'name="%s"' % field_name, expected_value
)
