def clean_check_type_errors(post_data, fields, pk_fields, choice_fields):
"""
clean posted data and check type errors, add type error messages
to errors dict returned. This is required because type errors on any
field prevent triggers from running so we don't get any other errors
from the datastore_upsert call.
:param post_data: form data
:param fields: recombinant fields
:param pk_fields: list of primary key field ids
:param choice_fields: {field id: choices, ...}
:return: cleaned data, errors
"""
data = {}
err = {}
for f in fields:
f_id = f['datastore_id']
if not f.get('import_template_include', True):
continue
else:
val = post_data.get(f['datastore_id'], '')
if isinstance(val, list):
val = u','.join(val)
val = canonicalize(
val,
f['datastore_type'],
primary_key=f['datastore_id'] in pk_fields,
choice_field=f_id in choice_fields)
if val:
if f['datastore_type'] in ('money', 'numeric'):
try:
decimal.Decimal(val)
except decimal.InvalidOperation:
err[f['datastore_id']] = [_(u'Number required')]
elif f['datastore_type'] == 'int':
try:
int(val)
except ValueError:
err[f['datastore_id']] = [_(u'Integer required')]
data[f['datastore_id']] = val
return data, err
def get_records(rows, fields, primary_key_fields, choice_fields):
"""
Truncate/pad empty/missing records to expected row length, canonicalize
cell content, and return resulting record list.
:param upload_data: generator producing rows of content
:type upload_data: generator
:param fields: collection of fields specified in JSON schema
:type fields: list or tuple
:param primary_key_fields: list of field ids making up the PK
:type primary_key_fields: list of strings
:param choice_fields: {field_id: 'full'/True/False}
:type choice_fields: dict
:return: canonicalized records of specified upload data
:rtype: tuple of dicts
"""
records = []
for n, row in rows:
# trailing cells might be empty: trim row to fit
while (row and
(len(row) > len(fields)) and
(row[-1] is None or row[-1] == '')):
row.pop()
while row and (len(row) < len(fields)):
row.append(None) # placeholder: canonicalize once only, below
try:
records.append(
(n, dict((
f['datastore_id'],
canonicalize(
v,
f['datastore_type'],
f['datastore_id'] in primary_key_fields,
choice_fields.get(f['datastore_id'], False)))
for f, v in zip(fields, row))))
except BadExcelData, e:
raise BadExcelData(u'Row {0}:'.format(n) + u' ' + e.message)
def update_pd_record(self, owner_org, resource_name, pk):
pk = [url_part_unescape(p) for p in pk.split(',')]
lc = LocalCKAN(username=c.user)
try:
chromo = h.recombinant_get_chromo(resource_name)
rcomb = lc.action.recombinant_show(
owner_org=owner_org,
dataset_type=chromo['dataset_type'])
[res] = [r for r in rcomb['resources'] if r['name'] == resource_name]
check_access(
'datastore_upsert',
{'user': c.user, 'auth_user_obj': c.userobj},
{'resource_id': res['id']})
except NotAuthorized:
abort(403, _('Unauthorized'))
choice_fields = {
f['datastore_id']: [
{'value': k, 'label': v} for (k, v) in f['choices']]
for f in h.recombinant_choice_fields(resource_name)}
pk_fields = aslist(chromo['datastore_primary_key'])
pk_filter = dict(zip(pk_fields, pk))
records = lc.action.datastore_search(
resource_id=res['id'],
filters=pk_filter)['records']
if len(records) == 0:
abort(404, _('Not found'))
if len(records) > 1:
abort(400, _('Multiple records found'))
record = records[0]
if request.method == 'POST':
post_data = parse_params(request.POST, ignore_keys=['save'] + pk_fields)
if 'cancel' in post_data:
return redirect(h.url_for(
controller='ckanext.recombinant.controller:UploadController',
action='preview_table',
resource_name=resource_name,
owner_org=rcomb['owner_org'],
))
data = {}
for f in chromo['fields']:
f_id = f['datastore_id']
if not f.get('import_template_include', True):
continue
if f_id in pk_fields:
data[f_id] = record[f_id]
else:
val = post_data.get(f['datastore_id'], '')
if isinstance(val, list):
val = u','.join(val)
val = canonicalize(
val,
f['datastore_type'],
primary_key=False,
choice_field=f_id in choice_fields)
data[f['datastore_id']] = val
try:
lc.action.datastore_upsert(
resource_id=res['id'],
#method='update', FIXME not raising ValidationErrors
records=[data])
except ValidationError as ve:
err = {
k: [_(e) for e in v]
for (k, v) in ve.error_dict['records'][0].items()}
return render('recombinant/update_pd_record.html',
extra_vars={
'data': data,
'resource_name': resource_name,
'chromo_title': chromo['title'],
'choice_fields': choice_fields,
'pk_fields': pk_fields,
'owner_org': rcomb['owner_org'],
'errors': err,
})
h.flash_notice(_(u'Record %s Updated') % u','.join(pk) )
return redirect(h.url_for(
controller='ckanext.recombinant.controller:UploadController',
action='preview_table',
resource_name=resource_name,
owner_org=rcomb['owner_org'],
))
data = {}
for f in chromo['fields']:
if not f.get('import_template_include', True):
continue
val = record[f['datastore_id']]
data[f['datastore_id']] = val
return render('recombinant/update_pd_record.html',
extra_vars={
'data': data,
'resource_name': resource_name,
'chromo_title': chromo['title'],
'choice_fields': choice_fields,
'pk_fields': pk_fields,
'owner_org': rcomb['owner_org'],
'errors': {},
})
def test_date():
dt = 'date'
assert_equal(canonicalize(2019, dt, False), '2019')
assert_equal(canonicalize(42.0, dt, False), '42.0')
assert_equal(canonicalize(42.25, dt, False), '42.25')
assert_equal(canonicalize(0, dt, False), '0')
assert_equal(canonicalize('2019', dt, False), '2019')
assert_equal(canonicalize('42.0', dt, False), '42.0')
assert_equal(canonicalize('42.25', dt, False), '42.25')
assert_equal(canonicalize('0', dt, False), '0')
assert_equal(canonicalize(None, dt, False), None)
assert_equal(canonicalize('', dt, False), None)
assert_equal(canonicalize('', dt, True), '')
assert_raises(BadExcelData, canonicalize, '=1+1', dt, False)
assert_equal(canonicalize(date(2020, 11, 15), dt, False), '2020-11-15')
assert_equal(canonicalize(datetime(2020, 11, 15), dt, False), '2020-11-15')
assert_equal(canonicalize('$1,000.50', dt, False), '$1,000.50')
assert_equal(canonicalize('=TRUE()', dt, False), 'TRUE')
assert_equal(canonicalize('=FALSE()', dt, False), 'FALSE')
assert_equal(canonicalize('AB,CD,E', dt, False), 'AB,CD,E')
def test_choice_field():
assert_equal(canonicalize('C1 ', 'text', False, False), 'C1 ')
assert_equal(canonicalize('C1 ', 'text', False, True), 'C1')
assert_equal(canonicalize(' C1: Value', 'text', False, False), ' C1: Value')
assert_equal(canonicalize(' C1: Value', 'text', False, True), 'C1: Value')
assert_equal(canonicalize(' C1: Value', 'text', False, 'full'), 'C1')
def test_primary_key():
dt = 'text'
assert_equal(canonicalize('OGP-324', dt, True), 'OGP-324')
assert_equal(canonicalize('\t OGP-324\n', dt, True), 'OGP-324')
assert_equal(canonicalize('OGP-\r\n\r\n324', dt, True), 'OGP-324')
assert_equal(canonicalize('OGP- 324', dt, True), 'OGP- 324')
def test_text_array():
dt = '_text'
assert_equal(canonicalize(2019, dt, False), ['2019'])
assert_equal(canonicalize(42.0, dt, False), ['42.0'])
assert_equal(canonicalize(42.25, dt, False), ['42.25'])
assert_equal(canonicalize(0, dt, False), ['0'])
assert_equal(canonicalize('2019', dt, False), ['2019'])
assert_equal(canonicalize('42.0', dt, False), ['42.0'])
assert_equal(canonicalize('42.25', dt, False), ['42.25'])
assert_equal(canonicalize('0', dt, False), ['0'])
assert_equal(canonicalize(None, dt, False), [])
assert_equal(canonicalize('', dt, False), [])
assert_equal(canonicalize('', dt, True), [])
assert_raises(BadExcelData, canonicalize, '=1+1', dt, False)
assert_equal(canonicalize(date(2020, 11, 15), dt, False), ['2020-11-15'])
assert_equal(canonicalize(datetime(2020, 11, 15), dt, False), ['2020-11-15 00:00:00'])
assert_equal(canonicalize('$1,000.50', dt, False), ['$1', '000.50'])
assert_equal(canonicalize('=TRUE()', dt, False), ['TRUE'])
assert_equal(canonicalize('=FALSE()', dt, False), ['FALSE'])
assert_equal(canonicalize('AB,CD,E', dt, False), ['AB', 'CD', 'E'])
def create_travela(self, id, resource_id):
lc = LocalCKAN(username=c.user)
pkg = lc.action.package_show(id=id)
res = lc.action.resource_show(id=resource_id)
org = lc.action.organization_show(id=pkg['owner_org'])
dataset = lc.action.recombinant_show(
dataset_type='travela', owner_org=org['name'])
chromo = h.recombinant_get_chromo('travela')
data = {}
data_prev = {}
form_data = {}
for f in chromo['fields']:
dirty = request.params.getone(f['datastore_id'])
data[f['datastore_id']] = canonicalize(dirty, f['datastore_type'], False)
if f['datastore_id'] + '_prev' in request.params:
data_prev[f['datastore_id']] = request.params.getone(f['datastore_id'] + '_prev')
form_data[f['datastore_id'] + '_prev'] = data_prev[f['datastore_id']]
form_data.update(data)
def error(errors):
return render('recombinant/resource_edit.html',
extra_vars={
'create_errors': errors,
'create_data': form_data,
'delete_errors': [],
'dataset': dataset,
'resource': res,
'organization': org,
'filters': {},
'action': 'edit'})
try:
year = int_validator(data['year'], None)
except Invalid:
year = None
if not year:
return error({'year': [_(u'Invalid year')]})
response = lc.action.datastore_search(resource_id=resource_id,
filters={'year': data['year']})
if response['records']:
return error({'year': [_(u'Data for this year has already been entered')]})
response = lc.action.datastore_search(resource_id=resource_id,
filters={'year': year - 1})
if response['records']:
prev = response['records'][0]
errors = {}
for p in data_prev:
if prev[p] != data_prev[p] and prev[p]:
errors[p + '_prev'] = [_(u'Does not match previous data "%s"') % prev[p]]
if errors:
return error(errors)
else:
lc.action.datastore_upsert(resource_id=resource_id,
method='insert',
records=[dict(data_prev, year=year - 1)])
h.flash_success(_("Record for %d added.") % (year - 1))
lc.action.datastore_upsert(resource_id=resource_id,
method='insert',
records=[data])
h.flash_success(_("Record for %d added.") % year)
redirect(h.url_for(
controller='ckanext.recombinant.controller:UploadController',
action='preview_table',
resource_name=res['name'],
owner_org=org['name'],
))
def create_travela(self, id, resource_id):
lc = LocalCKAN(username=c.user)
pkg = lc.action.package_show(id=id)
res = lc.action.resource_show(id=resource_id)
org = lc.action.organization_show(id=pkg['owner_org'])
dataset = lc.action.recombinant_show(dataset_type='travela',
owner_org=org['name'])
chromo = h.recombinant_get_chromo('travela')
data = {}
data_prev = {}
form_data = {}
for f in chromo['fields']:
dirty = request.params.getone(f['datastore_id'])
data[f['datastore_id']] = canonicalize(dirty, f['datastore_type'],
False)
if f['datastore_id'] + '_prev' in request.params:
data_prev[f['datastore_id']] = request.params.getone(
f['datastore_id'] + '_prev')
form_data[f['datastore_id'] +
'_prev'] = data_prev[f['datastore_id']]
form_data.update(data)
def error(errors):
return render('recombinant/resource_edit.html',
extra_vars={
'create_errors': errors,
'create_data': form_data,
'delete_errors': [],
'dataset': dataset,
'resource': res,
'organization': org,
'filters': {},
'action': 'edit'
})
try:
year = int_validator(data['year'], None)
except Invalid:
year = None
if not year:
return error({'year': [_(u'Invalid year')]})
response = lc.action.datastore_search(resource_id=resource_id,
filters={'year': data['year']})
if response['records']:
return error(
{'year': [_(u'Data for this year has already been entered')]})
response = lc.action.datastore_search(resource_id=resource_id,
filters={'year': year - 1})
if response['records']:
prev = response['records'][0]
errors = {}
for p in data_prev:
if prev[p] != data_prev[p]:
errors[p + '_prev'] = [
_(u'Does not match previous data "%s"') % prev[p]
]
if errors:
return error(errors)
else:
lc.action.datastore_upsert(
resource_id=resource_id,
method='insert',
records=[dict(data_prev, year=year - 1)])
h.flash_success(_("Record for %d added.") % (year - 1))
lc.action.datastore_upsert(resource_id=resource_id,
method='insert',
records=[data])
h.flash_success(_("Record for %d added.") % year)
redirect(
h.url_for(
controller='ckanext.recombinant.controller:UploadController',
action='preview_table',
resource_name=res['name'],
owner_org=org['name'],
))
def update_pd_record(self, owner_org, resource_name, pk):
pk = [url_part_unescape(p) for p in pk.split(',')]
lc = LocalCKAN(username=c.user)
try:
chromo = h.recombinant_get_chromo(resource_name)
rcomb = lc.action.recombinant_show(
owner_org=owner_org, dataset_type=chromo['dataset_type'])
[res
] = [r for r in rcomb['resources'] if r['name'] == resource_name]
check_access('datastore_upsert', {
'user': c.user,
'auth_user_obj': c.userobj
}, {'resource_id': res['id']})
except NotAuthorized:
abort(403, _('Unauthorized'))
choice_fields = {
f['datastore_id']: [{
'value': k,
'label': v
} for (k, v) in f['choices']]
for f in h.recombinant_choice_fields(resource_name)
}
pk_fields = aslist(chromo['datastore_primary_key'])
pk_filter = dict(zip(pk_fields, pk))
records = lc.action.datastore_search(resource_id=res['id'],
filters=pk_filter)['records']
if len(records) == 0:
abort(404, _('Not found'))
if len(records) > 1:
abort(400, _('Multiple records found'))
record = records[0]
if request.method == 'POST':
post_data = parse_params(request.POST,
ignore_keys=['save'] + pk_fields)
if 'cancel' in post_data:
return redirect(
h.url_for(
controller=
'ckanext.recombinant.controller:UploadController',
action='preview_table',
resource_name=resource_name,
owner_org=rcomb['owner_org'],
))
data = {}
for f in chromo['fields']:
f_id = f['datastore_id']
if not f.get('import_template_include', True):
continue
if f_id in pk_fields:
data[f_id] = record[f_id]
else:
val = post_data.get(f['datastore_id'], '')
if isinstance(val, list):
val = u','.join(val)
val = canonicalize(val,
f['datastore_type'],
primary_key=False,
choice_field=f_id in choice_fields)
data[f['datastore_id']] = val
try:
lc.action.datastore_upsert(
resource_id=res['id'],
#method='update', FIXME not raising ValidationErrors
records=[data])
except ValidationError as ve:
err = {
k: [_(e) for e in v]
for (k, v) in ve.error_dict['records'][0].items()
}
return render('recombinant/update_pd_record.html',
extra_vars={
'data': data,
'resource_name': resource_name,
'chromo_title': chromo['title'],
'choice_fields': choice_fields,
'pk_fields': pk_fields,
'owner_org': rcomb['owner_org'],
'errors': err,
})
h.flash_notice(_(u'Record %s Updated') % u','.join(pk))
return redirect(
h.url_for(
controller=
'ckanext.recombinant.controller:UploadController',
action='preview_table',
resource_name=resource_name,
owner_org=rcomb['owner_org'],
))
data = {}
for f in chromo['fields']:
if not f.get('import_template_include', True):
continue
val = record[f['datastore_id']]
data[f['datastore_id']] = val
return render('recombinant/update_pd_record.html',
extra_vars={
'data': data,
'resource_name': resource_name,
'chromo_title': chromo['title'],
'choice_fields': choice_fields,
'pk_fields': pk_fields,
'owner_org': rcomb['owner_org'],
'errors': {},
})