def datatable(self, resource_name, resource_id):
draw = int(request.params['draw'])
search_text = unicode(request.params['search[value]'])
offset = int(request.params['start'])
limit = int(request.params['length'])
sort_by_num = int(request.params['order[0][column]'])
sort_order = ('desc' if request.params['order[0][dir]'] == 'desc'
else 'asc'
)
chromo = h.recombinant_get_chromo(resource_name)
lc = LocalCKAN(username=c.user)
unfiltered_response = lc.action.datastore_search(
resource_id=resource_id,
limit=1,
)
cols = [f['datastore_id'] for f in chromo['fields']]
sort_str = cols[sort_by_num] + ' ' + sort_order
sort_str += ' NULLS LAST'
response = lc.action.datastore_search(
q=search_text,
resource_id=resource_id,
offset=offset,
limit=limit,
sort=sort_str
)
aadata = [
[datatablify(row.get(colname, u''), colname) for colname in cols]
for row in response['records']]
if chromo.get('edit_form', False):
res = lc.action.resource_show(id=resource_id)
pkg = lc.action.package_show(id=res['package_id'])
fids = [f['datastore_id'] for f in chromo['fields']]
pkids = [fids.index(k) for k in aslist(chromo['datastore_primary_key'])]
for row in aadata:
row.insert(0, (
u'<a href="{0}" aria-label"' + _("Edit") + '">'
u'<i class="fa fa-lg fa-edit" aria-hidden="true"></i></a>').format(
h.url_for(
controller='ckanext.canada.controller:PDUpdateController',
action='update_pd_record',
owner_org=pkg['organization']['name'],
resource_name=resource_name,
pk=','.join(url_part_escape(row[i]) for i in pkids)
)
)
)
return json.dumps({
'draw': draw,
'iTotalRecords': unfiltered_response.get('total', 0),
'iTotalDisplayRecords': response.get('total', 0),
'aaData': aadata,
})
def datatable(self, resource_name, resource_id):
draw = int(request.params['draw'])
search_text = unicode(request.params['search[value]'])
offset = int(request.params['start'])
limit = int(request.params['length'])
sort_by_num = int(request.params['order[0][column]'])
sort_order = ('desc' if request.params['order[0][dir]'] == 'desc'
else 'asc'
)
chromo = h.recombinant_get_chromo(resource_name)
lc = LocalCKAN(username=c.user)
unfiltered_response = lc.action.datastore_search(
resource_id=resource_id,
limit=1,
)
cols = [f['datastore_id'] for f in chromo['fields']]
sort_str = cols[sort_by_num] + ' ' + sort_order
sort_str += ' NULLS LAST'
response = lc.action.datastore_search(
q=search_text,
resource_id=resource_id,
offset=offset,
limit=limit,
sort=sort_str
)
aadata = [
[datatablify(row.get(colname, u''), colname) for colname in cols]
for row in response['records']]
# XXX custom business logic hack
if resource_name == 'consultations':
res = lc.action.resource_show(id=resource_id)
pkg = lc.action.package_show(id=res['package_id'])
for row in aadata:
row[0] = u'<a href="{0}">{1}</a>'.format(
h.url_for(
controller='ckanext.canada.controller:PDUpdateController',
action='update_pd_record',
owner_org=pkg['organization']['name'],
resource_name=resource_name,
pk=url_part_escape(row[0])),
row[0])
return json.dumps({
'draw': draw,
'iTotalRecords': unfiltered_response.get('total', 0),
'iTotalDisplayRecords': response.get('total', 0),
'aaData': aadata,
})
def get_snippet_datastore(activity, detail):
if activity['data'].get('pkg_type'):
org_name = activity['data']['owner_org']
resource_name = activity['data']['resource_name']
url = url_for(
resource_name=resource_name,
owner_org=org_name,
action='preview_table',
controller='ckanext.recombinant.controller:UploadController')
chromo = h.recombinant_get_chromo(resource_name)
return ''.join(['<a href="', url, '">', _(chromo['title']), '</a>'])
else:
return ''
def datatable(self, resource_name, resource_id):
draw = int(request.params['draw'])
search_text = unicode(request.params['search[value]'])
offset = int(request.params['start'])
limit = int(request.params['length'])
sort_by_num = int(request.params['order[0][column]'])
sort_order = ('desc'
if request.params['order[0][dir]'] == 'desc' else 'asc')
chromo = h.recombinant_get_chromo(resource_name)
lc = LocalCKAN(username=c.user)
unfiltered_response = lc.action.datastore_search(
resource_id=resource_id,
limit=1,
)
cols = [f['datastore_id'] for f in chromo['fields']]
sort_str = cols[sort_by_num] + ' ' + sort_order
sort_str += ' NULLS LAST'
response = lc.action.datastore_search(q=search_text,
resource_id=resource_id,
offset=offset,
limit=limit,
sort=sort_str)
aadata = [[
datatablify(row.get(colname, u''), colname) for colname in cols
] for row in response['records']]
# XXX custom business logic hack
if resource_name == 'consultations':
res = lc.action.resource_show(id=resource_id)
pkg = lc.action.package_show(id=res['package_id'])
for row in aadata:
row[0] = u'<a href="{0}">{1}</a>'.format(
h.url_for(controller=
'ckanext.canada.controller:PDUpdateController',
action='update_pd_record',
owner_org=pkg['organization']['name'],
resource_name=resource_name,
pk=url_part_escape(row[0])), row[0])
return json.dumps({
'draw': draw,
'iTotalRecords': unfiltered_response.get('total', 0),
'iTotalDisplayRecords': response.get('total', 0),
'aaData': aadata,
})
def datatable(self, resource_name, resource_id):
draw = int(request.params['draw'])
search_text = unicode(request.params['search[value]'])
offset = int(request.params['start'])
limit = int(request.params['length'])
sort_by_num = int(request.params['order[0][column]'])
sort_order = ('desc' if request.params['order[0][dir]'] == 'desc'
else 'asc'
)
chromo = h.recombinant_get_chromo(resource_name)
lc = LocalCKAN(username=c.user)
unfiltered_response = lc.action.datastore_search(
resource_id=resource_id,
limit=1,
)
cols = [f['datastore_id'] for f in chromo['fields']]
sort_str = cols[sort_by_num] + ' ' + sort_order
sort_str += ' NULLS LAST'
response = lc.action.datastore_search(
q=search_text,
resource_id=resource_id,
offset=offset,
limit=limit,
sort=sort_str
)
return json.dumps({
'draw': draw,
'iTotalRecords': unfiltered_response.get('total', 0),
'iTotalDisplayRecords': response.get('total', 0),
'aaData': [
[datatablify(row.get(colname, u''), colname) for colname in cols]
for row in response['records']
],
})
def datatable(self, resource_name, resource_id):
draw = int(request.params['draw'])
search_text = unicode(request.params['search[value]'])
offset = int(request.params['start'])
limit = int(request.params['length'])
sort_by_num = int(request.params['order[0][column]'])
sort_order = ('desc'
if request.params['order[0][dir]'] == 'desc' else 'asc')
chromo = h.recombinant_get_chromo(resource_name)
lc = LocalCKAN(username=c.user)
unfiltered_response = lc.action.datastore_search(
resource_id=resource_id,
limit=1,
)
cols = [f['datastore_id'] for f in chromo['fields']]
sort_str = cols[sort_by_num] + ' ' + sort_order
sort_str += ' NULLS LAST'
response = lc.action.datastore_search(q=search_text,
resource_id=resource_id,
offset=offset,
limit=limit,
sort=sort_str)
return json.dumps({
'draw':
draw,
'iTotalRecords':
unfiltered_response.get('total', 0),
'iTotalDisplayRecords':
response.get('total', 0),
'aaData':
[[datatablify(row.get(colname, u''), colname) for colname in cols]
for row in response['records']],
})
def update_pd_record(self, owner_org, resource_name, pk):
pk = [url_part_unescape(p) for p in pk.split(',')]
lc = LocalCKAN(username=c.user)
try:
chromo = h.recombinant_get_chromo(resource_name)
rcomb = lc.action.recombinant_show(
owner_org=owner_org, dataset_type=chromo['dataset_type'])
[res
] = [r for r in rcomb['resources'] if r['name'] == resource_name]
check_access('datastore_upsert', {
'user': c.user,
'auth_user_obj': c.userobj
}, {'resource_id': res['id']})
except NotAuthorized:
abort(403, _('Unauthorized'))
choice_fields = {
f['datastore_id']: [{
'value': k,
'label': v
} for (k, v) in f['choices']]
for f in h.recombinant_choice_fields(resource_name)
}
pk_fields = aslist(chromo['datastore_primary_key'])
pk_filter = dict(zip(pk_fields, pk))
records = lc.action.datastore_search(resource_id=res['id'],
filters=pk_filter)['records']
if len(records) == 0:
abort(404, _('Not found'))
if len(records) > 1:
abort(400, _('Multiple records found'))
record = records[0]
if request.method == 'POST':
post_data = parse_params(request.POST,
ignore_keys=['save'] + pk_fields)
if 'cancel' in post_data:
return redirect(
h.url_for(
controller=
'ckanext.recombinant.controller:UploadController',
action='preview_table',
resource_name=resource_name,
owner_org=rcomb['owner_org'],
))
data, err = clean_check_type_errors(post_data, chromo['fields'],
pk_fields, choice_fields)
# can't change pk fields
for f_id in data:
if f_id in pk_fields:
data[f_id] = record[f_id]
try:
lc.action.datastore_upsert(
resource_id=res['id'],
#method='update', FIXME not raising ValidationErrors
records=[{
k: None if k in err else v
for (k, v) in data.items()
}],
dry_run=bool(err))
except ValidationError as ve:
err = dict(
{
k: [_(e) for e in v]
for (k, v) in ve.error_dict['records'][0].items()
}, **err)
if err:
return render('recombinant/update_pd_record.html',
extra_vars={
'data': data,
'resource_name': resource_name,
'chromo_title': chromo['title'],
'choice_fields': choice_fields,
'pk_fields': pk_fields,
'owner_org': rcomb['owner_org'],
'errors': err,
})
h.flash_notice(_(u'Record %s Updated') % u','.join(pk))
return redirect(
h.url_for(
controller=
'ckanext.recombinant.controller:UploadController',
action='preview_table',
resource_name=resource_name,
owner_org=rcomb['owner_org'],
))
data = {}
for f in chromo['fields']:
if not f.get('import_template_include', True):
continue
val = record[f['datastore_id']]
data[f['datastore_id']] = val
return render('recombinant/update_pd_record.html',
extra_vars={
'data': data,
'resource_name': resource_name,
'chromo_title': chromo['title'],
'choice_fields': choice_fields,
'pk_fields': pk_fields,
'owner_org': rcomb['owner_org'],
'errors': {},
})
def create_pd_record(self, owner_org, resource_name):
lc = LocalCKAN(username=c.user)
try:
chromo = h.recombinant_get_chromo(resource_name)
rcomb = lc.action.recombinant_show(
owner_org=owner_org, dataset_type=chromo['dataset_type'])
[res
] = [r for r in rcomb['resources'] if r['name'] == resource_name]
check_access('datastore_upsert', {
'user': c.user,
'auth_user_obj': c.userobj
}, {'resource_id': res['id']})
except NotAuthorized:
return abort(403, _('Unauthorized'))
choice_fields = {
f['datastore_id']: [{
'value': k,
'label': v
} for (k, v) in f['choices']]
for f in h.recombinant_choice_fields(resource_name)
}
pk_fields = aslist(chromo['datastore_primary_key'])
if request.method == 'POST':
post_data = parse_params(request.POST, ignore_keys=['save'])
if 'cancel' in post_data:
return redirect(
h.url_for(
controller=
'ckanext.recombinant.controller:UploadController',
action='preview_table',
resource_name=resource_name,
owner_org=rcomb['owner_org'],
))
data, err = clean_check_type_errors(post_data, chromo['fields'],
pk_fields, choice_fields)
try:
lc.action.datastore_upsert(resource_id=res['id'],
method='insert',
records=[{
k: None if k in err else v
for (k, v) in data.items()
}],
dry_run=bool(err))
except ValidationError as ve:
if 'records' in ve.error_dict:
err = dict(
{
k: [_(e) for e in v]
for (k, v) in ve.error_dict['records'][0].items()
}, **err)
elif ve.error_dict.get('info', {}).get('pgcode',
'') == '23505':
err = dict(
{
k: [_("This record already exists")]
for k in pk_fields
}, **err)
if err:
return render('recombinant/create_pd_record.html',
extra_vars={
'data': data,
'resource_name': resource_name,
'chromo_title': chromo['title'],
'choice_fields': choice_fields,
'owner_org': rcomb['owner_org'],
'errors': err,
})
h.flash_notice(_(u'Record Created'))
return redirect(
h.url_for(
controller=
'ckanext.recombinant.controller:UploadController',
action='preview_table',
resource_name=resource_name,
owner_org=rcomb['owner_org'],
))
return render('recombinant/create_pd_record.html',
extra_vars={
'data': {},
'resource_name': resource_name,
'chromo_title': chromo['title'],
'choice_fields': choice_fields,
'owner_org': rcomb['owner_org'],
'errors': {},
})
def datatable(self, resource_name, resource_id):
draw = int(request.params['draw'])
search_text = unicode(request.params['search[value]'])
offset = int(request.params['start'])
limit = int(request.params['length'])
chromo = h.recombinant_get_chromo(resource_name)
lc = LocalCKAN(username=c.user)
try:
unfiltered_response = lc.action.datastore_search(
resource_id=resource_id,
limit=1,
)
except NotAuthorized:
# datatables js can't handle any sort of error response
# return no records instead
return json.dumps({
'draw': draw,
'iTotalRecords': -1, # with a hint that something is wrong
'iTotalDisplayRecords': -1,
'aaData': [],
})
cols = [f['datastore_id'] for f in chromo['fields']]
prefix_cols = 1 if chromo.get('edit_form', False) else 0
sort_list = []
i = 0
while True:
if u'order[%d][column]' % i not in request.params:
break
sort_by_num = int(request.params[u'order[%d][column]' % i])
sort_order = (u'desc NULLS LAST' if
request.params[u'order[%d][dir]' %
i] == u'desc' else u'asc NULLS LAST')
sort_list.append(cols[sort_by_num - prefix_cols] + u' ' +
sort_order)
i += 1
response = lc.action.datastore_search(
q=search_text,
resource_id=resource_id,
offset=offset,
limit=limit,
sort=u', '.join(sort_list),
)
aadata = [[
datatablify(row.get(colname, u''), colname) for colname in cols
] for row in response['records']]
if chromo.get('edit_form', False):
res = lc.action.resource_show(id=resource_id)
pkg = lc.action.package_show(id=res['package_id'])
fids = [f['datastore_id'] for f in chromo['fields']]
pkids = [
fids.index(k) for k in aslist(chromo['datastore_primary_key'])
]
for row in aadata:
row.insert(
0,
(u'<a href="{0}" aria-label"' + _("Edit") + '">'
u'<i class="fa fa-lg fa-edit" aria-hidden="true"></i></a>'
).format(
h.url_for(
controller=
'ckanext.canada.controller:PDUpdateController',
action='update_pd_record',
owner_org=pkg['organization']['name'],
resource_name=resource_name,
pk=','.join(
url_part_escape(row[i]) for i in pkids))))
return json.dumps({
'draw': draw,
'iTotalRecords': unfiltered_response.get('total', 0),
'iTotalDisplayRecords': response.get('total', 0),
'aaData': aadata,
})
def update_pd_record(self, owner_org, resource_name, pk):
pk = [url_part_unescape(p) for p in pk.split(',')]
lc = LocalCKAN(username=c.user)
try:
chromo = h.recombinant_get_chromo(resource_name)
rcomb = lc.action.recombinant_show(
owner_org=owner_org,
dataset_type=chromo['dataset_type'])
[res] = [r for r in rcomb['resources'] if r['name'] == resource_name]
check_access(
'datastore_upsert',
{'user': c.user, 'auth_user_obj': c.userobj},
{'resource_id': res['id']})
except NotAuthorized:
abort(403, _('Unauthorized'))
choice_fields = {
f['datastore_id']: [
{'value': k, 'label': v} for (k, v) in f['choices']]
for f in h.recombinant_choice_fields(resource_name)}
pk_fields = aslist(chromo['datastore_primary_key'])
pk_filter = dict(zip(pk_fields, pk))
records = lc.action.datastore_search(
resource_id=res['id'],
filters=pk_filter)['records']
if len(records) == 0:
abort(404, _('Not found'))
if len(records) > 1:
abort(400, _('Multiple records found'))
record = records[0]
if request.method == 'POST':
post_data = parse_params(request.POST, ignore_keys=['save'] + pk_fields)
if 'cancel' in post_data:
return redirect(h.url_for(
controller='ckanext.recombinant.controller:UploadController',
action='preview_table',
resource_name=resource_name,
owner_org=rcomb['owner_org'],
))
data = {}
for f in chromo['fields']:
f_id = f['datastore_id']
if not f.get('import_template_include', True):
continue
if f_id in pk_fields:
data[f_id] = record[f_id]
else:
val = post_data.get(f['datastore_id'], '')
if isinstance(val, list):
val = u','.join(val)
val = canonicalize(
val,
f['datastore_type'],
primary_key=False,
choice_field=f_id in choice_fields)
data[f['datastore_id']] = val
try:
lc.action.datastore_upsert(
resource_id=res['id'],
#method='update', FIXME not raising ValidationErrors
records=[data])
except ValidationError as ve:
err = {
k: [_(e) for e in v]
for (k, v) in ve.error_dict['records'][0].items()}
return render('recombinant/update_pd_record.html',
extra_vars={
'data': data,
'resource_name': resource_name,
'chromo_title': chromo['title'],
'choice_fields': choice_fields,
'pk_fields': pk_fields,
'owner_org': rcomb['owner_org'],
'errors': err,
})
h.flash_notice(_(u'Record %s Updated') % u','.join(pk) )
return redirect(h.url_for(
controller='ckanext.recombinant.controller:UploadController',
action='preview_table',
resource_name=resource_name,
owner_org=rcomb['owner_org'],
))
data = {}
for f in chromo['fields']:
if not f.get('import_template_include', True):
continue
val = record[f['datastore_id']]
data[f['datastore_id']] = val
return render('recombinant/update_pd_record.html',
extra_vars={
'data': data,
'resource_name': resource_name,
'chromo_title': chromo['title'],
'choice_fields': choice_fields,
'pk_fields': pk_fields,
'owner_org': rcomb['owner_org'],
'errors': {},
})
def create_travela(self, id, resource_id):
lc = LocalCKAN(username=c.user)
pkg = lc.action.package_show(id=id)
res = lc.action.resource_show(id=resource_id)
org = lc.action.organization_show(id=pkg['owner_org'])
dataset = lc.action.recombinant_show(
dataset_type='travela', owner_org=org['name'])
chromo = h.recombinant_get_chromo('travela')
data = {}
data_prev = {}
form_data = {}
for f in chromo['fields']:
dirty = request.params.getone(f['datastore_id'])
data[f['datastore_id']] = canonicalize(dirty, f['datastore_type'], False)
if f['datastore_id'] + '_prev' in request.params:
data_prev[f['datastore_id']] = request.params.getone(f['datastore_id'] + '_prev')
form_data[f['datastore_id'] + '_prev'] = data_prev[f['datastore_id']]
form_data.update(data)
def error(errors):
return render('recombinant/resource_edit.html',
extra_vars={
'create_errors': errors,
'create_data': form_data,
'delete_errors': [],
'dataset': dataset,
'resource': res,
'organization': org,
'filters': {},
'action': 'edit'})
try:
year = int_validator(data['year'], None)
except Invalid:
year = None
if not year:
return error({'year': [_(u'Invalid year')]})
response = lc.action.datastore_search(resource_id=resource_id,
filters={'year': data['year']})
if response['records']:
return error({'year': [_(u'Data for this year has already been entered')]})
response = lc.action.datastore_search(resource_id=resource_id,
filters={'year': year - 1})
if response['records']:
prev = response['records'][0]
errors = {}
for p in data_prev:
if prev[p] != data_prev[p] and prev[p]:
errors[p + '_prev'] = [_(u'Does not match previous data "%s"') % prev[p]]
if errors:
return error(errors)
else:
lc.action.datastore_upsert(resource_id=resource_id,
method='insert',
records=[dict(data_prev, year=year - 1)])
h.flash_success(_("Record for %d added.") % (year - 1))
lc.action.datastore_upsert(resource_id=resource_id,
method='insert',
records=[data])
h.flash_success(_("Record for %d added.") % year)
redirect(h.url_for(
controller='ckanext.recombinant.controller:UploadController',
action='preview_table',
resource_name=res['name'],
owner_org=org['name'],
))
def create_travela(self, id, resource_id):
lc = LocalCKAN(username=c.user)
pkg = lc.action.package_show(id=id)
res = lc.action.resource_show(id=resource_id)
org = lc.action.organization_show(id=pkg['owner_org'])
dataset = lc.action.recombinant_show(dataset_type='travela',
owner_org=org['name'])
chromo = h.recombinant_get_chromo('travela')
data = {}
data_prev = {}
form_data = {}
for f in chromo['fields']:
dirty = request.params.getone(f['datastore_id'])
data[f['datastore_id']] = canonicalize(dirty, f['datastore_type'],
False)
if f['datastore_id'] + '_prev' in request.params:
data_prev[f['datastore_id']] = request.params.getone(
f['datastore_id'] + '_prev')
form_data[f['datastore_id'] +
'_prev'] = data_prev[f['datastore_id']]
form_data.update(data)
def error(errors):
return render('recombinant/resource_edit.html',
extra_vars={
'create_errors': errors,
'create_data': form_data,
'delete_errors': [],
'dataset': dataset,
'resource': res,
'organization': org,
'filters': {},
'action': 'edit'
})
try:
year = int_validator(data['year'], None)
except Invalid:
year = None
if not year:
return error({'year': [_(u'Invalid year')]})
response = lc.action.datastore_search(resource_id=resource_id,
filters={'year': data['year']})
if response['records']:
return error(
{'year': [_(u'Data for this year has already been entered')]})
response = lc.action.datastore_search(resource_id=resource_id,
filters={'year': year - 1})
if response['records']:
prev = response['records'][0]
errors = {}
for p in data_prev:
if prev[p] != data_prev[p]:
errors[p + '_prev'] = [
_(u'Does not match previous data "%s"') % prev[p]
]
if errors:
return error(errors)
else:
lc.action.datastore_upsert(
resource_id=resource_id,
method='insert',
records=[dict(data_prev, year=year - 1)])
h.flash_success(_("Record for %d added.") % (year - 1))
lc.action.datastore_upsert(resource_id=resource_id,
method='insert',
records=[data])
h.flash_success(_("Record for %d added.") % year)
redirect(
h.url_for(
controller='ckanext.recombinant.controller:UploadController',
action='preview_table',
resource_name=res['name'],
owner_org=org['name'],
))
