def _annotations_post(req, mapid):
mapobj = resolve_object(req, Map, {'id':mapid}, permission='base.change_resourcebase')
# default action
action = 'upsert'
# default for json to unpack properties for each 'row'
get_props = lambda r: r['properties']
# operation to run on completion
finish = lambda: None
# track created annotations
created = []
# csv or client to account for differences
form_mode = 'client'
content_type = None
overwrite = False
error_format = None
def id_collector(form):
created.append(form.instance.id)
if not req.FILES:
# json body
data = json.loads(req.body)
if isinstance(data, dict):
action = data.get('action', action)
if 'features' in data:
data = data.get('features')
else:
fp = iter(req.FILES.values()).next()
# ugh, builtin csv reader chokes on unicode
data = unicode_csv_dict_reader(fp)
id_collector = lambda f: None
form_mode = 'csv'
content_type = 'text/html'
get_props = lambda r: r
ids = list(Annotation.objects.filter(map=mapobj).values_list('id', flat=True))
# delete existing, we overwrite
finish = lambda: Annotation.objects.filter(id__in=ids).delete()
overwrite = True
def error_format(row_errors):
response = []
for re in row_errors:
row = re[0] + 1
for e in re[1]:
response.append('[%s] %s : %s' % (row, e, re[1][e]))
return 'The following rows had problems:<ul><li>' + '</li><li>'.join(response) + "</li></ul>"
if action == 'delete':
Annotation.objects.filter(pk__in=data['ids'], map=mapobj).delete()
return json_response({'success': True})
if action != 'upsert':
return HttpResponse('%s not supported' % action, status=400)
errors = _write_annotations(data, get_props, id_collector, mapobj, overwrite, form_mode)
if errors:
transaction.rollback()
body = None
if error_format:
return HttpResponse(error_format(errors), status=400)
else:
finish()
transaction.commit()
body = {'success': True}
if created:
body['ids'] = created
return json_response(body=body, errors=errors, content_type=content_type)
def test_csv_upload(self):
'''test csv upload with update and insert'''
#@todo cleanup and break out into simpler cases
self.make_annotations(self.dummy, 2)
header = u"id,title,content,lat,lon,start_time,end_time,appearance\n"
# first row is insert, second update (as it has an id)
fp = tempfile.NamedTemporaryFile(delete=True)
fp.write((header + u'"",foo bar,blah,5,10,2001/01/01,2005\n'
u"1,bar foo,halb,10,20,2010-01-01,,\n"
u"2,bunk,\u201c,20,30,,,").encode('utf-8'))
fp.seek(0)
# verify failure before login
resp = self.c.post(reverse('annotations', args=[self.dummy.id]),
{'csv': fp})
self.assertLoginRedirect(resp)
# still only 2 annotations
self.assertEqual(2,
Annotation.objects.filter(map=self.dummy.id).count())
# login, rewind the buffer and verify
self.c.login(username='******', password='******')
fp.seek(0)
resp = self.c.post(reverse('annotations', args=[self.dummy.id]),
{'csv': fp})
# response type must be text/html for ext fileupload
self.assertEqual('text/html', resp['content-type'])
jsresp = json.loads(resp.content)
self.assertEqual(True, jsresp['success'])
ann = Annotation.objects.filter(map=self.dummy.id)
# we uploaded 3, the other 2 should be deleted (overwrite mode)
self.assertEqual(3, ann.count())
ann = Annotation.objects.get(title='bar foo')
get_x = lambda ann: int(json.loads(ann.the_geom)['coordinates'][0])
self.assertEqual(get_x(ann), 20.)
ann = Annotation.objects.get(title='bunk')
self.assertTrue(u'\u201c', ann.content)
ann = Annotation.objects.get(title='foo bar')
self.assertEqual('foo bar', ann.title)
self.assertEqual(get_x(ann), 10.)
resp = self.c.get(
reverse('annotations', args=[self.dummy.id]) + "?csv")
# the dict reader won't fill out keys for the missing entries
# verify each row has 7 fields
for l in resp.content.split('\r\n'):
if l.strip():
self.assertEqual(7, len(l.split(',')))
x = list(unicode_csv_dict_reader(resp.content))
self.assertEqual(3, len(x))
by_title = dict([(v['title'], v) for v in x])
# verify round trip of unicode quote
self.assertEqual(u'\u201c', by_title['bunk']['content'])
# and times
self.assertEqual('2010-01-01T00:00:00',
by_title['bar foo']['start_time'])
self.assertEqual('2001-01-01T00:00:00',
by_title['foo bar']['start_time'])
self.assertEqual('2005-01-01T00:00:00',
by_title['foo bar']['end_time'])
# verify windows codepage quotes
fp = tempfile.NamedTemporaryFile(delete=True)
fp.write((str(header) + ',\x93windows quotes\x94,yay,,,,'))
fp.seek(0)
resp = self.c.post(reverse('annotations', args=[self.dummy.id]),
{'csv': fp})
ann = Annotation.objects.get(map=self.dummy.id)
# windows quotes are unicode now
self.assertEqual(u'\u201cwindows quotes\u201d', ann.title)
# make sure a bad upload aborts the transaction (and prevents dropping existing)
fp = tempfile.NamedTemporaryFile(delete=True)
fp.write((str(header) * 2))
fp.seek(0)
resp = self.c.post(reverse('annotations', args=[self.dummy.id]),
{'csv': fp})
self.assertEqual(400, resp.status_code)
# there should only be one that we uploaded before
Annotation.objects.get(map=self.dummy.id)
self.assertEqual('yay', ann.content)
# and check for the errors related to the invalid data we sent
expected = [
'[1] lat : Invalid value for lat : "lat"',
'[1] start_time : Unable to read as date : start_time, please format as yyyy-mm-dd',
'[1] lon : Invalid value for lon : "lon"',
'[1] end_time : Unable to read as date : end_time, please format as yyyy-mm-dd'
]
self.assertEqual(expected, re.findall('<li>([^<]*)</li>',
resp.content))
def test_csv_upload(self):
'''test csv upload with update and insert'''
#@todo cleanup and break out into simpler cases
self.make_annotations(self.dummy, 2)
header = u"id,title,content,lat,lon,start_time,end_time,appearance\n"
# first row is insert, second update (as it has an id)
fp = tempfile.NamedTemporaryFile(delete=True)
fp.write((
header +
u'"",foo bar,blah,5,10,2001/01/01,2005\n'
u"1,bar foo,halb,10,20,2010-01-01,,\n"
u"2,bunk,\u201c,20,30,,,"
).encode('utf-8'))
fp.seek(0)
# verify failure before login
resp = self.c.post(reverse('annotations',args=[self.dummy.id]),{'csv':fp})
self.assertLoginRedirect(resp)
# still only 2 annotations
self.assertEqual(2, Annotation.objects.filter(map=self.dummy.id).count())
# login, rewind the buffer and verify
self.c.login(username='******',password='******')
fp.seek(0)
resp = self.c.post(reverse('annotations',args=[self.dummy.id]),{'csv':fp})
# response type must be text/html for ext fileupload
self.assertEqual('text/html', resp['content-type'])
jsresp = json.loads(resp.content)
self.assertEqual(True, jsresp['success'])
ann = Annotation.objects.filter(map=self.dummy.id)
# we uploaded 3, the other 2 should be deleted (overwrite mode)
self.assertEqual(3, ann.count())
ann = Annotation.objects.get(title='bar foo')
get_x = lambda ann: int(json.loads(ann.the_geom)['coordinates'][0])
self.assertEqual(get_x(ann), 20.)
ann = Annotation.objects.get(title='bunk')
self.assertTrue(u'\u201c', ann.content)
ann = Annotation.objects.get(title='foo bar')
self.assertEqual('foo bar', ann.title)
self.assertEqual(get_x(ann), 10.)
resp = self.c.get(reverse('annotations',args=[self.dummy.id]) + "?csv")
# the dict reader won't fill out keys for the missing entries
# verify each row has 7 fields
for l in resp.content.split('\r\n'):
if l.strip():
self.assertEqual(7, len(l.split(',')))
x = list(unicode_csv_dict_reader(resp.content))
self.assertEqual(3, len(x))
by_title = dict( [(v['title'],v) for v in x] )
# verify round trip of unicode quote
self.assertEqual(u'\u201c', by_title['bunk']['content'])
# and times
self.assertEqual('2010-01-01T00:00:00', by_title['bar foo']['start_time'])
self.assertEqual('2001-01-01T00:00:00', by_title['foo bar']['start_time'])
self.assertEqual('2005-01-01T00:00:00', by_title['foo bar']['end_time'])
# verify windows codepage quotes
fp = tempfile.NamedTemporaryFile(delete=True)
fp.write((
str(header) +
',\x93windows quotes\x94,yay,,,,'
))
fp.seek(0)
resp = self.c.post(reverse('annotations',args=[self.dummy.id]),{'csv':fp})
ann = Annotation.objects.get(map=self.dummy.id)
# windows quotes are unicode now
self.assertEqual(u'\u201cwindows quotes\u201d', ann.title)
# make sure a bad upload aborts the transaction (and prevents dropping existing)
fp = tempfile.NamedTemporaryFile(delete=True)
fp.write((
str(header) * 2
))
fp.seek(0)
resp = self.c.post(reverse('annotations',args=[self.dummy.id]),{'csv':fp})
self.assertEqual(400, resp.status_code)
# there should only be one that we uploaded before
Annotation.objects.get(map=self.dummy.id)
self.assertEqual('yay', ann.content)
# and check for the errors related to the invalid data we sent
expected = ['[1] lat : Invalid value for lat : "lat"',
'[1] start_time : Unable to read as date : start_time, please format as yyyy-mm-dd',
'[1] lon : Invalid value for lon : "lon"',
'[1] end_time : Unable to read as date : end_time, please format as yyyy-mm-dd']
self.assertEqual(expected, re.findall('<li>([^<]*)</li>', resp.content))