def post(self):
auth_key = self.request.get('key')
auth = model.Authorization.all().filter('auth_key =', auth_key).get()
if auth:
source_domain = auth.domain
try:
person_records, note_records = pfif.parse_file(self.request.body_file)
except Exception, e:
self.response.set_status(400)
self.write('Invalid XML: %s\n' % e)
return
self.response.headers['Content-Type'] = 'application/xml'
self.write('<?xml version="1.0"?>\n')
self.write('<status:status>\n')
written, skipped, total = importer.import_records(
source_domain, importer.create_person, person_records)
self.write_status('person', written, skipped, total, 'person_record_id')
written, skipped, total = importer.import_records(
source_domain, importer.create_note, note_records)
self.write_status('note', written, skipped, total, 'note_record_id')
self.write('</status:status>\n')
class Write(utils.Handler):
https_required = True
def post(self):
if not (self.auth and self.auth.domain_write_permission):
self.response.set_status(403)
self.write('Missing or invalid authorization key\n')
return
source_domain = self.auth.domain_write_permission
try:
person_records, note_records = \
pfif.parse_file(self.request.body_file)
except Exception, e:
self.response.set_status(400)
self.write('Invalid XML: %s\n' % e)
return
self.response.headers['Content-Type'] = 'application/xml'
self.write('<?xml version="1.0"?>\n')
self.write('<status:status>\n')
create_person = importer.create_person
written, skipped, total = importer.import_records(
self.subdomain, source_domain, create_person, person_records)
self.write_status('person', written, skipped, total,
'person_record_id')
create_note = importer.create_note
written, skipped, total = importer.import_records(
self.subdomain, source_domain, create_note, note_records)
self.write_status('note', written, skipped, total, 'note_record_id')
self.write('</status:status>\n')
def post(self):
if not (self.auth and self.auth.domain_write_permission):
self.response.set_status(403)
# TODO(ryok): i18n
self.write('Missing or invalid authorization key.')
return
content = self.request.get('content')
if not content:
self.response.set_status(400)
self.write('You need to specify at least one CSV file.')
return
# TODO(ryok): let the user select timezone.
# TODO(ryok): accept more flexible date time format.
# TODO(ryok): support non-UTF8 encodings.
source_domain = self.auth.domain_write_permission
records = importer.utf8_decoder(
csv.DictReader(StringIO.StringIO(content)))
try:
records = [complete_record_ids(r, source_domain) for r in records]
except csv.Error:
self.response.set_status(400)
self.write('The CSV file is formatted incorrectly.')
return
is_empty = lambda x: (x or '').strip()
persons = [r for r in records if is_empty(r.get('full_name'))]
notes = [r for r in records if is_empty(r.get('note_record_id'))]
people_written, people_skipped, people_total = importer.import_records(
self.repo, source_domain, importer.create_person, persons)
notes_written, notes_skipped, notes_total = importer.import_records(
self.repo, source_domain, importer.create_note, notes)
utils.log_api_action(self, ApiActionLog.WRITE,
people_written, notes_written,
len(people_skipped), len(notes_skipped))
self.render('import.html',
stats=[
Struct(type='Person',
written=people_written,
skipped=people_skipped,
total=people_total),
Struct(type='Note',
written=notes_written,
skipped=notes_skipped,
total=notes_total)],
**get_tag_params(self))
class Write(utils.BaseHandler):
https_required = True
def post(self):
if not (self.auth and self.auth.domain_write_permission):
self.info(
403,
message='Missing or invalid authorization key',
style='plain')
return
source_domain = self.auth.domain_write_permission
try:
person_records, note_records = \
pfif.parse_file(self.request.body_file)
except Exception, e:
self.info(400, message='Invalid XML: %s' % e, style='plain')
return
mark_notes_reviewed = bool(self.auth.mark_notes_reviewed)
believed_dead_permission = bool(
self.auth.believed_dead_permission)
self.response.headers['Content-Type'] = 'application/xml; charset=utf-8'
self.write('<?xml version="1.0"?>\n')
self.write('<status:status ' +
'xmlns:status="http://zesty.ca/pfif/1.4/status" ' +
'xmlns:pfif="http://zesty.ca/pfif/1.4">\n')
create_person = importer.create_person
num_people_written, people_skipped, total = importer.import_records(
self.repo, source_domain, create_person, person_records)
self.write_status(
'person', num_people_written, people_skipped, total,
'person_record_id')
create_note = importer.create_note
num_notes_written, notes_skipped, total = importer.import_records(
self.repo, source_domain, create_note, note_records,
mark_notes_reviewed, believed_dead_permission, self)
self.write_status(
'note', num_notes_written, notes_skipped, total, 'note_record_id')
self.write('</status:status>\n')
utils.log_api_action(self, ApiActionLog.WRITE,
num_people_written, num_notes_written,
len(people_skipped), len(notes_skipped))
def test_import_reviewed_note_records(self):
# Prepare person records which the notes will be added to.
for i in range(3):
put_dummy_person_record('haiti', 'test_domain/person_%d' % i)
records = []
for i in range(3):
source_date = '2010-01-01T01:23:45Z'
note_id = 'test_domain/record_%d' % i
person_id = 'test_domain/person_%d' % i
records.append({'person_record_id': person_id,
'note_record_id': note_id,
'source_date': source_date})
# Import reviewed notes.
written, skipped, total = importer.import_records(
'haiti', 'test_domain', importer.create_note, records, True,
True, None)
assert written == 3
assert len(skipped) == 0
assert total == 3
# Confirm all records are marked reviewed.
for note in model.Note.all():
assert note.reviewed == True
def test_import_reviewed_note_records(self):
# Prepare person records which the notes will be added to.
for i in range(3):
put_dummy_person_record('haiti', 'test_domain/person_%d' % i)
records = []
for i in range(3):
source_date = '2010-01-01T01:23:45Z'
note_id = 'test_domain/record_%d' % i
person_id = 'test_domain/person_%d' % i
records.append({
'person_record_id': person_id,
'note_record_id': note_id,
'source_date': source_date
})
# Import reviewed notes.
written, skipped, total = importer.import_records(
'haiti', 'test_domain', importer.create_note, records, True, True,
None)
assert written == 3
assert len(skipped) == 0
assert total == 3
# Confirm all records are marked reviewed.
for note in model.Note.all():
assert note.reviewed == True
def test_import_person_records(self):
records = []
for i in range(20):
given_name = "given_name_%d" % i
family_name = "family_name_%d" % i
source_date = "2010-01-01T01:23:45Z"
record_id = "test_domain/%d" % i
# Records 0, 8, and 16 have bad domains.
if not i % 8:
record_id = "other_domain/%d" % i
# Records 0, 9, and 18 have invalid dates.
elif not i % 9:
source_date = "2010-01-01 01:23:45"
records.append({
'given_name': given_name,
'family_name': family_name,
'person_record_id': record_id,
'source_date': source_date
})
written, skipped, total = importer.import_records(
'haiti', 'test_domain', importer.create_person, records, False,
True, None)
assert written == 15
assert len(skipped) == 5
assert skipped[0] == ('Not in authorized domain: u\'other_domain/0\'',
{
'given_name': 'given_name_0',
'family_name': 'family_name_0',
'source_date': '2010-01-01T01:23:45Z',
'person_record_id': 'other_domain/0'
})
assert skipped[3] == ('Not in authorized domain: u\'other_domain/16\'',
{
'given_name': 'given_name_16',
'family_name': 'family_name_16',
'source_date': '2010-01-01T01:23:45Z',
'person_record_id': 'other_domain/16'
})
assert skipped[2] == (
'ValueError: Bad datetime: \'2010-01-01 01:23:45\'', {
'given_name': 'given_name_9',
'family_name': 'family_name_9',
'source_date': '2010-01-01 01:23:45',
'person_record_id': 'test_domain/9'
})
assert skipped[4] == (
'ValueError: Bad datetime: \'2010-01-01 01:23:45\'', {
'given_name': 'given_name_18',
'family_name': 'family_name_18',
'source_date': '2010-01-01 01:23:45',
'person_record_id': 'test_domain/18'
})
assert total == 20
# Also confirm that 15 records were put into the datastore.
assert model.Person.all().count() == 15
def test_import_person_records(self):
records = []
for i in range(20):
given_name = "given_name_%d" % i
family_name = "family_name_%d" % i
source_date = "2010-01-01T01:23:45Z"
record_id = "test_domain/%d" % i
# Records 0, 8, and 16 have bad domains.
if not i % 8:
record_id = "other_domain/%d" % i
# Records 0, 9, and 18 have invalid dates.
elif not i % 9:
source_date = "2010-01-01 01:23:45"
records.append({'given_name': given_name,
'family_name': family_name,
'person_record_id': record_id,
'source_date': source_date})
written, skipped, total = importer.import_records(
'haiti', 'test_domain', importer.create_person, records, False,
True, None)
assert written == 15
assert len(skipped) == 5
assert skipped[0] == (
'Not in authorized domain: u\'other_domain/0\'', {
'given_name': 'given_name_0',
'family_name': 'family_name_0',
'source_date': '2010-01-01T01:23:45Z',
'person_record_id': 'other_domain/0'
})
assert skipped[3] == (
'Not in authorized domain: u\'other_domain/16\'', {
'given_name': 'given_name_16',
'family_name': 'family_name_16',
'source_date': '2010-01-01T01:23:45Z',
'person_record_id': 'other_domain/16'
})
assert skipped[2] == (
'ValueError: Bad datetime: \'2010-01-01 01:23:45\'', {
'given_name': 'given_name_9',
'family_name': 'family_name_9',
'source_date': '2010-01-01 01:23:45',
'person_record_id': 'test_domain/9'
})
assert skipped[4] == (
'ValueError: Bad datetime: \'2010-01-01 01:23:45\'', {
'given_name': 'given_name_18',
'family_name': 'family_name_18',
'source_date': '2010-01-01 01:23:45',
'person_record_id': 'test_domain/18'
})
assert total == 20
# Also confirm that 15 records were put into the datastore.
assert model.Person.all().count() == 15
def test_import_note_records(self):
records = []
for i in range(20):
source_date = '2010-01-01T01:23:45Z'
note_id = 'test_domain/record_%d' % i
person_id = 'test_domain/person_%d' % i
# Records 0, 8, and 16 have bad note record domains.
if not i % 8:
note_id = 'other_domain/record_%d' % i
# Records 0, 9, and 18 have bad person record domains.
# This should not matter for note records.
elif not i % 9:
person_id = 'other_domain/person_%d' % i
# Records 0, 5, 10, and 15 have invalid dates.
elif not i % 5:
source_date = '2010-01-01 01:23:45'
records.append({'person_record_id': person_id,
'note_record_id': note_id,
'source_date': source_date})
written, skipped, total = importer.import_records(
'haiti', 'test_domain', importer.create_note, records, False,
True, None)
assert written == 14
assert len(skipped) == 6
assert skipped[0] == (
'Not in authorized domain: u\'other_domain/record_0\'', {
'person_record_id': 'test_domain/person_0',
'source_date': '2010-01-01T01:23:45Z',
'note_record_id': 'other_domain/record_0'
})
assert skipped[2] == (
'Not in authorized domain: u\'other_domain/record_8\'', {
'person_record_id': 'test_domain/person_8',
'source_date': '2010-01-01T01:23:45Z',
'note_record_id': 'other_domain/record_8'
})
assert skipped[1] == (
'ValueError: Bad datetime: \'2010-01-01 01:23:45\'', {
'person_record_id': 'test_domain/person_5',
'source_date': '2010-01-01 01:23:45',
'note_record_id': 'test_domain/record_5'
})
assert skipped[4] == (
'ValueError: Bad datetime: \'2010-01-01 01:23:45\'', {
'person_record_id': 'test_domain/person_15',
'source_date': '2010-01-01 01:23:45',
'note_record_id': 'test_domain/record_15'
})
assert total == 20
# Also confirm that 14 records were put into the datastore.
assert model.Note.all().count() == 14
# Confirm all records are NOT marked reviewed.
for note in model.Note.all():
assert note.reviewed == False
def import_from_file(host, kind, converter, filename):
print '%s: importing %s records from %s' % (host, kind, filename)
written, skipped, total = importer.import_records(
source_domain, converter,
importer.utf8_decoder(csv.DictReader(open(filename))))
for error, record in skipped[:SHOW_ERRORS]:
print ' - %s: %r' % (error, record)
if len(skipped) > SHOW_ERRORS:
print ' (more errors not shown)'
print 'wrote %d of %d (skipped %d with errors)' % (
written, total, len(skipped))
def import_from_file(host, repo, kind, converter, filename):
print('%s: importing %s records from %s' % (host, kind, filename))
written, skipped, total = importer.import_records(
repo, source_domain, converter,
importer.utf8_decoder(csv.DictReader(open(filename))))
for error, record in skipped[:SHOW_ERRORS]:
print(' - %s: %r' % (error, record))
if len(skipped) > SHOW_ERRORS:
print(' (more errors not shown)')
print('wrote %d of %d (skipped %d with errors)' %
(written, total, len(skipped)))
def test_import_person_records(self):
records = []
for i in range(20):
first_name = "first_name_%d" % i
last_name = "last_name_%d" % i
source_date = "2010-01-01T01:23:45Z"
record_id = "test_domain/%d" % i
# Records 0, 8, and 16 have bad domains.
if not i % 8:
record_id = "other_domain/%d" % i
# Records 0, 9, and 18 have invalid dates.
elif not i % 9:
source_date = "2010-01-01 01:23:45"
records.append({"first_name": first_name,
"last_name": last_name,
"person_record_id": record_id,
"source_date": source_date})
written, skipped, total = importer.import_records(
'test_domain', importer.create_person, records)
self.assertEqual(written, 15)
self.assertEqual(len(skipped), 5)
self.assertEqual(skipped[0], ("Not in authorized domain: u'other_domain/0'",
{"first_name": "first_name_0",
"last_name": "last_name_0",
"source_date": "2010-01-01T01:23:45Z",
"person_record_id": "other_domain/0"}))
self.assertEqual(skipped[3],
("Not in authorized domain: u'other_domain/16'",
{"first_name": "first_name_16",
"last_name": "last_name_16",
"source_date": "2010-01-01T01:23:45Z",
"person_record_id": "other_domain/16"}))
self.assertEqual(skipped[2],
("ValueError: Bad datetime: '2010-01-01 01:23:45'",
{"first_name": "first_name_9",
"last_name": "last_name_9",
"source_date": "2010-01-01 01:23:45",
"person_record_id": "test_domain/9"}))
self.assertEqual(skipped[4],
("ValueError: Bad datetime: '2010-01-01 01:23:45'",
{"first_name": "first_name_18",
"last_name": "last_name_18",
"source_date": "2010-01-01 01:23:45",
"person_record_id": "test_domain/18"}))
self.assertEqual(total, 20)
# Also confirm that 15 records were put into the datastore.
self.assertEqual(model.Person.all().count(), 15)
def test_import_note_records(self):
records = []
for i in range(20):
source_date = "2010-01-01T01:23:45Z"
note_id = "test_domain/record_%d" % i
person_id = "test_domain/person_%d" % i
# Records 0, 8, and 16 have bad note record domains.
if not i % 8:
note_id = "other_domain/record_%d" % i
# Records 0, 9, and 18 have bad person record domains.
# This should not matter for note records.
elif not i % 9:
person_id = "other_domain/person_%d" % i
# Records 0, 5, 10, and 15 have invalid dates.
elif not i % 5:
source_date = "2010-01-01 01:23:45"
records.append({"person_record_id": person_id,
"note_record_id": note_id,
"source_date": source_date})
written, skipped, total = importer.import_records(
'test_domain', importer.create_note, records)
self.assertEqual(written, 14)
self.assertEqual(len(skipped), 6)
self.assertEqual(skipped[0],
("Not in authorized domain: u'other_domain/record_0'",
{"person_record_id": "test_domain/person_0",
"source_date": "2010-01-01T01:23:45Z",
"note_record_id": "other_domain/record_0"}))
self.assertEqual(skipped[2],
("Not in authorized domain: u'other_domain/record_8'",
{"person_record_id": "test_domain/person_8",
"source_date": "2010-01-01T01:23:45Z",
"note_record_id": "other_domain/record_8"}))
self.assertEqual(skipped[1],
("ValueError: Bad datetime: '2010-01-01 01:23:45'",
{"person_record_id": "test_domain/person_5",
"source_date": "2010-01-01 01:23:45",
"note_record_id": "test_domain/record_5"}))
self.assertEqual(skipped[4],
("ValueError: Bad datetime: '2010-01-01 01:23:45'",
{"person_record_id": "test_domain/person_15",
"source_date": "2010-01-01 01:23:45",
"note_record_id": "test_domain/record_15"}))
self.assertEqual(total, 20)
# Also confirm that 14 records were put into the datastore.
self.assertEqual(model.Note.all().count(), 14)
def test_import_note_records_for_non_existent_person(self):
records = [{
'person_record_id': 'test_domain/non_existent_person',
'note_record_id': 'test_domain/record_0',
'source_date': '2010-01-01T01:23:45Z',
}]
written, skipped, total = importer.import_records(
'haiti', 'test_domain', importer.create_note, records, False,
True, None)
assert written == 0
assert len(skipped) == 1
assert skipped[0] == (
"There is no person record with the person_record_id "
"'test_domain/non_existent_person'", {
'person_record_id': 'test_domain/non_existent_person',
'source_date': '2010-01-01T01:23:45Z',
'note_record_id': 'test_domain/record_0'
})
assert total == 1
assert model.Note.all().count() == 0
def test_import_note_records_for_non_existent_person(self):
records = [{
'person_record_id': 'test_domain/non_existent_person',
'note_record_id': 'test_domain/record_0',
'source_date': '2010-01-01T01:23:45Z',
}]
written, skipped, total = importer.import_records(
'haiti', 'test_domain', importer.create_note, records, False, True,
None)
assert written == 0
assert len(skipped) == 1
assert skipped[0] == (
"There is no person record with the person_record_id "
"'test_domain/non_existent_person'", {
'person_record_id': 'test_domain/non_existent_person',
'source_date': '2010-01-01T01:23:45Z',
'note_record_id': 'test_domain/record_0'
})
assert total == 1
assert model.Note.all().count() == 0
'smaller files (keeping the header rows in each file) and '
'uploading each part separately.')
def import_notes(self, lines):
source_domain = self.auth.domain_write_permission
records = importer.utf8_decoder(generate_note_record_ids(
convert_time_fields(csv.reader(lines))))
try:
records = [complete_record_ids(r, source_domain) for r in records]
except csv.Error, e:
self.error(400, message=
'The CSV file is formatted incorrectly. (%s)' % e)
return
notes_written, notes_skipped, notes_total = importer.import_records(
self.repo, source_domain, importer.create_note, records,
believed_dead_permission=self.auth.believed_dead_permission,
omit_duplicate_notes=True)
utils.log_api_action(self, ApiActionLog.WRITE,
0, notes_written, 0, len(notes_skipped))
self.render('import.html',
formats=get_requested_formats(self.env.path),
stats=[
Struct(type='Note',
written=notes_written,
skipped=notes_skipped,
total=notes_total)],
**get_tag_params(self))
def import_persons(self, lines):
def test_import_notes_disabled_note_records(self):
'''Check that notes will be rejected from API import when
notes_disabled is set to be True by the record author.'''
records = []
# First prepare and import two person records
for i in range(2):
given_name = "given_name_%d" % i
family_name = "family_name_%d" % i
source_date = "2010-01-01T01:23:45Z"
record_id = "test_domain/person_%d" % i
author_name = "test_author"
author_email = "test_email"
records.append({'given_name': given_name,
'family_name': family_name,
'person_record_id': record_id,
'source_date': source_date,
'author_name': author_name,
'author_email': author_email})
written, skipped, total = importer.import_records(
'haiti', 'test_domain', importer.create_person, records, False,
True, None)
assert written == 2
assert len(skipped) == 0
assert total == 2
assert model.Person.all().count() == 2
# Disable comments for first person record
person = model.Person.get('haiti', 'test_domain/person_0')
assert person
person.notes_disabled = True
db.put([person])
for person in model.Person.all():
if person.person_record_id == 'test_domain/person_0':
assert person.notes_disabled == True
# Import notes
records = []
for i in range(2):
source_date = '2010-01-01T01:23:45Z'
note_id = 'test_domain/record_%d' % i
person_id = 'test_domain/person_%d' % i
records.append({'person_record_id': person_id,
'note_record_id': note_id,
'source_date': source_date})
written, skipped, total = importer.import_records(
'haiti', 'test_domain', importer.create_note, records, False,
True, None)
# Check that the note associted with first person record is skipped.
assert written == 1
assert len(skipped) == 1
assert skipped[0] == (
'The author has disabled new commenting on this record',
{
'person_record_id': 'test_domain/person_0',
'source_date': '2010-01-01T01:23:45Z',
'note_record_id': 'test_domain/record_0',
})
assert total == 2
assert model.Note.all().count() == 1
def test_authorize_write_believed_dead_note_records(self):
# Prepare input data
records = []
for i in range(20):
source_date = '2010-01-01T01:23:45Z'
note_id = 'test_domain/record_%d' % i
person_id = 'test_domain/person_%d' % i
status = 'believed_missing'
# Records 0, 8, and 16 have status 'believed_dead'.
if not i % 8:
status = 'believed_dead'
records.append({'person_record_id': person_id,
'note_record_id': note_id,
'source_date': source_date,
'status': status})
# Disallow import notes with status 'believed_dead'.
written, skipped, total = importer.import_records(
'haiti', 'test_domain', importer.create_note, records, False,
False, None)
assert written == 17
assert len(skipped) == 3
assert skipped[0] == (
'Not authorized to post notes with the status'
' \"believed_dead\"', {
'person_record_id': 'test_domain/person_0',
'source_date': '2010-01-01T01:23:45Z',
'note_record_id': 'test_domain/record_0',
'status': 'believed_dead'
})
assert skipped[1] == (
'Not authorized to post notes with the status'
' \"believed_dead\"', {
'person_record_id': 'test_domain/person_8',
'source_date': '2010-01-01T01:23:45Z',
'note_record_id': 'test_domain/record_8',
'status': 'believed_dead'
})
assert skipped[2] == (
'Not authorized to post notes with the status'
' \"believed_dead\"', {
'person_record_id': 'test_domain/person_16',
'source_date': '2010-01-01T01:23:45Z',
'note_record_id': 'test_domain/record_16',
'status': 'believed_dead'
})
assert total == 20
assert model.Note.all().count() == 17
# Allow import notes with status 'believed_dead'.
model.Note.all().get().delete()
written, skipped, total = importer.import_records(
'haiti', 'test_domain', importer.create_note, records, False,
True, None)
assert written == 20
assert len(skipped) == 0
assert model.Note.all().count() == 20
for note in model.Note.all():
if note.person_record_id in ['test_domain/person_0',
'test_domain/person_8',
'test_domain/person_16']:
assert note.status == 'believed_dead'
else:
assert note.status == 'believed_missing'
