def post(self): auth_key = self.request.get('key') auth = model.Authorization.all().filter('auth_key =', auth_key).get() if auth: source_domain = auth.domain try: person_records, note_records = pfif.parse_file(self.request.body_file) except Exception, e: self.response.set_status(400) self.write('Invalid XML: %s\n' % e) return self.response.headers['Content-Type'] = 'application/xml' self.write('<?xml version="1.0"?>\n') self.write('<status:status>\n') written, skipped, total = importer.import_records( source_domain, importer.create_person, person_records) self.write_status('person', written, skipped, total, 'person_record_id') written, skipped, total = importer.import_records( source_domain, importer.create_note, note_records) self.write_status('note', written, skipped, total, 'note_record_id') self.write('</status:status>\n')
class Write(utils.Handler): https_required = True def post(self): if not (self.auth and self.auth.domain_write_permission): self.response.set_status(403) self.write('Missing or invalid authorization key\n') return source_domain = self.auth.domain_write_permission try: person_records, note_records = \ pfif.parse_file(self.request.body_file) except Exception, e: self.response.set_status(400) self.write('Invalid XML: %s\n' % e) return self.response.headers['Content-Type'] = 'application/xml' self.write('<?xml version="1.0"?>\n') self.write('<status:status>\n') create_person = importer.create_person written, skipped, total = importer.import_records( self.subdomain, source_domain, create_person, person_records) self.write_status('person', written, skipped, total, 'person_record_id') create_note = importer.create_note written, skipped, total = importer.import_records( self.subdomain, source_domain, create_note, note_records) self.write_status('note', written, skipped, total, 'note_record_id') self.write('</status:status>\n')
def post(self): if not (self.auth and self.auth.domain_write_permission): self.response.set_status(403) # TODO(ryok): i18n self.write('Missing or invalid authorization key.') return content = self.request.get('content') if not content: self.response.set_status(400) self.write('You need to specify at least one CSV file.') return # TODO(ryok): let the user select timezone. # TODO(ryok): accept more flexible date time format. # TODO(ryok): support non-UTF8 encodings. source_domain = self.auth.domain_write_permission records = importer.utf8_decoder( csv.DictReader(StringIO.StringIO(content))) try: records = [complete_record_ids(r, source_domain) for r in records] except csv.Error: self.response.set_status(400) self.write('The CSV file is formatted incorrectly.') return is_empty = lambda x: (x or '').strip() persons = [r for r in records if is_empty(r.get('full_name'))] notes = [r for r in records if is_empty(r.get('note_record_id'))] people_written, people_skipped, people_total = importer.import_records( self.repo, source_domain, importer.create_person, persons) notes_written, notes_skipped, notes_total = importer.import_records( self.repo, source_domain, importer.create_note, notes) utils.log_api_action(self, ApiActionLog.WRITE, people_written, notes_written, len(people_skipped), len(notes_skipped)) self.render('import.html', stats=[ Struct(type='Person', written=people_written, skipped=people_skipped, total=people_total), Struct(type='Note', written=notes_written, skipped=notes_skipped, total=notes_total)], **get_tag_params(self))
class Write(utils.BaseHandler): https_required = True def post(self): if not (self.auth and self.auth.domain_write_permission): self.info( 403, message='Missing or invalid authorization key', style='plain') return source_domain = self.auth.domain_write_permission try: person_records, note_records = \ pfif.parse_file(self.request.body_file) except Exception, e: self.info(400, message='Invalid XML: %s' % e, style='plain') return mark_notes_reviewed = bool(self.auth.mark_notes_reviewed) believed_dead_permission = bool( self.auth.believed_dead_permission) self.response.headers['Content-Type'] = 'application/xml; charset=utf-8' self.write('<?xml version="1.0"?>\n') self.write('<status:status ' + 'xmlns:status="http://zesty.ca/pfif/1.4/status" ' + 'xmlns:pfif="http://zesty.ca/pfif/1.4">\n') create_person = importer.create_person num_people_written, people_skipped, total = importer.import_records( self.repo, source_domain, create_person, person_records) self.write_status( 'person', num_people_written, people_skipped, total, 'person_record_id') create_note = importer.create_note num_notes_written, notes_skipped, total = importer.import_records( self.repo, source_domain, create_note, note_records, mark_notes_reviewed, believed_dead_permission, self) self.write_status( 'note', num_notes_written, notes_skipped, total, 'note_record_id') self.write('</status:status>\n') utils.log_api_action(self, ApiActionLog.WRITE, num_people_written, num_notes_written, len(people_skipped), len(notes_skipped))
def test_import_reviewed_note_records(self): # Prepare person records which the notes will be added to. for i in range(3): put_dummy_person_record('haiti', 'test_domain/person_%d' % i) records = [] for i in range(3): source_date = '2010-01-01T01:23:45Z' note_id = 'test_domain/record_%d' % i person_id = 'test_domain/person_%d' % i records.append({'person_record_id': person_id, 'note_record_id': note_id, 'source_date': source_date}) # Import reviewed notes. written, skipped, total = importer.import_records( 'haiti', 'test_domain', importer.create_note, records, True, True, None) assert written == 3 assert len(skipped) == 0 assert total == 3 # Confirm all records are marked reviewed. for note in model.Note.all(): assert note.reviewed == True
def test_import_reviewed_note_records(self): # Prepare person records which the notes will be added to. for i in range(3): put_dummy_person_record('haiti', 'test_domain/person_%d' % i) records = [] for i in range(3): source_date = '2010-01-01T01:23:45Z' note_id = 'test_domain/record_%d' % i person_id = 'test_domain/person_%d' % i records.append({ 'person_record_id': person_id, 'note_record_id': note_id, 'source_date': source_date }) # Import reviewed notes. written, skipped, total = importer.import_records( 'haiti', 'test_domain', importer.create_note, records, True, True, None) assert written == 3 assert len(skipped) == 0 assert total == 3 # Confirm all records are marked reviewed. for note in model.Note.all(): assert note.reviewed == True
def test_import_person_records(self): records = [] for i in range(20): given_name = "given_name_%d" % i family_name = "family_name_%d" % i source_date = "2010-01-01T01:23:45Z" record_id = "test_domain/%d" % i # Records 0, 8, and 16 have bad domains. if not i % 8: record_id = "other_domain/%d" % i # Records 0, 9, and 18 have invalid dates. elif not i % 9: source_date = "2010-01-01 01:23:45" records.append({ 'given_name': given_name, 'family_name': family_name, 'person_record_id': record_id, 'source_date': source_date }) written, skipped, total = importer.import_records( 'haiti', 'test_domain', importer.create_person, records, False, True, None) assert written == 15 assert len(skipped) == 5 assert skipped[0] == ('Not in authorized domain: u\'other_domain/0\'', { 'given_name': 'given_name_0', 'family_name': 'family_name_0', 'source_date': '2010-01-01T01:23:45Z', 'person_record_id': 'other_domain/0' }) assert skipped[3] == ('Not in authorized domain: u\'other_domain/16\'', { 'given_name': 'given_name_16', 'family_name': 'family_name_16', 'source_date': '2010-01-01T01:23:45Z', 'person_record_id': 'other_domain/16' }) assert skipped[2] == ( 'ValueError: Bad datetime: \'2010-01-01 01:23:45\'', { 'given_name': 'given_name_9', 'family_name': 'family_name_9', 'source_date': '2010-01-01 01:23:45', 'person_record_id': 'test_domain/9' }) assert skipped[4] == ( 'ValueError: Bad datetime: \'2010-01-01 01:23:45\'', { 'given_name': 'given_name_18', 'family_name': 'family_name_18', 'source_date': '2010-01-01 01:23:45', 'person_record_id': 'test_domain/18' }) assert total == 20 # Also confirm that 15 records were put into the datastore. assert model.Person.all().count() == 15
def test_import_person_records(self): records = [] for i in range(20): given_name = "given_name_%d" % i family_name = "family_name_%d" % i source_date = "2010-01-01T01:23:45Z" record_id = "test_domain/%d" % i # Records 0, 8, and 16 have bad domains. if not i % 8: record_id = "other_domain/%d" % i # Records 0, 9, and 18 have invalid dates. elif not i % 9: source_date = "2010-01-01 01:23:45" records.append({'given_name': given_name, 'family_name': family_name, 'person_record_id': record_id, 'source_date': source_date}) written, skipped, total = importer.import_records( 'haiti', 'test_domain', importer.create_person, records, False, True, None) assert written == 15 assert len(skipped) == 5 assert skipped[0] == ( 'Not in authorized domain: u\'other_domain/0\'', { 'given_name': 'given_name_0', 'family_name': 'family_name_0', 'source_date': '2010-01-01T01:23:45Z', 'person_record_id': 'other_domain/0' }) assert skipped[3] == ( 'Not in authorized domain: u\'other_domain/16\'', { 'given_name': 'given_name_16', 'family_name': 'family_name_16', 'source_date': '2010-01-01T01:23:45Z', 'person_record_id': 'other_domain/16' }) assert skipped[2] == ( 'ValueError: Bad datetime: \'2010-01-01 01:23:45\'', { 'given_name': 'given_name_9', 'family_name': 'family_name_9', 'source_date': '2010-01-01 01:23:45', 'person_record_id': 'test_domain/9' }) assert skipped[4] == ( 'ValueError: Bad datetime: \'2010-01-01 01:23:45\'', { 'given_name': 'given_name_18', 'family_name': 'family_name_18', 'source_date': '2010-01-01 01:23:45', 'person_record_id': 'test_domain/18' }) assert total == 20 # Also confirm that 15 records were put into the datastore. assert model.Person.all().count() == 15
def test_import_note_records(self): records = [] for i in range(20): source_date = '2010-01-01T01:23:45Z' note_id = 'test_domain/record_%d' % i person_id = 'test_domain/person_%d' % i # Records 0, 8, and 16 have bad note record domains. if not i % 8: note_id = 'other_domain/record_%d' % i # Records 0, 9, and 18 have bad person record domains. # This should not matter for note records. elif not i % 9: person_id = 'other_domain/person_%d' % i # Records 0, 5, 10, and 15 have invalid dates. elif not i % 5: source_date = '2010-01-01 01:23:45' records.append({'person_record_id': person_id, 'note_record_id': note_id, 'source_date': source_date}) written, skipped, total = importer.import_records( 'haiti', 'test_domain', importer.create_note, records, False, True, None) assert written == 14 assert len(skipped) == 6 assert skipped[0] == ( 'Not in authorized domain: u\'other_domain/record_0\'', { 'person_record_id': 'test_domain/person_0', 'source_date': '2010-01-01T01:23:45Z', 'note_record_id': 'other_domain/record_0' }) assert skipped[2] == ( 'Not in authorized domain: u\'other_domain/record_8\'', { 'person_record_id': 'test_domain/person_8', 'source_date': '2010-01-01T01:23:45Z', 'note_record_id': 'other_domain/record_8' }) assert skipped[1] == ( 'ValueError: Bad datetime: \'2010-01-01 01:23:45\'', { 'person_record_id': 'test_domain/person_5', 'source_date': '2010-01-01 01:23:45', 'note_record_id': 'test_domain/record_5' }) assert skipped[4] == ( 'ValueError: Bad datetime: \'2010-01-01 01:23:45\'', { 'person_record_id': 'test_domain/person_15', 'source_date': '2010-01-01 01:23:45', 'note_record_id': 'test_domain/record_15' }) assert total == 20 # Also confirm that 14 records were put into the datastore. assert model.Note.all().count() == 14 # Confirm all records are NOT marked reviewed. for note in model.Note.all(): assert note.reviewed == False
def import_from_file(host, kind, converter, filename): print '%s: importing %s records from %s' % (host, kind, filename) written, skipped, total = importer.import_records( source_domain, converter, importer.utf8_decoder(csv.DictReader(open(filename)))) for error, record in skipped[:SHOW_ERRORS]: print ' - %s: %r' % (error, record) if len(skipped) > SHOW_ERRORS: print ' (more errors not shown)' print 'wrote %d of %d (skipped %d with errors)' % ( written, total, len(skipped))
def import_from_file(host, repo, kind, converter, filename): print('%s: importing %s records from %s' % (host, kind, filename)) written, skipped, total = importer.import_records( repo, source_domain, converter, importer.utf8_decoder(csv.DictReader(open(filename)))) for error, record in skipped[:SHOW_ERRORS]: print(' - %s: %r' % (error, record)) if len(skipped) > SHOW_ERRORS: print(' (more errors not shown)') print('wrote %d of %d (skipped %d with errors)' % (written, total, len(skipped)))
def test_import_person_records(self): records = [] for i in range(20): first_name = "first_name_%d" % i last_name = "last_name_%d" % i source_date = "2010-01-01T01:23:45Z" record_id = "test_domain/%d" % i # Records 0, 8, and 16 have bad domains. if not i % 8: record_id = "other_domain/%d" % i # Records 0, 9, and 18 have invalid dates. elif not i % 9: source_date = "2010-01-01 01:23:45" records.append({"first_name": first_name, "last_name": last_name, "person_record_id": record_id, "source_date": source_date}) written, skipped, total = importer.import_records( 'test_domain', importer.create_person, records) self.assertEqual(written, 15) self.assertEqual(len(skipped), 5) self.assertEqual(skipped[0], ("Not in authorized domain: u'other_domain/0'", {"first_name": "first_name_0", "last_name": "last_name_0", "source_date": "2010-01-01T01:23:45Z", "person_record_id": "other_domain/0"})) self.assertEqual(skipped[3], ("Not in authorized domain: u'other_domain/16'", {"first_name": "first_name_16", "last_name": "last_name_16", "source_date": "2010-01-01T01:23:45Z", "person_record_id": "other_domain/16"})) self.assertEqual(skipped[2], ("ValueError: Bad datetime: '2010-01-01 01:23:45'", {"first_name": "first_name_9", "last_name": "last_name_9", "source_date": "2010-01-01 01:23:45", "person_record_id": "test_domain/9"})) self.assertEqual(skipped[4], ("ValueError: Bad datetime: '2010-01-01 01:23:45'", {"first_name": "first_name_18", "last_name": "last_name_18", "source_date": "2010-01-01 01:23:45", "person_record_id": "test_domain/18"})) self.assertEqual(total, 20) # Also confirm that 15 records were put into the datastore. self.assertEqual(model.Person.all().count(), 15)
def test_import_note_records(self): records = [] for i in range(20): source_date = "2010-01-01T01:23:45Z" note_id = "test_domain/record_%d" % i person_id = "test_domain/person_%d" % i # Records 0, 8, and 16 have bad note record domains. if not i % 8: note_id = "other_domain/record_%d" % i # Records 0, 9, and 18 have bad person record domains. # This should not matter for note records. elif not i % 9: person_id = "other_domain/person_%d" % i # Records 0, 5, 10, and 15 have invalid dates. elif not i % 5: source_date = "2010-01-01 01:23:45" records.append({"person_record_id": person_id, "note_record_id": note_id, "source_date": source_date}) written, skipped, total = importer.import_records( 'test_domain', importer.create_note, records) self.assertEqual(written, 14) self.assertEqual(len(skipped), 6) self.assertEqual(skipped[0], ("Not in authorized domain: u'other_domain/record_0'", {"person_record_id": "test_domain/person_0", "source_date": "2010-01-01T01:23:45Z", "note_record_id": "other_domain/record_0"})) self.assertEqual(skipped[2], ("Not in authorized domain: u'other_domain/record_8'", {"person_record_id": "test_domain/person_8", "source_date": "2010-01-01T01:23:45Z", "note_record_id": "other_domain/record_8"})) self.assertEqual(skipped[1], ("ValueError: Bad datetime: '2010-01-01 01:23:45'", {"person_record_id": "test_domain/person_5", "source_date": "2010-01-01 01:23:45", "note_record_id": "test_domain/record_5"})) self.assertEqual(skipped[4], ("ValueError: Bad datetime: '2010-01-01 01:23:45'", {"person_record_id": "test_domain/person_15", "source_date": "2010-01-01 01:23:45", "note_record_id": "test_domain/record_15"})) self.assertEqual(total, 20) # Also confirm that 14 records were put into the datastore. self.assertEqual(model.Note.all().count(), 14)
def test_import_note_records_for_non_existent_person(self): records = [{ 'person_record_id': 'test_domain/non_existent_person', 'note_record_id': 'test_domain/record_0', 'source_date': '2010-01-01T01:23:45Z', }] written, skipped, total = importer.import_records( 'haiti', 'test_domain', importer.create_note, records, False, True, None) assert written == 0 assert len(skipped) == 1 assert skipped[0] == ( "There is no person record with the person_record_id " "'test_domain/non_existent_person'", { 'person_record_id': 'test_domain/non_existent_person', 'source_date': '2010-01-01T01:23:45Z', 'note_record_id': 'test_domain/record_0' }) assert total == 1 assert model.Note.all().count() == 0
'smaller files (keeping the header rows in each file) and ' 'uploading each part separately.') def import_notes(self, lines): source_domain = self.auth.domain_write_permission records = importer.utf8_decoder(generate_note_record_ids( convert_time_fields(csv.reader(lines)))) try: records = [complete_record_ids(r, source_domain) for r in records] except csv.Error, e: self.error(400, message= 'The CSV file is formatted incorrectly. (%s)' % e) return notes_written, notes_skipped, notes_total = importer.import_records( self.repo, source_domain, importer.create_note, records, believed_dead_permission=self.auth.believed_dead_permission, omit_duplicate_notes=True) utils.log_api_action(self, ApiActionLog.WRITE, 0, notes_written, 0, len(notes_skipped)) self.render('import.html', formats=get_requested_formats(self.env.path), stats=[ Struct(type='Note', written=notes_written, skipped=notes_skipped, total=notes_total)], **get_tag_params(self)) def import_persons(self, lines):
def test_import_notes_disabled_note_records(self): '''Check that notes will be rejected from API import when notes_disabled is set to be True by the record author.''' records = [] # First prepare and import two person records for i in range(2): given_name = "given_name_%d" % i family_name = "family_name_%d" % i source_date = "2010-01-01T01:23:45Z" record_id = "test_domain/person_%d" % i author_name = "test_author" author_email = "test_email" records.append({'given_name': given_name, 'family_name': family_name, 'person_record_id': record_id, 'source_date': source_date, 'author_name': author_name, 'author_email': author_email}) written, skipped, total = importer.import_records( 'haiti', 'test_domain', importer.create_person, records, False, True, None) assert written == 2 assert len(skipped) == 0 assert total == 2 assert model.Person.all().count() == 2 # Disable comments for first person record person = model.Person.get('haiti', 'test_domain/person_0') assert person person.notes_disabled = True db.put([person]) for person in model.Person.all(): if person.person_record_id == 'test_domain/person_0': assert person.notes_disabled == True # Import notes records = [] for i in range(2): source_date = '2010-01-01T01:23:45Z' note_id = 'test_domain/record_%d' % i person_id = 'test_domain/person_%d' % i records.append({'person_record_id': person_id, 'note_record_id': note_id, 'source_date': source_date}) written, skipped, total = importer.import_records( 'haiti', 'test_domain', importer.create_note, records, False, True, None) # Check that the note associted with first person record is skipped. assert written == 1 assert len(skipped) == 1 assert skipped[0] == ( 'The author has disabled new commenting on this record', { 'person_record_id': 'test_domain/person_0', 'source_date': '2010-01-01T01:23:45Z', 'note_record_id': 'test_domain/record_0', }) assert total == 2 assert model.Note.all().count() == 1
def test_authorize_write_believed_dead_note_records(self): # Prepare input data records = [] for i in range(20): source_date = '2010-01-01T01:23:45Z' note_id = 'test_domain/record_%d' % i person_id = 'test_domain/person_%d' % i status = 'believed_missing' # Records 0, 8, and 16 have status 'believed_dead'. if not i % 8: status = 'believed_dead' records.append({'person_record_id': person_id, 'note_record_id': note_id, 'source_date': source_date, 'status': status}) # Disallow import notes with status 'believed_dead'. written, skipped, total = importer.import_records( 'haiti', 'test_domain', importer.create_note, records, False, False, None) assert written == 17 assert len(skipped) == 3 assert skipped[0] == ( 'Not authorized to post notes with the status' ' \"believed_dead\"', { 'person_record_id': 'test_domain/person_0', 'source_date': '2010-01-01T01:23:45Z', 'note_record_id': 'test_domain/record_0', 'status': 'believed_dead' }) assert skipped[1] == ( 'Not authorized to post notes with the status' ' \"believed_dead\"', { 'person_record_id': 'test_domain/person_8', 'source_date': '2010-01-01T01:23:45Z', 'note_record_id': 'test_domain/record_8', 'status': 'believed_dead' }) assert skipped[2] == ( 'Not authorized to post notes with the status' ' \"believed_dead\"', { 'person_record_id': 'test_domain/person_16', 'source_date': '2010-01-01T01:23:45Z', 'note_record_id': 'test_domain/record_16', 'status': 'believed_dead' }) assert total == 20 assert model.Note.all().count() == 17 # Allow import notes with status 'believed_dead'. model.Note.all().get().delete() written, skipped, total = importer.import_records( 'haiti', 'test_domain', importer.create_note, records, False, True, None) assert written == 20 assert len(skipped) == 0 assert model.Note.all().count() == 20 for note in model.Note.all(): if note.person_record_id in ['test_domain/person_0', 'test_domain/person_8', 'test_domain/person_16']: assert note.status == 'believed_dead' else: assert note.status == 'believed_missing'