def test_sync_all_schemata(self): """ Tests synchronizing all schemata from self (no xmlns posted) """ manager = XFormManager() # load data schema_1 = create_xsd_and_populate("pf_followup.xsd", path = DATA_DIR) schema_2 = create_xsd_and_populate("pf_new_reg.xsd", path = DATA_DIR) schema_3 = create_xsd_and_populate("pf_ref_completed.xsd", path = DATA_DIR) starting_schemata_count = FormDefModel.objects.all().count() # get sync file from self schemata_file = "schemata.tar" generate_schemata(serverhost, 'brian', 'test', latest=False, download=True, to=schemata_file) manager.remove_schema(schema_1.id, remove_submissions = True) manager.remove_schema(schema_2.id, remove_submissions = True) manager.remove_schema(schema_3.id, remove_submissions = True) # load data from sync file load_schemata(schemata_file, "127.0.0.1:8000") try: # verify that the submissions etc. count are correct self.assertEqual( starting_schemata_count, FormDefModel.objects.all().count()) finally: # clean up self._delete_schema_from_filename("pf_followup.xsd", path = DATA_DIR) self._delete_schema_from_filename("pf_new_reg.xsd", path = DATA_DIR) self._delete_schema_from_filename("pf_ref_completed.xsd", path = DATA_DIR)
def testSaveFormData_8(self): """ Test repeated form definition created and data saved """ formdefmodel = create_xsd_and_populate("data/8_singlerepeat_2.xsd") self.assertEquals(int(formdefmodel.version), 2) self.assertEquals(int(formdefmodel.uiversion), 3) submission = populate("data/8_singlerepeat_2.xml") m = Metadata.objects.get(attachment__submission=submission) self.assertEquals(int(m.version), 2) self.assertEquals(int(m.uiversion), 2) cursor = connection.cursor() cursor.execute("SELECT * FROM schema_xml_singlerepeat_2") try: row = cursor.fetchone() self.assertEquals(row[9],"deviceid0") self.assertEquals(row[10],"starttime") self.assertEquals(row[11],"endtime") cursor.execute("SELECT * FROM schema_xml_singlerepeat_root_userid_2") row = cursor.fetchall() self.assertEquals(row[0][1],"userid0") self.assertEquals(row[1][1],"userid2") self.assertEquals(row[2][1],"userid3") self.assertEquals(row[0][2],1) self.assertEquals(row[1][2],1) self.assertEquals(row[2][2],1) finally: manager = XFormManager() manager.remove_schema(formdefmodel.id)
def test_generate_all_submissions(self): """ Tests downloading all submissions from self """ # setup schema_1 = create_xsd_and_populate("pf_followup.xsd", path = DATA_DIR) submit_1 = populate("pf_followup_1.xml", path = DATA_DIR) submit_2 = populate("pf_followup_2.xml", path = DATA_DIR) schema_2 = create_xsd_and_populate("pf_new_reg.xsd", path = DATA_DIR) submit_3 = populate("pf_new_reg_1.xml", path = DATA_DIR) submit_4 = populate("pf_new_reg_2.xml", path = DATA_DIR) schema_3 = create_xsd_and_populate("pf_ref_completed.xsd", path = DATA_DIR) submit_5 = populate("pf_ref_completed_1.xml", path = DATA_DIR) submit_6 = populate("pf_ref_completed_2.xml", path = DATA_DIR) # download and check submissions_file = "submissions.tar" generate_submissions(serverhost, 'brian', 'test', latest=False, download=True, to=submissions_file) try: self._assert_tar_count_equals(submissions_file, Submission.objects.all().count()) # cleanup finally: # delete all data on self manager = XFormManager() submit_1.delete() submit_2.delete() submit_3.delete() submit_4.delete() submit_5.delete() submit_6.delete() manager.remove_schema(schema_1.id, remove_submissions = True) manager.remove_schema(schema_2.id, remove_submissions = True) manager.remove_schema(schema_3.id, remove_submissions = True)
def test_sync_no_schemata(self): """ Tests synchronizing no data from self (posts all MD5s) """ manager = XFormManager() # load data schema_1 = create_xsd_and_populate("pf_followup.xsd", path = DATA_DIR) schema_2 = create_xsd_and_populate("pf_new_reg.xsd", path = DATA_DIR) schema_3 = create_xsd_and_populate("pf_ref_completed.xsd", path = DATA_DIR) # get sync file from self schemata_file = 'schemata.tar' generate_schemata(serverhost, 'brian', 'test', download=True, to=schemata_file) # test that the received schemata file is empty self._assert_tar_count_equals(schemata_file, 0) starting_schemata_count = FormDefModel.objects.all().count() load_schemata(schemata_file, "127.0.0.1:8000") try: # verify that no new schemata were loaded self.assertEqual( starting_schemata_count, FormDefModel.objects.all().count()) finally: # clean up manager.remove_schema(schema_1.id, remove_submissions = True) manager.remove_schema(schema_2.id, remove_submissions = True) manager.remove_schema(schema_3.id, remove_submissions = True)
def test_generate_debug_submissions(self): """ Tests downloading some submissions from self This is only useful to make sure that the test_load_diff_submissions test below is working properly. """ #setup schema_1 = create_xsd_and_populate("pf_followup.xsd", \ "pf_followup_1.xml", path = DATA_DIR) populate("pf_followup_2.xml", path = DATA_DIR) schema_2 = create_xsd_and_populate("pf_new_reg.xsd", \ "pf_new_reg_1.xml", path = DATA_DIR) populate("pf_new_reg_2.xml", path = DATA_DIR) schema_3 = create_xsd_and_populate("pf_ref_completed.xsd", \ "pf_ref_completed_1.xml", path = DATA_DIR) populate("pf_ref_completed_2.xml", path = DATA_DIR) # the 'debug' flag limits the generated MD5s to a count of 5 submissions_file = "submissions.tar" # debug means we only post 5 submissions (instead of all) generate_submissions(serverhost, 'brian', 'test', debug=True, download=True, to=submissions_file) try: self._assert_tar_count_equals(submissions_file, Submission.objects.all().count()-5) # cleanup finally: # delete all data on self manager = XFormManager() manager.remove_schema(schema_1.id, remove_submissions = True) manager.remove_schema(schema_2.id, remove_submissions = True) manager.remove_schema(schema_3.id, remove_submissions = True)
def test_sync_some_submissions(self): """ Tests synchronizing some data from self (posts a few MD5s) """ manager = XFormManager() # populate some files schema_1 = create_xsd_and_populate("pf_followup.xsd", "pf_followup_1.xml", path = DATA_DIR) schema_2 = create_xsd_and_populate("pf_new_reg.xsd", "pf_new_reg_1.xml", path = DATA_DIR) schema_3 = create_xsd_and_populate("pf_ref_completed.xsd", "pf_ref_completed_1.xml", path = DATA_DIR) # get MD5 of all current submissions MD5_buffer = rest_util.get_field_as_bz2(Submission, 'checksum') # populate a few more files submit_1 = populate("pf_followup_2.xml", path = DATA_DIR) submit_2 = populate("pf_new_reg_2.xml", path = DATA_DIR) submit_3 = populate("pf_ref_completed_2.xml", path = DATA_DIR) submit_4 = populate("pf_ref_completed_3.xml", path = DATA_DIR) starting_submissions_count = Submission.objects.all().count() starting_schemata_count = FormDefModel.objects.all().count() # get the difference between the first 3 files and the current # set of files (i.e. the last 4 files) submissions_file = "submissions.tar" self._POST_MD5s(MD5_buffer, submissions_file) # save checksums and delete the ones just populated (d,e,f) checksums = [ submit_1.checksum, submit_2.checksum, submit_3.checksum, submit_3.checksum ] manager.remove_data(schema_1.id, Metadata.objects.get(attachment=submit_1.xform).raw_data, \ remove_submission = True) manager.remove_data(schema_2.id, Metadata.objects.get(attachment=submit_2.xform).raw_data, \ remove_submission = True) manager.remove_data(schema_3.id, Metadata.objects.get(attachment=submit_3.xform).raw_data, \ remove_submission = True) manager.remove_data(schema_3.id, Metadata.objects.get(attachment=submit_4.xform).raw_data, \ remove_submission = True) # load data from sync file (d,e,f) load_submissions(submissions_file, "127.0.0.1:8000") try: # verify that the submissions etc. count are correct (d,e,f) self.assertEqual( starting_submissions_count, Submission.objects.all().count()) submits = Submission.objects.all().order_by('-submit_time')[:4] # verify that the correct submissions were loaded Submission.objects.get(checksum=checksums[0]) Submission.objects.get(checksum=checksums[1]) Submission.objects.get(checksum=checksums[2]) Submission.objects.get(checksum=checksums[3]) except Submission.DoesNotExist: self.fail("Incorrect submission received") finally: # clean up manager = XFormManager() manager.remove_schema(schema_1.id, remove_submissions = True) manager.remove_schema(schema_2.id, remove_submissions = True) manager.remove_schema(schema_3.id, remove_submissions = True)
def _delete_schema_from_filename(self, file_name, path): schema = open(os.path.join(path, file_name), 'r') formdef = FormDef(schema) schema.close() try: formdef = FormDefModel.objects.get(target_namespace=formdef.target_namespace) except FormDefModel.DoesNotExist: return manager = XFormManager() manager.remove_schema(formdef.id, remove_submissions=True)
def test_sync_dupe_submissions(self): """ Tests synchronizing duplicate data from self""" manager = XFormManager() # populate some files schema_1 = create_xsd_and_populate("pf_followup.xsd", "pf_followup_1.xml", path = DATA_DIR) schema_2 = create_xsd_and_populate("pf_new_reg.xsd", "pf_new_reg_1.xml", path = DATA_DIR) schema_3 = create_xsd_and_populate("pf_ref_completed.xsd", "pf_ref_completed_1.xml", path = DATA_DIR) starting_submissions_count = Submission.objects.all().count() # <STATE 1/> # get MD5 of 3 populated files MD5_buffer = rest_util.get_field_as_bz2(Submission, 'checksum') # add 3 dupes and 1 new file submit_1 = populate("pf_followup_1.xml", path = DATA_DIR) submit_2 = populate("pf_new_reg_1.xml", path = DATA_DIR) submit_3 = populate("pf_ref_completed_1.xml", path = DATA_DIR) # <STATE 2/> submissions_file = "submissions.tar" self._POST_MD5s(MD5_buffer, submissions_file) self._assert_tar_count_equals(submissions_file, 0) submit_4 = populate("pf_ref_completed_3.xml", path = DATA_DIR) # <STATE 3/> # get the difference between state 1 and state 3 self._POST_MD5s(MD5_buffer, submissions_file) # save checksum and delete the ones just populated checksum_4 = submit_4.checksum submit_1.delete() submit_2.delete() submit_3.delete() submit_4.delete() # should get the same 3 schemas we registered above self._assert_tar_count_equals(submissions_file, 1) # load data from sync file (d,e,f) load_submissions(submissions_file, "127.0.0.1:8000") try: # verify that we only have 4 submissions self.assertEqual( starting_submissions_count+1, Submission.objects.all().count() ) Submission.objects.get(checksum=checksum_4) except Submission.DoesNotExist: self.fail("Incorrect submission received") finally: # clean up manager = XFormManager() manager.remove_schema(schema_1.id, remove_submissions = True) manager.remove_schema(schema_2.id, remove_submissions = True) manager.remove_schema(schema_3.id, remove_submissions = True)
def remove_xform(request, form_id=None, template='confirm_delete.html'): context = {} form = get_object_or_404(FormDefModel, pk=form_id) if request.method == "POST": if request.POST["confirm_delete"]: # The user has already confirmed the deletion. xformmanager = XFormManager() xformmanager.remove_schema(form_id) logging.debug("Schema %s deleted ", form_id) #self.message_user(request, _('The %(name)s "%(obj)s" was deleted successfully.') % {'name': force_unicode(opts.verbose_name), 'obj': force_unicode(obj_display)}) return HttpResponseRedirect(reverse('xformmanager.views.dashboard')) context['form_name'] = form.form_display_name return render_to_response(request, template, context)
def _testErrors(self, schema_file, instance_file, id): su = SU() xfm = XFM() xsd_file_name = os.path.join(os.path.dirname(__file__),schema_file) xml_file_name = os.path.join(os.path.dirname(__file__),instance_file) schema = xfm._add_schema_from_file(xsd_file_name) formdef = su.get_formdef_from_schema_file(xsd_file_name) data_tree = su._get_data_tree_from_file(xml_file_name) populator = XFormDBTablePopulator( formdef, schema ) queries = populator.populate( data_tree ) xfm.remove_schema(schema.id) return populator.errors
def testSaveFormData_7(self): """ Test very deep form definition created and data saved """ formdefmodel = create_xsd_and_populate("data/7_verydeep_2.xsd", "data/7_verydeep_2.xml") cursor = connection.cursor() cursor.execute("SELECT * FROM schema_xml_verydeep_1") row = cursor.fetchone() try: self.assertEquals(row[9],"userid0") self.assertEquals(row[10],"great_grand1") self.assertEquals(row[11],222) self.assertEquals(row[12],1159) self.assertEquals(row[13],2002) finally: manager = XFormManager() manager.remove_schema(formdefmodel.id)
def test_sync_weird_submissions(self): """ Tests synchronizing some data from self (posts a few MD5s) """ # setup - if we don't do this, we just get back "no submissions found" manager = XFormManager() # populate some files schema_1 = create_xsd_and_populate("pf_followup.xsd", "pf_followup_1.xml", path = DATA_DIR) schema_2 = create_xsd_and_populate("pf_new_reg.xsd", "pf_new_reg_1.xml", path = DATA_DIR) schema_3 = create_xsd_and_populate("pf_ref_completed.xsd", "pf_ref_completed_1.xml", path = DATA_DIR) submissions_count = Submission.objects.count() url = 'http://%s/api/submissions/' % (serverhost) up = urlparse(url) conn = httplib.HTTPConnection(up.netloc) # test posting junk md5 MD5_buffer = "sadfndan;ofansdn" conn.request('POST', up.path, MD5_buffer, {'Content-Type': 'application/bz2', 'User-Agent': 'CCHQ-submitfromfile-python-v0.1'}) response = conn.getresponse().read() self.assertTrue( response.lower().find('poorly formatted') != -1 ) # test posting non-existent md5s md5 = "e402f026c762a6bc999f9f2703efd367\n" bz2_md5 = bz2.compress(md5) submissions_file = "submissions.tar" self._POST_MD5s(bz2_md5, submissions_file) # should get the same 3 schemas we registered above self._assert_tar_count_equals(submissions_file, submissions_count) # test posting duplicate md5s string = cStringIO.StringIO() submits = Submission.objects.all().order_by('checksum')[:2] for submit in submits: string.write(unicode( submit.checksum ) + '\n') string.write(unicode( submit.checksum ) + '\n') MD5s = string.getvalue() dupe_buffer = bz2.compress(MD5s) submissions_file = "submissions.tar" self._POST_MD5s(dupe_buffer, submissions_file) self._assert_tar_count_equals(submissions_file, submissions_count-2) manager.remove_schema(schema_1.id, remove_submissions = True) manager.remove_schema(schema_2.id, remove_submissions = True) manager.remove_schema(schema_3.id, remove_submissions = True)
def test_sync_weird_schemata(self): """ Tests synchronizing some data from self (posts a few MD5s) """ # setup - if we don't do this, we just get back "no submissions found" manager = XFormManager() # populate some files starting_schemata_count = FormDefModel.objects.count() schema_1 = create_xsd_and_populate("pf_followup.xsd", path = DATA_DIR) schema_2 = create_xsd_and_populate("pf_new_reg.xsd", path = DATA_DIR) schema_3 = create_xsd_and_populate("pf_ref_completed.xsd", path = DATA_DIR) url = 'http://%s/api/xforms/?format=sync' % (serverhost) up = urlparse(url) conn = httplib.HTTPConnection(up.netloc) # test posting junk namespace namespace_buffer = "sadfndan;ofansdn" conn.request('POST', up.path, namespace_buffer, {'Content-Type': 'application/bz2', 'User-Agent': 'CCHQ-submitfromfile-python-v0.1'}) response = conn.getresponse().read() self.assertTrue( response.lower().find('poorly formatted') != -1 ) # test posting non-existent namespaces namespace = "http://zilch.com" bz2_namespace = bz2.compress(namespace) schemata_file = "schemata.tar" self._POST_XMLNS(bz2_namespace, schemata_file) # should get all the schemas back self._assert_tar_count_equals(schemata_file, starting_schemata_count+3) # test posting duplicate namespaces string = cStringIO.StringIO() formdefs = FormDefModel.objects.all().order_by('target_namespace')[:2] for formdef in formdefs: string.write(unicode( formdef.target_namespace ) + '\n') string.write(unicode( formdef.target_namespace ) + '\n') dupe_buffer = bz2.compress(string.getvalue()) self._POST_XMLNS(dupe_buffer, schemata_file) self._assert_tar_count_equals(schemata_file, starting_schemata_count+1) manager.remove_schema(schema_1.id, remove_submissions = True) manager.remove_schema(schema_2.id, remove_submissions = True) manager.remove_schema(schema_3.id, remove_submissions = True)
def testBasicViews(self): domain = Domain.objects.get(name='mockdomain') formdef = create_xsd_and_populate("data/pf_followup.xsd", "data/pf_followup_1.xml", domain) instance = populate("data/pf_followup_2.xml") response = self.client.get('/xforms/') self.assertNotContains(response,"Error", status_code=200) self.assertNotContains(response,"Exception", status_code=200) response = self.client.get('/xforms/register/') self.assertNotContains(response,"Error", status_code=200) self.assertNotContains(response,"Exception", status_code=200) response = self.client.get('/xforms/reregister/mockdomain/') self.assertNotContains(response,"Error", status_code=200) self.assertNotContains(response,"Exception", status_code=200) response = self.client.get('/xforms/remove/%s/' % formdef.id) self.assertNotContains(response,"Error", status_code=200) self.assertNotContains(response,"Exception", status_code=200) response = self.client.get('/xforms/%s/submit/' % formdef.id) self.assertNotContains(response,"Error", status_code=200) self.assertNotContains(response,"Exception", status_code=200) urls = [ '/xforms/show/%s/' % formdef.id, '/xforms/show/%s/%s/' % (formdef.id, instance.id), '/xforms/show/%s/%s/csv/' % (formdef.id, instance.id), '/xforms/data/%s/delete/' % formdef.id, '/xforms/data/%s/' % formdef.id, '/xforms/data/%s/csv/' % formdef.id, '/xforms/data/%s/xml/' % formdef.id, ] for url in urls: self._test_valid_and_permissions(url, self.authuser, self.unauthuser) # format url variables like so: # response = self.client.get('/api/xforms/',{'format':'json'}) manager = XFormManager() manager.remove_schema(formdef.id)
def testSaveFormData_10(self): """ Test nested repeated form definition created and data saved """ formdefmodel = create_xsd_and_populate("6_nestedrepeats.xsd", "6_nestedrepeats.xml") formdefmodel_2 = create_xsd_and_populate("data/9_nestedrepeats_2.xsd", "data/9_nestedrepeats_2.xml") formdefmodel_3 = create_xsd_and_populate("data/10_other_v3.xsd", "data/10_other_v3.xml") try: cursor = connection.cursor() cursor.execute("SELECT * FROM schema_xml_nestedrepeats") row = cursor.fetchone() self.assertEquals(row[9],"foo") self.assertEquals(row[10],"bar") self.assertEquals(row[11],"yes") self.assertEquals(row[12],"no") cursor.execute("SELECT * FROM schema_xml_nestedrepeats_root_nested") row = cursor.fetchall() self.assertEquals(row[0][1],"userid0") self.assertEquals(row[0][2],"deviceid0") if settings.DATABASE_ENGINE=='mysql' : self.assertEquals(row[0][3],datetime(2009,10,9,11,4,30) ) self.assertEquals(row[0][4],datetime(2009,10,9,11,9,30) ) else: self.assertEquals(row[0][3],"2009-10-9 11:04:30" ) self.assertEquals(row[0][4],"2009-10-9 11:09:30" ) self.assertEquals(row[0][5],1) self.assertEquals(row[1][1],"userid2") self.assertEquals(row[1][2],"deviceid2") if settings.DATABASE_ENGINE=='mysql' : self.assertEquals(row[1][3],datetime(2009,11,12,11,11,11) ) self.assertEquals(row[1][4],datetime(2009,11,12,11,16,11) ) else: self.assertEquals(row[1][3],"2009-11-12 11:11:11" ) self.assertEquals(row[1][4],"2009-11-12 11:16:11" ) self.assertEquals(row[1][5],1) finally: manager = XFormManager() manager.remove_schema(formdefmodel.id) manager.remove_schema(formdefmodel_2.id) manager.remove_schema(formdefmodel_3.id)
def test_sync_some_schemata(self): """ Tests synchronizing some schemata from self (posts a few xmlns) """ manager = XFormManager() # populate some files schema_1 = create_xsd_and_populate("pf_followup.xsd", path = DATA_DIR) # get xmlns of populated schemas xmlns_buffer = rest_util.get_field_as_bz2(FormDefModel, 'target_namespace') # populate a few more schema schema_2 = create_xsd_and_populate("pf_new_reg.xsd", path = DATA_DIR) schema_3 = create_xsd_and_populate("pf_ref_completed.xsd", path = DATA_DIR) starting_schemata_count = FormDefModel.objects.all().count() # get the difference between the first schema and current state schemata_file = "schemata.tar" self._POST_XMLNS(xmlns_buffer, schemata_file) # delete the ones just populated (d,e,f) manager.remove_schema(schema_2.id, remove_submissions = True) manager.remove_schema(schema_3.id, remove_submissions = True) # load data from sync file (d,e,f) load_schemata(schemata_file, "127.0.0.1:8000") try: # verify that the schematas etc. count are correct (d,e,f) self.assertEqual( starting_schemata_count, FormDefModel.objects.all().count()) self._assert_schema_registered("pf_followup.xsd", DATA_DIR) self._assert_schema_registered("pf_new_reg.xsd", DATA_DIR) self._assert_schema_registered("pf_ref_completed.xsd", DATA_DIR) finally: # clean up manager = XFormManager() manager.remove_schema(schema_1.id, remove_submissions = True) self._delete_schema_from_filename("pf_new_reg.xsd", path = DATA_DIR) self._delete_schema_from_filename("pf_ref_completed.xsd", path = DATA_DIR)
def tearDown(self): manager = XFormManager() manager.remove_schema(1)