def test_generate_debug_submissions(self):
""" Tests downloading some submissions from self
This is only useful to make sure that the test_load_diff_submissions test
below is working properly.
"""
#setup
schema_1 = create_xsd_and_populate("pf_followup.xsd", \
"pf_followup_1.xml", path = DATA_DIR)
populate("pf_followup_2.xml", path = DATA_DIR)
schema_2 = create_xsd_and_populate("pf_new_reg.xsd", \
"pf_new_reg_1.xml", path = DATA_DIR)
populate("pf_new_reg_2.xml", path = DATA_DIR)
schema_3 = create_xsd_and_populate("pf_ref_completed.xsd", \
"pf_ref_completed_1.xml", path = DATA_DIR)
populate("pf_ref_completed_2.xml", path = DATA_DIR)
# the 'debug' flag limits the generated MD5s to a count of 5
submissions_file = "submissions.tar"
# debug means we only post 5 submissions (instead of all)
generate_submissions(serverhost, 'brian', 'test', debug=True, download=True, to=submissions_file)
try:
self._assert_tar_count_equals(submissions_file, Submission.objects.all().count()-5)
# cleanup
finally:
# delete all data on self
manager = XFormManager()
manager.remove_schema(schema_1.id, remove_submissions = True)
manager.remove_schema(schema_2.id, remove_submissions = True)
manager.remove_schema(schema_3.id, remove_submissions = True)
def test_generate_all_submissions(self):
""" Tests downloading all submissions from self """
# setup
schema_1 = create_xsd_and_populate("pf_followup.xsd", path = DATA_DIR)
submit_1 = populate("pf_followup_1.xml", path = DATA_DIR)
submit_2 = populate("pf_followup_2.xml", path = DATA_DIR)
schema_2 = create_xsd_and_populate("pf_new_reg.xsd", path = DATA_DIR)
submit_3 = populate("pf_new_reg_1.xml", path = DATA_DIR)
submit_4 = populate("pf_new_reg_2.xml", path = DATA_DIR)
schema_3 = create_xsd_and_populate("pf_ref_completed.xsd", path = DATA_DIR)
submit_5 = populate("pf_ref_completed_1.xml", path = DATA_DIR)
submit_6 = populate("pf_ref_completed_2.xml", path = DATA_DIR)
# download and check
submissions_file = "submissions.tar"
generate_submissions(serverhost, 'brian', 'test', latest=False, download=True, to=submissions_file)
try:
self._assert_tar_count_equals(submissions_file, Submission.objects.all().count())
# cleanup
finally:
# delete all data on self
manager = XFormManager()
submit_1.delete()
submit_2.delete()
submit_3.delete()
submit_4.delete()
submit_5.delete()
submit_6.delete()
manager.remove_schema(schema_1.id, remove_submissions = True)
manager.remove_schema(schema_2.id, remove_submissions = True)
manager.remove_schema(schema_3.id, remove_submissions = True)
def test_sync_no_schemata(self):
""" Tests synchronizing no data from self (posts all MD5s) """
manager = XFormManager()
# load data
schema_1 = create_xsd_and_populate("pf_followup.xsd", path = DATA_DIR)
schema_2 = create_xsd_and_populate("pf_new_reg.xsd", path = DATA_DIR)
schema_3 = create_xsd_and_populate("pf_ref_completed.xsd", path = DATA_DIR)
# get sync file from self
schemata_file = 'schemata.tar'
generate_schemata(serverhost, 'brian', 'test', download=True, to=schemata_file)
# test that the received schemata file is empty
self._assert_tar_count_equals(schemata_file, 0)
starting_schemata_count = FormDefModel.objects.all().count()
load_schemata(schemata_file, "127.0.0.1:8000")
try:
# verify that no new schemata were loaded
self.assertEqual( starting_schemata_count, FormDefModel.objects.all().count())
finally:
# clean up
manager.remove_schema(schema_1.id, remove_submissions = True)
manager.remove_schema(schema_2.id, remove_submissions = True)
manager.remove_schema(schema_3.id, remove_submissions = True)
def test_sync_all_schemata(self):
""" Tests synchronizing all schemata from self (no xmlns posted) """
manager = XFormManager()
# load data
schema_1 = create_xsd_and_populate("pf_followup.xsd", path = DATA_DIR)
schema_2 = create_xsd_and_populate("pf_new_reg.xsd", path = DATA_DIR)
schema_3 = create_xsd_and_populate("pf_ref_completed.xsd", path = DATA_DIR)
starting_schemata_count = FormDefModel.objects.all().count()
# get sync file from self
schemata_file = "schemata.tar"
generate_schemata(serverhost, 'brian', 'test', latest=False, download=True, to=schemata_file)
manager.remove_schema(schema_1.id, remove_submissions = True)
manager.remove_schema(schema_2.id, remove_submissions = True)
manager.remove_schema(schema_3.id, remove_submissions = True)
# load data from sync file
load_schemata(schemata_file, "127.0.0.1:8000")
try:
# verify that the submissions etc. count are correct
self.assertEqual( starting_schemata_count, FormDefModel.objects.all().count())
finally:
# clean up
self._delete_schema_from_filename("pf_followup.xsd", path = DATA_DIR)
self._delete_schema_from_filename("pf_new_reg.xsd", path = DATA_DIR)
self._delete_schema_from_filename("pf_ref_completed.xsd", path = DATA_DIR)
def setUp(self):
# we cannot load this using django built-in fixtures
# because django filters are model dependent
# (and we have a whack of dynamically generated non-model db tables)
clear_data()
user, domain = create_user_and_domain()
create_xsd_and_populate("data/brac_chw.xsd", "data/brac_chw_1.xml", domain)
self.client.login(username='******',password='******')
def test_sync_some_submissions(self):
""" Tests synchronizing some data from self (posts a few MD5s) """
manager = XFormManager()
# populate some files
schema_1 = create_xsd_and_populate("pf_followup.xsd", "pf_followup_1.xml", path = DATA_DIR)
schema_2 = create_xsd_and_populate("pf_new_reg.xsd", "pf_new_reg_1.xml", path = DATA_DIR)
schema_3 = create_xsd_and_populate("pf_ref_completed.xsd", "pf_ref_completed_1.xml", path = DATA_DIR)
# get MD5 of all current submissions
MD5_buffer = rest_util.get_field_as_bz2(Submission, 'checksum')
# populate a few more files
submit_1 = populate("pf_followup_2.xml", path = DATA_DIR)
submit_2 = populate("pf_new_reg_2.xml", path = DATA_DIR)
submit_3 = populate("pf_ref_completed_2.xml", path = DATA_DIR)
submit_4 = populate("pf_ref_completed_3.xml", path = DATA_DIR)
starting_submissions_count = Submission.objects.all().count()
starting_schemata_count = FormDefModel.objects.all().count()
# get the difference between the first 3 files and the current
# set of files (i.e. the last 4 files)
submissions_file = "submissions.tar"
self._POST_MD5s(MD5_buffer, submissions_file)
# save checksums and delete the ones just populated (d,e,f)
checksums = [ submit_1.checksum, submit_2.checksum, submit_3.checksum, submit_3.checksum ]
manager.remove_data(schema_1.id, Metadata.objects.get(attachment=submit_1.xform).raw_data, \
remove_submission = True)
manager.remove_data(schema_2.id, Metadata.objects.get(attachment=submit_2.xform).raw_data, \
remove_submission = True)
manager.remove_data(schema_3.id, Metadata.objects.get(attachment=submit_3.xform).raw_data, \
remove_submission = True)
manager.remove_data(schema_3.id, Metadata.objects.get(attachment=submit_4.xform).raw_data, \
remove_submission = True)
# load data from sync file (d,e,f)
load_submissions(submissions_file, "127.0.0.1:8000")
try:
# verify that the submissions etc. count are correct (d,e,f)
self.assertEqual( starting_submissions_count, Submission.objects.all().count())
submits = Submission.objects.all().order_by('-submit_time')[:4]
# verify that the correct submissions were loaded
Submission.objects.get(checksum=checksums[0])
Submission.objects.get(checksum=checksums[1])
Submission.objects.get(checksum=checksums[2])
Submission.objects.get(checksum=checksums[3])
except Submission.DoesNotExist:
self.fail("Incorrect submission received")
finally:
# clean up
manager = XFormManager()
manager.remove_schema(schema_1.id, remove_submissions = True)
manager.remove_schema(schema_2.id, remove_submissions = True)
manager.remove_schema(schema_3.id, remove_submissions = True)
def testFull(self):
"""Tests the creation of a form group from several forms,
including added, deleted, and changed fields."""
fd2_dup = create_xsd_and_populate("data/versioning/base.2.xsd")
fd3_add = create_xsd_and_populate("data/versioning/base.3.addition.xsd")
fd4_del = create_xsd_and_populate("data/versioning/base.4.deletion.xsd")
fd5_mod = create_xsd_and_populate("data/versioning/base.5.data_type_change.xsd")
original_list = [self.original_formdef, fd2_dup, fd3_add, fd4_del, fd5_mod]
dom = Domain.objects.all()[0]
group = FormDataGroup.from_forms(original_list, dom)
self.assertEqual(5, len(group.forms.all()))
for form in group.forms.all():
self.assertTrue(form in original_list)
columns = self.original_formdef.get_data_column_names()
# this is added by form 3
columns.append("root_added_field")
# a second one of these is added by form 5
columns.append("meta_username_2")
self.assertEqual(len(columns), len(group.columns.all()))
for group_column in group.columns.all():
self.assertTrue(group_column.name in columns,
"%s was found in the list of columns: %s" % \
(group_column.name, columns))
for form in original_list:
self._check_columns(form, group)
# also make sure the view was created.
query = "SELECT * from %s" % group.view_name
cursor = connection.cursor()
cursor.execute(query)
view_cols_expected = ["form_id"]
view_cols_expected.extend(columns)
view_column_names = get_column_names(cursor)
for view_column in view_column_names:
self.assertTrue(view_column in view_cols_expected,
"%s was found in the list of view columns" % \
view_column)
# test deletion.
orig_col_count = group.columns.count()
group.remove_form(fd3_add)
group = FormDataGroup.objects.get(id=group.id)
self.assertEqual(group.columns.count(), orig_col_count - 1)
self.assertFalse("root_added_field" in group.columns.values_list("name", flat=True))
# test adding it back
group.add_form(fd3_add)
group = FormDataGroup.objects.get(id=group.id)
self.assertEqual(group.columns.count(), orig_col_count)
columns.append("root_added_field")
self.assertTrue("root_added_field" in group.columns.values_list("name", flat=True))
def test_sync_dupe_submissions(self):
""" Tests synchronizing duplicate data from self"""
manager = XFormManager()
# populate some files
schema_1 = create_xsd_and_populate("pf_followup.xsd", "pf_followup_1.xml", path = DATA_DIR)
schema_2 = create_xsd_and_populate("pf_new_reg.xsd", "pf_new_reg_1.xml", path = DATA_DIR)
schema_3 = create_xsd_and_populate("pf_ref_completed.xsd", "pf_ref_completed_1.xml", path = DATA_DIR)
starting_submissions_count = Submission.objects.all().count()
# <STATE 1/>
# get MD5 of 3 populated files
MD5_buffer = rest_util.get_field_as_bz2(Submission, 'checksum')
# add 3 dupes and 1 new file
submit_1 = populate("pf_followup_1.xml", path = DATA_DIR)
submit_2 = populate("pf_new_reg_1.xml", path = DATA_DIR)
submit_3 = populate("pf_ref_completed_1.xml", path = DATA_DIR)
# <STATE 2/>
submissions_file = "submissions.tar"
self._POST_MD5s(MD5_buffer, submissions_file)
self._assert_tar_count_equals(submissions_file, 0)
submit_4 = populate("pf_ref_completed_3.xml", path = DATA_DIR)
# <STATE 3/>
# get the difference between state 1 and state 3
self._POST_MD5s(MD5_buffer, submissions_file)
# save checksum and delete the ones just populated
checksum_4 = submit_4.checksum
submit_1.delete()
submit_2.delete()
submit_3.delete()
submit_4.delete()
# should get the same 3 schemas we registered above
self._assert_tar_count_equals(submissions_file, 1)
# load data from sync file (d,e,f)
load_submissions(submissions_file, "127.0.0.1:8000")
try:
# verify that we only have 4 submissions
self.assertEqual( starting_submissions_count+1, Submission.objects.all().count() )
Submission.objects.get(checksum=checksum_4)
except Submission.DoesNotExist:
self.fail("Incorrect submission received")
finally:
# clean up
manager = XFormManager()
manager.remove_schema(schema_1.id, remove_submissions = True)
manager.remove_schema(schema_2.id, remove_submissions = True)
manager.remove_schema(schema_3.id, remove_submissions = True)
def test_sync_weird_submissions(self):
""" Tests synchronizing some data from self (posts a few MD5s) """
# setup - if we don't do this, we just get back "no submissions found"
manager = XFormManager()
# populate some files
schema_1 = create_xsd_and_populate("pf_followup.xsd", "pf_followup_1.xml", path = DATA_DIR)
schema_2 = create_xsd_and_populate("pf_new_reg.xsd", "pf_new_reg_1.xml", path = DATA_DIR)
schema_3 = create_xsd_and_populate("pf_ref_completed.xsd", "pf_ref_completed_1.xml", path = DATA_DIR)
submissions_count = Submission.objects.count()
url = 'http://%s/api/submissions/' % (serverhost)
up = urlparse(url)
conn = httplib.HTTPConnection(up.netloc)
# test posting junk md5
MD5_buffer = "sadfndan;ofansdn"
conn.request('POST', up.path, MD5_buffer, {'Content-Type': 'application/bz2', 'User-Agent': 'CCHQ-submitfromfile-python-v0.1'})
response = conn.getresponse().read()
self.assertTrue( response.lower().find('poorly formatted') != -1 )
# test posting non-existent md5s
md5 = "e402f026c762a6bc999f9f2703efd367\n"
bz2_md5 = bz2.compress(md5)
submissions_file = "submissions.tar"
self._POST_MD5s(bz2_md5, submissions_file)
# should get the same 3 schemas we registered above
self._assert_tar_count_equals(submissions_file, submissions_count)
# test posting duplicate md5s
string = cStringIO.StringIO()
submits = Submission.objects.all().order_by('checksum')[:2]
for submit in submits:
string.write(unicode( submit.checksum ) + '\n')
string.write(unicode( submit.checksum ) + '\n')
MD5s = string.getvalue()
dupe_buffer = bz2.compress(MD5s)
submissions_file = "submissions.tar"
self._POST_MD5s(dupe_buffer, submissions_file)
self._assert_tar_count_equals(submissions_file, submissions_count-2)
manager.remove_schema(schema_1.id, remove_submissions = True)
manager.remove_schema(schema_2.id, remove_submissions = True)
manager.remove_schema(schema_3.id, remove_submissions = True)
def test_sync_weird_schemata(self):
""" Tests synchronizing some data from self (posts a few MD5s) """
# setup - if we don't do this, we just get back "no submissions found"
manager = XFormManager()
# populate some files
starting_schemata_count = FormDefModel.objects.count()
schema_1 = create_xsd_and_populate("pf_followup.xsd", path = DATA_DIR)
schema_2 = create_xsd_and_populate("pf_new_reg.xsd", path = DATA_DIR)
schema_3 = create_xsd_and_populate("pf_ref_completed.xsd", path = DATA_DIR)
url = 'http://%s/api/xforms/?format=sync' % (serverhost)
up = urlparse(url)
conn = httplib.HTTPConnection(up.netloc)
# test posting junk namespace
namespace_buffer = "sadfndan;ofansdn"
conn.request('POST', up.path, namespace_buffer, {'Content-Type': 'application/bz2', 'User-Agent': 'CCHQ-submitfromfile-python-v0.1'})
response = conn.getresponse().read()
self.assertTrue( response.lower().find('poorly formatted') != -1 )
# test posting non-existent namespaces
namespace = "http://zilch.com"
bz2_namespace = bz2.compress(namespace)
schemata_file = "schemata.tar"
self._POST_XMLNS(bz2_namespace, schemata_file)
# should get all the schemas back
self._assert_tar_count_equals(schemata_file, starting_schemata_count+3)
# test posting duplicate namespaces
string = cStringIO.StringIO()
formdefs = FormDefModel.objects.all().order_by('target_namespace')[:2]
for formdef in formdefs:
string.write(unicode( formdef.target_namespace ) + '\n')
string.write(unicode( formdef.target_namespace ) + '\n')
dupe_buffer = bz2.compress(string.getvalue())
self._POST_XMLNS(dupe_buffer, schemata_file)
self._assert_tar_count_equals(schemata_file, starting_schemata_count+1)
manager.remove_schema(schema_1.id, remove_submissions = True)
manager.remove_schema(schema_2.id, remove_submissions = True)
manager.remove_schema(schema_3.id, remove_submissions = True)
def test_sync_some_schemata(self):
""" Tests synchronizing some schemata from self (posts a few xmlns) """
manager = XFormManager()
# populate some files
schema_1 = create_xsd_and_populate("pf_followup.xsd", path = DATA_DIR)
# get xmlns of populated schemas
xmlns_buffer = rest_util.get_field_as_bz2(FormDefModel, 'target_namespace')
# populate a few more schema
schema_2 = create_xsd_and_populate("pf_new_reg.xsd", path = DATA_DIR)
schema_3 = create_xsd_and_populate("pf_ref_completed.xsd", path = DATA_DIR)
starting_schemata_count = FormDefModel.objects.all().count()
# get the difference between the first schema and current state
schemata_file = "schemata.tar"
self._POST_XMLNS(xmlns_buffer, schemata_file)
# delete the ones just populated (d,e,f)
manager.remove_schema(schema_2.id, remove_submissions = True)
manager.remove_schema(schema_3.id, remove_submissions = True)
# load data from sync file (d,e,f)
load_schemata(schemata_file, "127.0.0.1:8000")
try:
# verify that the schematas etc. count are correct (d,e,f)
self.assertEqual( starting_schemata_count, FormDefModel.objects.all().count())
self._assert_schema_registered("pf_followup.xsd", DATA_DIR)
self._assert_schema_registered("pf_new_reg.xsd", DATA_DIR)
self._assert_schema_registered("pf_ref_completed.xsd", DATA_DIR)
finally:
# clean up
manager = XFormManager()
manager.remove_schema(schema_1.id, remove_submissions = True)
self._delete_schema_from_filename("pf_new_reg.xsd", path = DATA_DIR)
self._delete_schema_from_filename("pf_ref_completed.xsd", path = DATA_DIR)
def testFromIdentical(self):
"""Tests the creation of a form group from two identical forms
(with different version numbers)."""
duplicate_formdef = create_xsd_and_populate("data/versioning/base.2.xsd")
forms = [self.original_formdef, duplicate_formdef]
dom = Domain.objects.all()[0]
group = FormDataGroup.from_forms(forms, dom)
self.assertEqual(2, len(group.forms.all()))
for form in group.forms.all():
self.assertTrue(form in forms)
columns = self.original_formdef.get_data_column_names()
self.assertEqual(len(columns), len(group.columns.all()))
for column in columns:
column_def = group.columns.get(name=column)
self.assertEqual(2, len(column_def.fields.all()))
for field in column_def.fields.all():
self.assertTrue(field.form in forms)
self.assertEqual(column, field.column_name)
def testBasicViews(self):
domain = Domain.objects.get(name='mockdomain')
formdef = create_xsd_and_populate("data/pf_followup.xsd", "data/pf_followup_1.xml", domain)
instance = populate("data/pf_followup_2.xml")
response = self.client.get('/xforms/')
self.assertNotContains(response,"Error", status_code=200)
self.assertNotContains(response,"Exception", status_code=200)
response = self.client.get('/xforms/register/')
self.assertNotContains(response,"Error", status_code=200)
self.assertNotContains(response,"Exception", status_code=200)
response = self.client.get('/xforms/reregister/mockdomain/')
self.assertNotContains(response,"Error", status_code=200)
self.assertNotContains(response,"Exception", status_code=200)
response = self.client.get('/xforms/remove/%s/' % formdef.id)
self.assertNotContains(response,"Error", status_code=200)
self.assertNotContains(response,"Exception", status_code=200)
response = self.client.get('/xforms/%s/submit/' % formdef.id)
self.assertNotContains(response,"Error", status_code=200)
self.assertNotContains(response,"Exception", status_code=200)
urls = [
'/xforms/show/%s/' % formdef.id,
'/xforms/show/%s/%s/' % (formdef.id, instance.id),
'/xforms/show/%s/%s/csv/' % (formdef.id, instance.id),
'/xforms/data/%s/delete/' % formdef.id,
'/xforms/data/%s/' % formdef.id,
'/xforms/data/%s/csv/' % formdef.id,
'/xforms/data/%s/xml/' % formdef.id,
]
for url in urls:
self._test_valid_and_permissions(url, self.authuser, self.unauthuser)
# format url variables like so:
# response = self.client.get('/api/xforms/',{'format':'json'})
manager = XFormManager()
manager.remove_schema(formdef.id)
def setUp(self):
clear_data()
self.original_formdef = create_xsd_and_populate("data/versioning/base.xsd")