def test_sync_all_schemata(self):
""" Tests synchronizing all schemata from self (no xmlns posted) """
manager = XFormManager()
# load data
schema_1 = create_xsd_and_populate("pf_followup.xsd", path = DATA_DIR)
schema_2 = create_xsd_and_populate("pf_new_reg.xsd", path = DATA_DIR)
schema_3 = create_xsd_and_populate("pf_ref_completed.xsd", path = DATA_DIR)
starting_schemata_count = FormDefModel.objects.all().count()
# get sync file from self
schemata_file = "schemata.tar"
generate_schemata(serverhost, 'brian', 'test', latest=False, download=True, to=schemata_file)
manager.remove_schema(schema_1.id, remove_submissions = True)
manager.remove_schema(schema_2.id, remove_submissions = True)
manager.remove_schema(schema_3.id, remove_submissions = True)
# load data from sync file
load_schemata(schemata_file, "127.0.0.1:8000")
try:
# verify that the submissions etc. count are correct
self.assertEqual( starting_schemata_count, FormDefModel.objects.all().count())
finally:
# clean up
self._delete_schema_from_filename("pf_followup.xsd", path = DATA_DIR)
self._delete_schema_from_filename("pf_new_reg.xsd", path = DATA_DIR)
self._delete_schema_from_filename("pf_ref_completed.xsd", path = DATA_DIR)
def test_generate_debug_submissions(self):
""" Tests downloading some submissions from self
This is only useful to make sure that the test_load_diff_submissions test
below is working properly.
"""
#setup
schema_1 = create_xsd_and_populate("pf_followup.xsd", \
"pf_followup_1.xml", path = DATA_DIR)
populate("pf_followup_2.xml", path = DATA_DIR)
schema_2 = create_xsd_and_populate("pf_new_reg.xsd", \
"pf_new_reg_1.xml", path = DATA_DIR)
populate("pf_new_reg_2.xml", path = DATA_DIR)
schema_3 = create_xsd_and_populate("pf_ref_completed.xsd", \
"pf_ref_completed_1.xml", path = DATA_DIR)
populate("pf_ref_completed_2.xml", path = DATA_DIR)
# the 'debug' flag limits the generated MD5s to a count of 5
submissions_file = "submissions.tar"
# debug means we only post 5 submissions (instead of all)
generate_submissions(serverhost, 'brian', 'test', debug=True, download=True, to=submissions_file)
try:
self._assert_tar_count_equals(submissions_file, Submission.objects.all().count()-5)
# cleanup
finally:
# delete all data on self
manager = XFormManager()
manager.remove_schema(schema_1.id, remove_submissions = True)
manager.remove_schema(schema_2.id, remove_submissions = True)
manager.remove_schema(schema_3.id, remove_submissions = True)
def handle(self, *app_labels, **options):
form, domain = self.get_form_and_domain(**options)
manager = XFormManager()
if form:
print "Migrating Single form: %s." % form
manager.repost_schema(form)
return
if domain:
forms = FormDefModel.objects.filter(domain=domain)
else:
print "Migrating ALL forms."
forms = FormDefModel.objects.order_by("domain__name").all()
if not forms:
print "Whoops. Nothing to migrate. Are you sure you registered forms here?"
return
current_domain = forms[0].domain
print "Migrating forms in %s" % current_domain
for form in forms:
if form.domain != current_domain:
current_domain = form.domain
print "Migrating forms in %s" % current_domain
print "Migrating %s" % form
manager.repost_schema(form)
def test_sync_no_schemata(self):
""" Tests synchronizing no data from self (posts all MD5s) """
manager = XFormManager()
# load data
schema_1 = create_xsd_and_populate("pf_followup.xsd", path = DATA_DIR)
schema_2 = create_xsd_and_populate("pf_new_reg.xsd", path = DATA_DIR)
schema_3 = create_xsd_and_populate("pf_ref_completed.xsd", path = DATA_DIR)
# get sync file from self
schemata_file = 'schemata.tar'
generate_schemata(serverhost, 'brian', 'test', download=True, to=schemata_file)
# test that the received schemata file is empty
self._assert_tar_count_equals(schemata_file, 0)
starting_schemata_count = FormDefModel.objects.all().count()
load_schemata(schemata_file, "127.0.0.1:8000")
try:
# verify that no new schemata were loaded
self.assertEqual( starting_schemata_count, FormDefModel.objects.all().count())
finally:
# clean up
manager.remove_schema(schema_1.id, remove_submissions = True)
manager.remove_schema(schema_2.id, remove_submissions = True)
manager.remove_schema(schema_3.id, remove_submissions = True)
def test_generate_all_submissions(self):
""" Tests downloading all submissions from self """
# setup
schema_1 = create_xsd_and_populate("pf_followup.xsd", path = DATA_DIR)
submit_1 = populate("pf_followup_1.xml", path = DATA_DIR)
submit_2 = populate("pf_followup_2.xml", path = DATA_DIR)
schema_2 = create_xsd_and_populate("pf_new_reg.xsd", path = DATA_DIR)
submit_3 = populate("pf_new_reg_1.xml", path = DATA_DIR)
submit_4 = populate("pf_new_reg_2.xml", path = DATA_DIR)
schema_3 = create_xsd_and_populate("pf_ref_completed.xsd", path = DATA_DIR)
submit_5 = populate("pf_ref_completed_1.xml", path = DATA_DIR)
submit_6 = populate("pf_ref_completed_2.xml", path = DATA_DIR)
# download and check
submissions_file = "submissions.tar"
generate_submissions(serverhost, 'brian', 'test', latest=False, download=True, to=submissions_file)
try:
self._assert_tar_count_equals(submissions_file, Submission.objects.all().count())
# cleanup
finally:
# delete all data on self
manager = XFormManager()
submit_1.delete()
submit_2.delete()
submit_3.delete()
submit_4.delete()
submit_5.delete()
submit_6.delete()
manager.remove_schema(schema_1.id, remove_submissions = True)
manager.remove_schema(schema_2.id, remove_submissions = True)
manager.remove_schema(schema_3.id, remove_submissions = True)
def delete_data(request, formdef_id, template='confirm_multiple_delete.html'):
context = {}
form = get_object_or_404(FormDefModel, pk=formdef_id)
if request.method == "POST":
if 'instance' in request.POST:
request.session['xform_data'] = []
metadata = []
for i in request.POST.getlist('instance'):
# user has selected items and clicked 'delete'
# redirect to confirmation
if 'checked_'+ i in request.POST:
meta = Metadata.objects.get(formdefmodel=form, raw_data=int(i))
metadata.append(meta)
request.session['xform_data'].append(int(i))
context['xform_data'] = metadata
elif 'confirm_delete' in request.POST:
# user has confirmed deletion. Proceed.
xformmanager = XFormManager()
for id in request.session['xform_data']:
xformmanager.remove_data(formdef_id, id)
logging.debug("Instances %s of schema %s were deleted.", \
(unicode(request.session['xform_data']), formdef_id))
request.session['xform_data'] = None
return HttpResponseRedirect( reverse("xformmanager.views.data", \
args=[formdef_id]) )
else:
request.session['xform_data'] = None
context['form_name'] = form.form_display_name
context['formdef_id'] = formdef_id
return render_to_response(request, template, context)
def reregister_xform(request, domain_name, template='register_and_list_xforms.html'):
# registers an xform without having a user context, for
# server-generated submissions
context = {}
if request.method == 'POST':
# must add_schema to storage provide first since forms are dependent upon elements
try:
metadata = request.META
domain = Domain.objects.get(name=domain_name)
type = metadata["HTTP_SCHEMA_TYPE"]
schema = request.raw_post_data
xformmanager = XFormManager()
formdefmodel = xformmanager.add_schema_manually(schema, type, domain)
except IOError, e:
logging.error("xformmanager.manager: " + unicode(e) )
context['errors'] = "Could not convert xform to schema. Please verify correct xform format."
context['upload_form'] = RegisterXForm()
context['registered_forms'] = FormDefModel.objects.all().filter(domain__name=domain_name)
return render_to_response(request, template, context)
except Exception, e:
logging.error(e)
logging.error("Unable to write raw post data<br/>")
logging.error("Unable to write raw post data: Exception: " + unicode(sys.exc_info()[0]) + "<br/>")
logging.error("Unable to write raw post data: Traceback: " + unicode(sys.exc_info()[1]))
type, value, tb = sys.exc_info()
logging.error(unicode(type.__name__), ":", unicode(value))
logging.error("error parsing attachments: Traceback: " + '\n'.join(traceback.format_tb(tb)))
logging.error("Transaction rolled back")
context['errors'] = "Unable to write raw post data" + unicode(sys.exc_info()[0]) + unicode(sys.exc_info()[1])
transaction.rollback()
def testSaveFormData_8(self):
""" Test repeated form definition created and data saved """
formdefmodel = create_xsd_and_populate("data/8_singlerepeat_2.xsd")
self.assertEquals(int(formdefmodel.version), 2)
self.assertEquals(int(formdefmodel.uiversion), 3)
submission = populate("data/8_singlerepeat_2.xml")
m = Metadata.objects.get(attachment__submission=submission)
self.assertEquals(int(m.version), 2)
self.assertEquals(int(m.uiversion), 2)
cursor = connection.cursor()
cursor.execute("SELECT * FROM schema_xml_singlerepeat_2")
try:
row = cursor.fetchone()
self.assertEquals(row[9],"deviceid0")
self.assertEquals(row[10],"starttime")
self.assertEquals(row[11],"endtime")
cursor.execute("SELECT * FROM schema_xml_singlerepeat_root_userid_2")
row = cursor.fetchall()
self.assertEquals(row[0][1],"userid0")
self.assertEquals(row[1][1],"userid2")
self.assertEquals(row[2][1],"userid3")
self.assertEquals(row[0][2],1)
self.assertEquals(row[1][2],1)
self.assertEquals(row[2][2],1)
finally:
manager = XFormManager()
manager.remove_schema(formdefmodel.id)
def _delete_schema_from_filename(self, file_name, path):
schema = open(os.path.join(path, file_name), 'r')
formdef = FormDef(schema)
schema.close()
try:
formdef = FormDefModel.objects.get(target_namespace=formdef.target_namespace)
except FormDefModel.DoesNotExist:
return
manager = XFormManager()
manager.remove_schema(formdef.id, remove_submissions=True)
def testReSubmit(self):
# original submission
submission = populate("data/pf_followup_1.xml", domain=self.domain)
self.assertEquals(submission.is_orphaned(),True)
# register schema
create_xsd_and_populate("data/pf_followup.xsd", domain=self.domain)
# xformmanagger resubmission
xformmanager = XFormManager()
status = xformmanager.save_form_data(submission.xform)
self.assertEquals(status,True)
def dashboard(request, template='register_and_list_xforms.html'):
context = {}
if request.method == 'POST':
if 'confirm_register' in request.POST:
# user has already confirmed registration
# process saved file without bothering with validation
try:
formdefmodel = _register_xform(request,
request.session['schema_file'],
request.session['display_name'],
request.session['REMOTE_ADDR'],
request.session['file_size'],
request.session['xform_file'])
except Exception, e:
logging.error(unicode(e))
context['errors'] = unicode(e)
transaction.rollback()
else:
transaction.commit()
context['newsubmit'] = formdefmodel
else:
# validate and attempt to process schema
form = RegisterXForm(request.POST, request.FILES)
if form.is_valid():
xformmanager = XFormManager()
try:
xsd_file_name, xform_file_name = \
xformmanager.save_schema_POST_to_file(\
request.FILES['file'], request.FILES['file'].name
)
except Exception, e:
# typically this error is because we could not translate xform to schema
logging.error(unicode(e))
context['errors'] = unicode(e)
transaction.rollback()
else:
is_valid, exception = xformmanager.validate_schema(xsd_file_name)
if is_valid:
try:
formdefmodel = _register_xform(request, xsd_file_name, \
form.cleaned_data['form_display_name'], \
request.META['REMOTE_ADDR'], \
request.FILES['file'].size,
xform_file_name
)
except Exception, e:
logging.error(unicode(e))
context['errors'] = unicode(e)
transaction.rollback()
else:
transaction.commit()
context['newsubmit'] = formdefmodel
else:
def remove_xform(request, form_id=None, template='confirm_delete.html'):
context = {}
form = get_object_or_404(FormDefModel, pk=form_id)
if request.method == "POST":
if request.POST["confirm_delete"]: # The user has already confirmed the deletion.
xformmanager = XFormManager()
xformmanager.remove_schema(form_id)
logging.debug("Schema %s deleted ", form_id)
#self.message_user(request, _('The %(name)s "%(obj)s" was deleted successfully.') % {'name': force_unicode(opts.verbose_name), 'obj': force_unicode(obj_display)})
return HttpResponseRedirect(reverse('xformmanager.views.dashboard'))
context['form_name'] = form.form_display_name
return render_to_response(request, template, context)
def _testErrors(self, schema_file, instance_file, id):
su = SU()
xfm = XFM()
xsd_file_name = os.path.join(os.path.dirname(__file__),schema_file)
xml_file_name = os.path.join(os.path.dirname(__file__),instance_file)
schema = xfm._add_schema_from_file(xsd_file_name)
formdef = su.get_formdef_from_schema_file(xsd_file_name)
data_tree = su._get_data_tree_from_file(xml_file_name)
populator = XFormDBTablePopulator( formdef, schema )
queries = populator.populate( data_tree )
xfm.remove_schema(schema.id)
return populator.errors
def testRepostingPreservesRelations(self):
"""Testing reposting entire schemas with linked objects."""
form = create_xsd_and_populate("data/pf_followup.xsd", "data/pf_followup_1.xml", self.domain)
# add some linked objects - one and two levels deep, and track them
clear_case_data()
fid1 = FormIdentifier.objects.create(form=form, identity_column="id")
fid2 = FormIdentifier.objects.create(form=form, identity_column="meta_username")
case = Case.objects.create(name="somecase", domain=self.domain)
cfid11 = CaseFormIdentifier.objects.create(form_identifier=fid1, case=case,
sequence_id=1, form_type="follow")
cfid12 = CaseFormIdentifier.objects.create(form_identifier=fid1, case=case,
sequence_id=2, form_type="follow")
cfid13 = CaseFormIdentifier.objects.create(form_identifier=fid2, case=case,
sequence_id=3, form_type="follow")
self.assertEqual(3, CaseFormIdentifier.objects.count())
self.assertEqual(2, FormIdentifier.objects.count())
self.assertEqual(1, Case.objects.count())
# repost
manager = XFormManager()
new_form = manager.repost_schema(form)
self.assertEqual(1, FormDefModel.objects.count())
# make sure the counts are still correct
self.assertEqual(3, CaseFormIdentifier.objects.count())
self.assertEqual(2, FormIdentifier.objects.count())
self.assertEqual(1, Case.objects.count())
# load up the new objects by unique charateristics (NOT ids)
newfid1 = FormIdentifier.objects.get(identity_column="id")
newfid2 = FormIdentifier.objects.get(identity_column="meta_username")
newcfid11 = CaseFormIdentifier.objects.get(sequence_id=1)
newcfid12 = CaseFormIdentifier.objects.get(sequence_id=2)
newcfid13 = CaseFormIdentifier.objects.get(sequence_id=3)
# make sure relationships are the same (with new objs) but not ids
self.assertEqual(new_form, newfid1.form)
self.assertEqual(new_form, newfid2.form)
self.assertNotEqual(fid1.id, newfid1.id)
self.assertNotEqual(fid2.id, newfid2.id)
self.assertEqual(newcfid11.form_identifier, newfid1)
self.assertEqual(newcfid11.case, case)
self.assertEqual(newcfid12.form_identifier, newfid1)
self.assertEqual(newcfid12.case, case)
self.assertEqual(newcfid13.form_identifier, newfid2)
self.assertEqual(newcfid13.case, case)
self.assertNotEqual(newcfid11.id, cfid11.id)
self.assertNotEqual(newcfid12.id, cfid12.id)
self.assertNotEqual(newcfid13.id, cfid13.id)
def _register_xform(request, file_name, display_name, remote_addr, file_size, xform_file_name):
""" does the actual creation and saving of the formdef model """
xformmanager = XFormManager()
formdefmodel = xformmanager.create_schema_from_file(file_name,
request.user.selected_domain,
xform_file_name)
formdefmodel.submit_ip = remote_addr
formdefmodel.bytes_received = file_size
formdefmodel.form_display_name = display_name
formdefmodel.uploaded_by = request.user
formdefmodel.domain = request.user.selected_domain
formdefmodel.save()
logging.debug("xform registered")
return formdefmodel
def testSaveFormData_7(self):
""" Test very deep form definition created and data saved """
formdefmodel = create_xsd_and_populate("data/7_verydeep_2.xsd", "data/7_verydeep_2.xml")
cursor = connection.cursor()
cursor.execute("SELECT * FROM schema_xml_verydeep_1")
row = cursor.fetchone()
try:
self.assertEquals(row[9],"userid0")
self.assertEquals(row[10],"great_grand1")
self.assertEquals(row[11],222)
self.assertEquals(row[12],1159)
self.assertEquals(row[13],2002)
finally:
manager = XFormManager()
manager.remove_schema(formdefmodel.id)
def test_sync_weird_submissions(self):
""" Tests synchronizing some data from self (posts a few MD5s) """
# setup - if we don't do this, we just get back "no submissions found"
manager = XFormManager()
# populate some files
schema_1 = create_xsd_and_populate("pf_followup.xsd", "pf_followup_1.xml", path = DATA_DIR)
schema_2 = create_xsd_and_populate("pf_new_reg.xsd", "pf_new_reg_1.xml", path = DATA_DIR)
schema_3 = create_xsd_and_populate("pf_ref_completed.xsd", "pf_ref_completed_1.xml", path = DATA_DIR)
submissions_count = Submission.objects.count()
url = 'http://%s/api/submissions/' % (serverhost)
up = urlparse(url)
conn = httplib.HTTPConnection(up.netloc)
# test posting junk md5
MD5_buffer = "sadfndan;ofansdn"
conn.request('POST', up.path, MD5_buffer, {'Content-Type': 'application/bz2', 'User-Agent': 'CCHQ-submitfromfile-python-v0.1'})
response = conn.getresponse().read()
self.assertTrue( response.lower().find('poorly formatted') != -1 )
# test posting non-existent md5s
md5 = "e402f026c762a6bc999f9f2703efd367\n"
bz2_md5 = bz2.compress(md5)
submissions_file = "submissions.tar"
self._POST_MD5s(bz2_md5, submissions_file)
# should get the same 3 schemas we registered above
self._assert_tar_count_equals(submissions_file, submissions_count)
# test posting duplicate md5s
string = cStringIO.StringIO()
submits = Submission.objects.all().order_by('checksum')[:2]
for submit in submits:
string.write(unicode( submit.checksum ) + '\n')
string.write(unicode( submit.checksum ) + '\n')
MD5s = string.getvalue()
dupe_buffer = bz2.compress(MD5s)
submissions_file = "submissions.tar"
self._POST_MD5s(dupe_buffer, submissions_file)
self._assert_tar_count_equals(submissions_file, submissions_count-2)
manager.remove_schema(schema_1.id, remove_submissions = True)
manager.remove_schema(schema_2.id, remove_submissions = True)
manager.remove_schema(schema_3.id, remove_submissions = True)
def test_sync_weird_schemata(self):
""" Tests synchronizing some data from self (posts a few MD5s) """
# setup - if we don't do this, we just get back "no submissions found"
manager = XFormManager()
# populate some files
starting_schemata_count = FormDefModel.objects.count()
schema_1 = create_xsd_and_populate("pf_followup.xsd", path = DATA_DIR)
schema_2 = create_xsd_and_populate("pf_new_reg.xsd", path = DATA_DIR)
schema_3 = create_xsd_and_populate("pf_ref_completed.xsd", path = DATA_DIR)
url = 'http://%s/api/xforms/?format=sync' % (serverhost)
up = urlparse(url)
conn = httplib.HTTPConnection(up.netloc)
# test posting junk namespace
namespace_buffer = "sadfndan;ofansdn"
conn.request('POST', up.path, namespace_buffer, {'Content-Type': 'application/bz2', 'User-Agent': 'CCHQ-submitfromfile-python-v0.1'})
response = conn.getresponse().read()
self.assertTrue( response.lower().find('poorly formatted') != -1 )
# test posting non-existent namespaces
namespace = "http://zilch.com"
bz2_namespace = bz2.compress(namespace)
schemata_file = "schemata.tar"
self._POST_XMLNS(bz2_namespace, schemata_file)
# should get all the schemas back
self._assert_tar_count_equals(schemata_file, starting_schemata_count+3)
# test posting duplicate namespaces
string = cStringIO.StringIO()
formdefs = FormDefModel.objects.all().order_by('target_namespace')[:2]
for formdef in formdefs:
string.write(unicode( formdef.target_namespace ) + '\n')
string.write(unicode( formdef.target_namespace ) + '\n')
dupe_buffer = bz2.compress(string.getvalue())
self._POST_XMLNS(dupe_buffer, schemata_file)
self._assert_tar_count_equals(schemata_file, starting_schemata_count+1)
manager.remove_schema(schema_1.id, remove_submissions = True)
manager.remove_schema(schema_2.id, remove_submissions = True)
manager.remove_schema(schema_3.id, remove_submissions = True)
def testBasicViews(self):
domain = Domain.objects.get(name='mockdomain')
formdef = create_xsd_and_populate("data/pf_followup.xsd", "data/pf_followup_1.xml", domain)
instance = populate("data/pf_followup_2.xml")
response = self.client.get('/xforms/')
self.assertNotContains(response,"Error", status_code=200)
self.assertNotContains(response,"Exception", status_code=200)
response = self.client.get('/xforms/register/')
self.assertNotContains(response,"Error", status_code=200)
self.assertNotContains(response,"Exception", status_code=200)
response = self.client.get('/xforms/reregister/mockdomain/')
self.assertNotContains(response,"Error", status_code=200)
self.assertNotContains(response,"Exception", status_code=200)
response = self.client.get('/xforms/remove/%s/' % formdef.id)
self.assertNotContains(response,"Error", status_code=200)
self.assertNotContains(response,"Exception", status_code=200)
response = self.client.get('/xforms/%s/submit/' % formdef.id)
self.assertNotContains(response,"Error", status_code=200)
self.assertNotContains(response,"Exception", status_code=200)
urls = [
'/xforms/show/%s/' % formdef.id,
'/xforms/show/%s/%s/' % (formdef.id, instance.id),
'/xforms/show/%s/%s/csv/' % (formdef.id, instance.id),
'/xforms/data/%s/delete/' % formdef.id,
'/xforms/data/%s/' % formdef.id,
'/xforms/data/%s/csv/' % formdef.id,
'/xforms/data/%s/xml/' % formdef.id,
]
for url in urls:
self._test_valid_and_permissions(url, self.authuser, self.unauthuser)
# format url variables like so:
# response = self.client.get('/api/xforms/',{'format':'json'})
manager = XFormManager()
manager.remove_schema(formdef.id)
def testSaveFormData_10(self):
""" Test nested repeated form definition created and data saved """
formdefmodel = create_xsd_and_populate("6_nestedrepeats.xsd", "6_nestedrepeats.xml")
formdefmodel_2 = create_xsd_and_populate("data/9_nestedrepeats_2.xsd", "data/9_nestedrepeats_2.xml")
formdefmodel_3 = create_xsd_and_populate("data/10_other_v3.xsd", "data/10_other_v3.xml")
try:
cursor = connection.cursor()
cursor.execute("SELECT * FROM schema_xml_nestedrepeats")
row = cursor.fetchone()
self.assertEquals(row[9],"foo")
self.assertEquals(row[10],"bar")
self.assertEquals(row[11],"yes")
self.assertEquals(row[12],"no")
cursor.execute("SELECT * FROM schema_xml_nestedrepeats_root_nested")
row = cursor.fetchall()
self.assertEquals(row[0][1],"userid0")
self.assertEquals(row[0][2],"deviceid0")
if settings.DATABASE_ENGINE=='mysql' :
self.assertEquals(row[0][3],datetime(2009,10,9,11,4,30) )
self.assertEquals(row[0][4],datetime(2009,10,9,11,9,30) )
else:
self.assertEquals(row[0][3],"2009-10-9 11:04:30" )
self.assertEquals(row[0][4],"2009-10-9 11:09:30" )
self.assertEquals(row[0][5],1)
self.assertEquals(row[1][1],"userid2")
self.assertEquals(row[1][2],"deviceid2")
if settings.DATABASE_ENGINE=='mysql' :
self.assertEquals(row[1][3],datetime(2009,11,12,11,11,11) )
self.assertEquals(row[1][4],datetime(2009,11,12,11,16,11) )
else:
self.assertEquals(row[1][3],"2009-11-12 11:11:11" )
self.assertEquals(row[1][4],"2009-11-12 11:16:11" )
self.assertEquals(row[1][5],1)
finally:
manager = XFormManager()
manager.remove_schema(formdefmodel.id)
manager.remove_schema(formdefmodel_2.id)
manager.remove_schema(formdefmodel_3.id)
def handle(self, *app_labels, **options):
form, domain = self.get_form_and_domain(**options)
manager = XFormManager()
if form:
print "Checking form %s" % form
errors, warnings = manager.check_schema(form)
self.display(form, errors, warnings)
return
if domain:
forms = FormDefModel.objects.filter(domain=domain)
else:
forms = FormDefModel.objects.order_by("domain__name").all()
current_domain = forms[0].domain
print "Checking forms in %s" % current_domain
for form in forms:
if form.domain != current_domain:
current_domain = form.domain
print "Checking forms in %s" % current_domain
errors, warnings = manager.check_schema(form)
self.display(form, errors, warnings)
def testRepostingPreservesData(self):
"""Testing reposting entire schemas."""
form = create_xsd_and_populate("data/pf_followup.xsd", "data/pf_followup_1.xml", self.domain)
original_id = form.id
populate("data/pf_followup_2.xml", self.domain)
# sanity checks to make sure things are behaving normally
self.assertEqual(1, FormDefModel.objects.count())
self.assertEqual(2, Metadata.objects.count())
original_meta_ids = [meta.id for meta in Metadata.objects.all()]
cursor = connection.cursor()
cursor.execute("SELECT count(*) FROM schema_basicdomain_pathfinder_pathfinder_cc_follow_0_0_2a")
self.assertEqual(2, cursor.fetchone()[0])
# repost - numbers should be the same, but the ids should all be different
manager = XFormManager()
new_form = manager.repost_schema(form)
self.assertEqual(1, FormDefModel.objects.count())
# compare basic form properties
for field in FormDefModel._meta.fields:
if field.name not in ("id", "xsd_file_location", "element"):
self.assertEqual(getattr(form, field.name), getattr(new_form, field.name),
"Field %s was not successfully migrated. Previous value: %s, new value: %s" \
% (field.name, getattr(form, field.name), getattr(new_form, field.name)))
# check metadata
self.assertEqual(2, Metadata.objects.count())
self.assertNotEqual(original_id, new_form.id)
for meta in Metadata.objects.all():
self.assertFalse(meta.id in original_meta_ids)
# check parsed data
cursor = connection.cursor()
cursor.execute("SELECT count(*) FROM schema_basicdomain_pathfinder_pathfinder_cc_follow_0_0_2a")
self.assertEqual(2, cursor.fetchone()[0])
def test_sync_dupe_submissions(self):
""" Tests synchronizing duplicate data from self"""
manager = XFormManager()
# populate some files
schema_1 = create_xsd_and_populate("pf_followup.xsd", "pf_followup_1.xml", path = DATA_DIR)
schema_2 = create_xsd_and_populate("pf_new_reg.xsd", "pf_new_reg_1.xml", path = DATA_DIR)
schema_3 = create_xsd_and_populate("pf_ref_completed.xsd", "pf_ref_completed_1.xml", path = DATA_DIR)
starting_submissions_count = Submission.objects.all().count()
# <STATE 1/>
# get MD5 of 3 populated files
MD5_buffer = rest_util.get_field_as_bz2(Submission, 'checksum')
# add 3 dupes and 1 new file
submit_1 = populate("pf_followup_1.xml", path = DATA_DIR)
submit_2 = populate("pf_new_reg_1.xml", path = DATA_DIR)
submit_3 = populate("pf_ref_completed_1.xml", path = DATA_DIR)
# <STATE 2/>
submissions_file = "submissions.tar"
self._POST_MD5s(MD5_buffer, submissions_file)
self._assert_tar_count_equals(submissions_file, 0)
submit_4 = populate("pf_ref_completed_3.xml", path = DATA_DIR)
# <STATE 3/>
# get the difference between state 1 and state 3
self._POST_MD5s(MD5_buffer, submissions_file)
# save checksum and delete the ones just populated
checksum_4 = submit_4.checksum
submit_1.delete()
submit_2.delete()
submit_3.delete()
submit_4.delete()
# should get the same 3 schemas we registered above
self._assert_tar_count_equals(submissions_file, 1)
# load data from sync file (d,e,f)
load_submissions(submissions_file, "127.0.0.1:8000")
try:
# verify that we only have 4 submissions
self.assertEqual( starting_submissions_count+1, Submission.objects.all().count() )
Submission.objects.get(checksum=checksum_4)
except Submission.DoesNotExist:
self.fail("Incorrect submission received")
finally:
# clean up
manager = XFormManager()
manager.remove_schema(schema_1.id, remove_submissions = True)
manager.remove_schema(schema_2.id, remove_submissions = True)
manager.remove_schema(schema_3.id, remove_submissions = True)
def test_sync_some_schemata(self):
""" Tests synchronizing some schemata from self (posts a few xmlns) """
manager = XFormManager()
# populate some files
schema_1 = create_xsd_and_populate("pf_followup.xsd", path = DATA_DIR)
# get xmlns of populated schemas
xmlns_buffer = rest_util.get_field_as_bz2(FormDefModel, 'target_namespace')
# populate a few more schema
schema_2 = create_xsd_and_populate("pf_new_reg.xsd", path = DATA_DIR)
schema_3 = create_xsd_and_populate("pf_ref_completed.xsd", path = DATA_DIR)
starting_schemata_count = FormDefModel.objects.all().count()
# get the difference between the first schema and current state
schemata_file = "schemata.tar"
self._POST_XMLNS(xmlns_buffer, schemata_file)
# delete the ones just populated (d,e,f)
manager.remove_schema(schema_2.id, remove_submissions = True)
manager.remove_schema(schema_3.id, remove_submissions = True)
# load data from sync file (d,e,f)
load_schemata(schemata_file, "127.0.0.1:8000")
try:
# verify that the schematas etc. count are correct (d,e,f)
self.assertEqual( starting_schemata_count, FormDefModel.objects.all().count())
self._assert_schema_registered("pf_followup.xsd", DATA_DIR)
self._assert_schema_registered("pf_new_reg.xsd", DATA_DIR)
self._assert_schema_registered("pf_ref_completed.xsd", DATA_DIR)
finally:
# clean up
manager = XFormManager()
manager.remove_schema(schema_1.id, remove_submissions = True)
self._delete_schema_from_filename("pf_new_reg.xsd", path = DATA_DIR)
self._delete_schema_from_filename("pf_ref_completed.xsd", path = DATA_DIR)
def tearDown(self):
manager = XFormManager()
manager.remove_schema(1)
to_skip.append(form)
else:
raise BuildError("""Schema %s is not compatible with %s.
Because of the following differences:
%s
You must update your version number!"""
% (existing_formdef, formdef, differences))
else:
# this must be registered
to_register.append(form)
except Exception, e:
errors.append(e)
if errors:
raise BuildError("Problem validating xforms for %s!" % self, errors)
# finally register
manager = XFormManager()
# TODO: we need transaction management
for form in to_register:
try:
formdefmodel = manager.add_schema(form.get_file_name(),
form.as_filestream())
upload_info = self.upload_information
if upload_info:
formdefmodel.submit_ip = upload_info.ip
user = upload_info.user
else:
formdefmodel.submit_ip = UNKNOWN_IP
user = self.uploaded_by
if user:
try:
to_skip.append(form)
else:
raise BuildError("""Schema %s is not compatible with %s.
Because of the following differences:
%s
You must update your version number!"""
% (existing_formdef, formdef, differences))
else:
# this must be registered
to_register.append(form)
except Exception, e:
errors.append(e)
if errors:
raise BuildError("Problem validating xforms for %s!" % self, errors)
# finally register
manager = XFormManager()
# TODO: we need transaction management
for form in to_register:
try:
formdefmodel = manager.add_schema(form.get_file_name(),
form.as_filestream(),
self.project.domain)
upload_info = self.upload_information
if upload_info:
formdefmodel.submit_ip = upload_info.ip
user = upload_info.user
else:
formdefmodel.submit_ip = UNKNOWN_IP
user = self.uploaded_by
formdefmodel.uploaded_by = user
def test_sync_some_submissions(self):
""" Tests synchronizing some data from self (posts a few MD5s) """
manager = XFormManager()
# populate some files
schema_1 = create_xsd_and_populate("pf_followup.xsd", "pf_followup_1.xml", path = DATA_DIR)
schema_2 = create_xsd_and_populate("pf_new_reg.xsd", "pf_new_reg_1.xml", path = DATA_DIR)
schema_3 = create_xsd_and_populate("pf_ref_completed.xsd", "pf_ref_completed_1.xml", path = DATA_DIR)
# get MD5 of all current submissions
MD5_buffer = rest_util.get_field_as_bz2(Submission, 'checksum')
# populate a few more files
submit_1 = populate("pf_followup_2.xml", path = DATA_DIR)
submit_2 = populate("pf_new_reg_2.xml", path = DATA_DIR)
submit_3 = populate("pf_ref_completed_2.xml", path = DATA_DIR)
submit_4 = populate("pf_ref_completed_3.xml", path = DATA_DIR)
starting_submissions_count = Submission.objects.all().count()
starting_schemata_count = FormDefModel.objects.all().count()
# get the difference between the first 3 files and the current
# set of files (i.e. the last 4 files)
submissions_file = "submissions.tar"
self._POST_MD5s(MD5_buffer, submissions_file)
# save checksums and delete the ones just populated (d,e,f)
checksums = [ submit_1.checksum, submit_2.checksum, submit_3.checksum, submit_3.checksum ]
manager.remove_data(schema_1.id, Metadata.objects.get(attachment=submit_1.xform).raw_data, \
remove_submission = True)
manager.remove_data(schema_2.id, Metadata.objects.get(attachment=submit_2.xform).raw_data, \
remove_submission = True)
manager.remove_data(schema_3.id, Metadata.objects.get(attachment=submit_3.xform).raw_data, \
remove_submission = True)
manager.remove_data(schema_3.id, Metadata.objects.get(attachment=submit_4.xform).raw_data, \
remove_submission = True)
# load data from sync file (d,e,f)
load_submissions(submissions_file, "127.0.0.1:8000")
try:
# verify that the submissions etc. count are correct (d,e,f)
self.assertEqual( starting_submissions_count, Submission.objects.all().count())
submits = Submission.objects.all().order_by('-submit_time')[:4]
# verify that the correct submissions were loaded
Submission.objects.get(checksum=checksums[0])
Submission.objects.get(checksum=checksums[1])
Submission.objects.get(checksum=checksums[2])
Submission.objects.get(checksum=checksums[3])
except Submission.DoesNotExist:
self.fail("Incorrect submission received")
finally:
# clean up
manager = XFormManager()
manager.remove_schema(schema_1.id, remove_submissions = True)
manager.remove_schema(schema_2.id, remove_submissions = True)
manager.remove_schema(schema_3.id, remove_submissions = True)