def generate_schemata(remote_url, username, password, latest=True, download=False, to="schemata.tar", debug=False):
""" Generate sync data from remote server
remote_url: url of remote server (ip:port)
username, password: credentials for logging in
"""
status = rest_util.login(remote_url, username, password)
if not status:
print "Sorry. Your credentials were not accepted."
sys.exit()
url = 'http://%s/api/xforms/?format=sync' % remote_url
if latest:
xmlns_buffer = rest_util.get_field_as_bz2(FormDefModel, 'target_namespace', debug)
print "Generating latest remote schemata..."
else:
xmlns_buffer = ''
print "Generating all remote submissions archive..."
response = rest_util.request(url, username, password, xmlns_buffer)
print "Generated"
if download:
fout = open(to, 'w+b')
fout.write(response.read())
fout.close()
print "Schemata downloaded to %s" % to
return response
def generate_submissions(remote_url, username, password, latest=True, debug=False, download=False, to='submissions.tar'):
""" Generate sync data from remote server
remote_url: url of remote server (ip:port)
username, password: credentials for logging in
"""
status = rest_util.login(remote_url, username, password)
if not status:
print "Sorry. Your credentials were not accepted."
sys.exit()
url = 'http://%s/api/submissions/' % remote_url
if latest:
MD5_buffer = rest_util.get_field_as_bz2(Submission, 'checksum', debug)
response = rest_util.request(url, username, password, MD5_buffer)
print "Generated latest remote submissions"
else:
response = urllib2.urlopen(url)
print "Generated all remote submissions archive"
if download:
fout = open(to, 'w+b')
fout.write(response.read())
fout.close()
print "Submissions downloaded to %s" % to
else:
# Check for status messages
# (i think tar payloads always begin 'BZ'...)
response = response.read(255)
if response[:2] != "BZ":
print response
return response
def test_sync_some_submissions(self):
""" Tests synchronizing some data from self (posts a few MD5s) """
manager = XFormManager()
# populate some files
schema_1 = create_xsd_and_populate("pf_followup.xsd", "pf_followup_1.xml", path = DATA_DIR)
schema_2 = create_xsd_and_populate("pf_new_reg.xsd", "pf_new_reg_1.xml", path = DATA_DIR)
schema_3 = create_xsd_and_populate("pf_ref_completed.xsd", "pf_ref_completed_1.xml", path = DATA_DIR)
# get MD5 of all current submissions
MD5_buffer = rest_util.get_field_as_bz2(Submission, 'checksum')
# populate a few more files
submit_1 = populate("pf_followup_2.xml", path = DATA_DIR)
submit_2 = populate("pf_new_reg_2.xml", path = DATA_DIR)
submit_3 = populate("pf_ref_completed_2.xml", path = DATA_DIR)
submit_4 = populate("pf_ref_completed_3.xml", path = DATA_DIR)
starting_submissions_count = Submission.objects.all().count()
starting_schemata_count = FormDefModel.objects.all().count()
# get the difference between the first 3 files and the current
# set of files (i.e. the last 4 files)
submissions_file = "submissions.tar"
self._POST_MD5s(MD5_buffer, submissions_file)
# save checksums and delete the ones just populated (d,e,f)
checksums = [ submit_1.checksum, submit_2.checksum, submit_3.checksum, submit_3.checksum ]
manager.remove_data(schema_1.id, Metadata.objects.get(attachment=submit_1.xform).raw_data, \
remove_submission = True)
manager.remove_data(schema_2.id, Metadata.objects.get(attachment=submit_2.xform).raw_data, \
remove_submission = True)
manager.remove_data(schema_3.id, Metadata.objects.get(attachment=submit_3.xform).raw_data, \
remove_submission = True)
manager.remove_data(schema_3.id, Metadata.objects.get(attachment=submit_4.xform).raw_data, \
remove_submission = True)
# load data from sync file (d,e,f)
load_submissions(submissions_file, "127.0.0.1:8000")
try:
# verify that the submissions etc. count are correct (d,e,f)
self.assertEqual( starting_submissions_count, Submission.objects.all().count())
submits = Submission.objects.all().order_by('-submit_time')[:4]
# verify that the correct submissions were loaded
Submission.objects.get(checksum=checksums[0])
Submission.objects.get(checksum=checksums[1])
Submission.objects.get(checksum=checksums[2])
Submission.objects.get(checksum=checksums[3])
except Submission.DoesNotExist:
self.fail("Incorrect submission received")
finally:
# clean up
manager = XFormManager()
manager.remove_schema(schema_1.id, remove_submissions = True)
manager.remove_schema(schema_2.id, remove_submissions = True)
manager.remove_schema(schema_3.id, remove_submissions = True)
def test_sync_dupe_submissions(self):
""" Tests synchronizing duplicate data from self"""
manager = XFormManager()
# populate some files
schema_1 = create_xsd_and_populate("pf_followup.xsd", "pf_followup_1.xml", path = DATA_DIR)
schema_2 = create_xsd_and_populate("pf_new_reg.xsd", "pf_new_reg_1.xml", path = DATA_DIR)
schema_3 = create_xsd_and_populate("pf_ref_completed.xsd", "pf_ref_completed_1.xml", path = DATA_DIR)
starting_submissions_count = Submission.objects.all().count()
# <STATE 1/>
# get MD5 of 3 populated files
MD5_buffer = rest_util.get_field_as_bz2(Submission, 'checksum')
# add 3 dupes and 1 new file
submit_1 = populate("pf_followup_1.xml", path = DATA_DIR)
submit_2 = populate("pf_new_reg_1.xml", path = DATA_DIR)
submit_3 = populate("pf_ref_completed_1.xml", path = DATA_DIR)
# <STATE 2/>
submissions_file = "submissions.tar"
self._POST_MD5s(MD5_buffer, submissions_file)
self._assert_tar_count_equals(submissions_file, 0)
submit_4 = populate("pf_ref_completed_3.xml", path = DATA_DIR)
# <STATE 3/>
# get the difference between state 1 and state 3
self._POST_MD5s(MD5_buffer, submissions_file)
# save checksum and delete the ones just populated
checksum_4 = submit_4.checksum
submit_1.delete()
submit_2.delete()
submit_3.delete()
submit_4.delete()
# should get the same 3 schemas we registered above
self._assert_tar_count_equals(submissions_file, 1)
# load data from sync file (d,e,f)
load_submissions(submissions_file, "127.0.0.1:8000")
try:
# verify that we only have 4 submissions
self.assertEqual( starting_submissions_count+1, Submission.objects.all().count() )
Submission.objects.get(checksum=checksum_4)
except Submission.DoesNotExist:
self.fail("Incorrect submission received")
finally:
# clean up
manager = XFormManager()
manager.remove_schema(schema_1.id, remove_submissions = True)
manager.remove_schema(schema_2.id, remove_submissions = True)
manager.remove_schema(schema_3.id, remove_submissions = True)
def test_sync_some_schemata(self):
""" Tests synchronizing some schemata from self (posts a few xmlns) """
manager = XFormManager()
# populate some files
schema_1 = create_xsd_and_populate("pf_followup.xsd", path = DATA_DIR)
# get xmlns of populated schemas
xmlns_buffer = rest_util.get_field_as_bz2(FormDefModel, 'target_namespace')
# populate a few more schema
schema_2 = create_xsd_and_populate("pf_new_reg.xsd", path = DATA_DIR)
schema_3 = create_xsd_and_populate("pf_ref_completed.xsd", path = DATA_DIR)
starting_schemata_count = FormDefModel.objects.all().count()
# get the difference between the first schema and current state
schemata_file = "schemata.tar"
self._POST_XMLNS(xmlns_buffer, schemata_file)
# delete the ones just populated (d,e,f)
manager.remove_schema(schema_2.id, remove_submissions = True)
manager.remove_schema(schema_3.id, remove_submissions = True)
# load data from sync file (d,e,f)
load_schemata(schemata_file, "127.0.0.1:8000")
try:
# verify that the schematas etc. count are correct (d,e,f)
self.assertEqual( starting_schemata_count, FormDefModel.objects.all().count())
self._assert_schema_registered("pf_followup.xsd", DATA_DIR)
self._assert_schema_registered("pf_new_reg.xsd", DATA_DIR)
self._assert_schema_registered("pf_ref_completed.xsd", DATA_DIR)
finally:
# clean up
manager = XFormManager()
manager.remove_schema(schema_1.id, remove_submissions = True)
self._delete_schema_from_filename("pf_new_reg.xsd", path = DATA_DIR)
self._delete_schema_from_filename("pf_ref_completed.xsd", path = DATA_DIR)