def generate_schemata(remote_url, username, password, latest=True, download=False, to="schemata.tar", debug=False): """ Generate sync data from remote server remote_url: url of remote server (ip:port) username, password: credentials for logging in """ status = rest_util.login(remote_url, username, password) if not status: print "Sorry. Your credentials were not accepted." sys.exit() url = 'http://%s/api/xforms/?format=sync' % remote_url if latest: xmlns_buffer = rest_util.get_field_as_bz2(FormDefModel, 'target_namespace', debug) print "Generating latest remote schemata..." else: xmlns_buffer = '' print "Generating all remote submissions archive..." response = rest_util.request(url, username, password, xmlns_buffer) print "Generated" if download: fout = open(to, 'w+b') fout.write(response.read()) fout.close() print "Schemata downloaded to %s" % to return response
def generate_submissions(remote_url, username, password, latest=True, debug=False, download=False, to='submissions.tar'): """ Generate sync data from remote server remote_url: url of remote server (ip:port) username, password: credentials for logging in """ status = rest_util.login(remote_url, username, password) if not status: print "Sorry. Your credentials were not accepted." sys.exit() url = 'http://%s/api/submissions/' % remote_url if latest: MD5_buffer = rest_util.get_field_as_bz2(Submission, 'checksum', debug) response = rest_util.request(url, username, password, MD5_buffer) print "Generated latest remote submissions" else: response = urllib2.urlopen(url) print "Generated all remote submissions archive" if download: fout = open(to, 'w+b') fout.write(response.read()) fout.close() print "Submissions downloaded to %s" % to else: # Check for status messages # (i think tar payloads always begin 'BZ'...) response = response.read(255) if response[:2] != "BZ": print response return response
def test_sync_some_submissions(self): """ Tests synchronizing some data from self (posts a few MD5s) """ manager = XFormManager() # populate some files schema_1 = create_xsd_and_populate("pf_followup.xsd", "pf_followup_1.xml", path = DATA_DIR) schema_2 = create_xsd_and_populate("pf_new_reg.xsd", "pf_new_reg_1.xml", path = DATA_DIR) schema_3 = create_xsd_and_populate("pf_ref_completed.xsd", "pf_ref_completed_1.xml", path = DATA_DIR) # get MD5 of all current submissions MD5_buffer = rest_util.get_field_as_bz2(Submission, 'checksum') # populate a few more files submit_1 = populate("pf_followup_2.xml", path = DATA_DIR) submit_2 = populate("pf_new_reg_2.xml", path = DATA_DIR) submit_3 = populate("pf_ref_completed_2.xml", path = DATA_DIR) submit_4 = populate("pf_ref_completed_3.xml", path = DATA_DIR) starting_submissions_count = Submission.objects.all().count() starting_schemata_count = FormDefModel.objects.all().count() # get the difference between the first 3 files and the current # set of files (i.e. the last 4 files) submissions_file = "submissions.tar" self._POST_MD5s(MD5_buffer, submissions_file) # save checksums and delete the ones just populated (d,e,f) checksums = [ submit_1.checksum, submit_2.checksum, submit_3.checksum, submit_3.checksum ] manager.remove_data(schema_1.id, Metadata.objects.get(attachment=submit_1.xform).raw_data, \ remove_submission = True) manager.remove_data(schema_2.id, Metadata.objects.get(attachment=submit_2.xform).raw_data, \ remove_submission = True) manager.remove_data(schema_3.id, Metadata.objects.get(attachment=submit_3.xform).raw_data, \ remove_submission = True) manager.remove_data(schema_3.id, Metadata.objects.get(attachment=submit_4.xform).raw_data, \ remove_submission = True) # load data from sync file (d,e,f) load_submissions(submissions_file, "127.0.0.1:8000") try: # verify that the submissions etc. count are correct (d,e,f) self.assertEqual( starting_submissions_count, Submission.objects.all().count()) submits = Submission.objects.all().order_by('-submit_time')[:4] # verify that the correct submissions were loaded Submission.objects.get(checksum=checksums[0]) Submission.objects.get(checksum=checksums[1]) Submission.objects.get(checksum=checksums[2]) Submission.objects.get(checksum=checksums[3]) except Submission.DoesNotExist: self.fail("Incorrect submission received") finally: # clean up manager = XFormManager() manager.remove_schema(schema_1.id, remove_submissions = True) manager.remove_schema(schema_2.id, remove_submissions = True) manager.remove_schema(schema_3.id, remove_submissions = True)
def test_sync_dupe_submissions(self): """ Tests synchronizing duplicate data from self""" manager = XFormManager() # populate some files schema_1 = create_xsd_and_populate("pf_followup.xsd", "pf_followup_1.xml", path = DATA_DIR) schema_2 = create_xsd_and_populate("pf_new_reg.xsd", "pf_new_reg_1.xml", path = DATA_DIR) schema_3 = create_xsd_and_populate("pf_ref_completed.xsd", "pf_ref_completed_1.xml", path = DATA_DIR) starting_submissions_count = Submission.objects.all().count() # <STATE 1/> # get MD5 of 3 populated files MD5_buffer = rest_util.get_field_as_bz2(Submission, 'checksum') # add 3 dupes and 1 new file submit_1 = populate("pf_followup_1.xml", path = DATA_DIR) submit_2 = populate("pf_new_reg_1.xml", path = DATA_DIR) submit_3 = populate("pf_ref_completed_1.xml", path = DATA_DIR) # <STATE 2/> submissions_file = "submissions.tar" self._POST_MD5s(MD5_buffer, submissions_file) self._assert_tar_count_equals(submissions_file, 0) submit_4 = populate("pf_ref_completed_3.xml", path = DATA_DIR) # <STATE 3/> # get the difference between state 1 and state 3 self._POST_MD5s(MD5_buffer, submissions_file) # save checksum and delete the ones just populated checksum_4 = submit_4.checksum submit_1.delete() submit_2.delete() submit_3.delete() submit_4.delete() # should get the same 3 schemas we registered above self._assert_tar_count_equals(submissions_file, 1) # load data from sync file (d,e,f) load_submissions(submissions_file, "127.0.0.1:8000") try: # verify that we only have 4 submissions self.assertEqual( starting_submissions_count+1, Submission.objects.all().count() ) Submission.objects.get(checksum=checksum_4) except Submission.DoesNotExist: self.fail("Incorrect submission received") finally: # clean up manager = XFormManager() manager.remove_schema(schema_1.id, remove_submissions = True) manager.remove_schema(schema_2.id, remove_submissions = True) manager.remove_schema(schema_3.id, remove_submissions = True)
def test_sync_some_schemata(self): """ Tests synchronizing some schemata from self (posts a few xmlns) """ manager = XFormManager() # populate some files schema_1 = create_xsd_and_populate("pf_followup.xsd", path = DATA_DIR) # get xmlns of populated schemas xmlns_buffer = rest_util.get_field_as_bz2(FormDefModel, 'target_namespace') # populate a few more schema schema_2 = create_xsd_and_populate("pf_new_reg.xsd", path = DATA_DIR) schema_3 = create_xsd_and_populate("pf_ref_completed.xsd", path = DATA_DIR) starting_schemata_count = FormDefModel.objects.all().count() # get the difference between the first schema and current state schemata_file = "schemata.tar" self._POST_XMLNS(xmlns_buffer, schemata_file) # delete the ones just populated (d,e,f) manager.remove_schema(schema_2.id, remove_submissions = True) manager.remove_schema(schema_3.id, remove_submissions = True) # load data from sync file (d,e,f) load_schemata(schemata_file, "127.0.0.1:8000") try: # verify that the schematas etc. count are correct (d,e,f) self.assertEqual( starting_schemata_count, FormDefModel.objects.all().count()) self._assert_schema_registered("pf_followup.xsd", DATA_DIR) self._assert_schema_registered("pf_new_reg.xsd", DATA_DIR) self._assert_schema_registered("pf_ref_completed.xsd", DATA_DIR) finally: # clean up manager = XFormManager() manager.remove_schema(schema_1.id, remove_submissions = True) self._delete_schema_from_filename("pf_new_reg.xsd", path = DATA_DIR) self._delete_schema_from_filename("pf_ref_completed.xsd", path = DATA_DIR)