def handle(self, *args, **kwargs): if args.__len__() < 2: raise CommandError(_(u"path(xform instances) username")) path = args[0] username = args[1] try: user = User.objects.get(username=username) except User.DoesNotExist: raise CommandError(_(u"Invalid username %s") % username) debug = False if debug: print (_(u"[Importing XForm Instances from %(path)s]\n") % {'path': path}) im_count = len(glob.glob(os.path.join(IMAGES_DIR, '*'))) instance_count = Instance.objects.count() print _(u"Before Parse:") print _(u" --> Images: %(nb)d") % {'nb': im_count} print (_(u" --> Instances: %(nb)d") % {'nb': Instance.objects.count()}) import_instances_from_zip(path, user) if debug: im_count2 = len(glob.glob(os.path.join(IMAGES_DIR, '*'))) print _(u"After Parse:") print _(u" --> Images: %(nb)d") % {'nb': im_count2} print (_(u" --> Instances: %(nb)d") % {'nb': Instance.objects.count()})
def test_importing_b1_and_b2(self): """ b1 and b2 are from the *same phone* at different times. (this might not be a realistic test) b1: 1 photo survey (completed) 1 simple survey (not marked complete) b2: 1 photo survey (duplicate, completed) 1 simple survey (marked as complete) """ # import from sd card initial_instance_count = Instance.objects.count() initial_image_count = images_count() import_instances_from_zip(os.path.join(DB_FIXTURES_PATH, "bulk_submission.zip"), self.user) instance_count = Instance.objects.count() image_count = images_count() # Images are not duplicated # TODO: Figure out how to get this test passing. self.assertEqual(image_count, initial_image_count + 2) # Instance count should have incremented # by 1 (or 2) based on the b1 & b2 data sets self.assertEqual(instance_count, initial_instance_count + 2)
def test_importing_b1_and_b2(self): """ b1 and b2 are from the *same phone* at different times. (this might not be a realistic test) b1: 1 photo survey (completed) 1 simple survey (not marked complete) b2: 1 photo survey (duplicate, completed) 1 simple survey (marked as complete) """ # import from sd card initial_instance_count = Instance.objects.count() initial_image_count = images_count() import_instances_from_zip( os.path.join(DB_FIXTURES_PATH, "bulk_submission.zip"), self.user) instance_count = Instance.objects.count() image_count = images_count() #Images are not duplicated # TODO: Figure out how to get this test passing. self.assertEqual(image_count, initial_image_count + 2) # Instance count should have incremented # by 1 (or 2) based on the b1 & b2 data sets self.assertEqual(instance_count, initial_instance_count + 2)
def handle(self, *args, **kwargs): if len(args) < 2: raise CommandError(_("Usage: <command> username file/path.")) username = args[0] path = args[1] try: user = User.objects.get(username=username) except User.DoesNotExist: raise CommandError(_("The specified user '%s' does not exist.") % username) # make sure path exists if not os.path.exists(path): raise CommandError(_("The specified path '%s' does not exist.") % path) for dir, subdirs, files in os.walk(path): # check if the dir has an odk directory if "odk" in subdirs: # dont walk further down this dir subdirs.remove("odk") self.stdout.write(_("Importing from dir %s..\n") % dir) (total_count, success_count, errors) = import_instances_from_path(dir, user) self.stdout.write(_("Total: %(total)d, Imported: %(imported)d, Errors: %(errors)s\n------------------------------\n") % {'total': total_count, 'imported': success_count, 'errors': errors}) for file in files: filepath = os.path.join(path, file) if os.path.isfile(filepath) and os.path.splitext(filepath)[1].lower() == ".zip": self.stdout.write(_("Importing from zip at %s..\n") % filepath) (total_count, success_count, errors) = import_instances_from_zip(filepath, user) self.stdout.write(_("Total: %(total)d, Imported: %(imported)d, Errors: %(errors)s\n------------------------------\n") % {'total': total_count, 'imported': success_count, 'errors': errors})
def handle(self, *args, **kwargs): path = args[0] debug = False if debug: print "[Importing XForm Instances from %s]\n" % path im_count = len(glob.glob(os.path.join(IMAGES_DIR, '*'))) instance_count = Instance.objects.count() print "Before Parse:" print " --> Images: %d" % im_count print " --> Instances: %d" % Instance.objects.count() import_instances_from_zip(path) if debug: im_count2 = len(glob.glob(os.path.join(IMAGES_DIR, '*'))) print "After Parse:" print " --> Images: %d" % im_count2 print " --> Instances: %d" % Instance.objects.count()
def bulksubmission(request, username): # puts it in a temp directory. # runs "import_tools(temp_directory)" # deletes try: posting_user = User.objects.get(username=username) except User.DoesNotExist: return HttpResponseBadRequest("User %s not found" % username) # request.FILES is a django.utils.datastructures.MultiValueDict # for each key we have a list of values temp_postfile = request.FILES.pop("zip_submission_file", []) if len(temp_postfile) == 1: postfile = temp_postfile[0] tempdir = tempfile.gettempdir() our_tfpath = os.path.join(tempdir, postfile.name) our_tempfile = open(our_tfpath, 'wb') our_tempfile.write(postfile.read()) our_tempfile.close() our_tf = open(our_tfpath, 'rb') count = import_instances_from_zip(our_tf, user=posting_user) os.remove(our_tfpath) response = HttpResponse("Your ODK submission was successful. %d surveys imported. Your user now has %d instances." % \ (count, posting_user.surveys.count())) response.status_code = 200 response['Location'] = request.build_absolute_uri(request.path) return response else: return HttpResponseBadRequest("There was a problem receiving your ODK submission. [Error: multiple submission files (?)]")
def handle(self, *args, **kwargs): path = args[0] debug = False if debug: print "[Importing XForm Instances from %s]\n" % path im_count = len(glob.glob(os.path.join(IMAGES_DIR, '*'))) instance_count = Instance.objects.count() print "Before Parse:" print " --> Images: %d" % im_count print " --> Instances: %d" % Instance.objects.count() import_instances_from_zip(path) if debug: im_count2 = len(glob.glob(os.path.join(IMAGES_DIR, '*'))) print "After Parse:" print " --> Images: %d" % im_count2 print " --> Instances: %d" % Instance.objects.count()
def test_badzipfile_import(self): total, success, errors = import_instances_from_zip( os.path.join(CUR_DIR, "Water_Translated_2011_03_10.xml"), self.user ) self.assertEqual(total, 0) self.assertEqual(success, 0) expected_errors = [u"File is not a zip file"] self.assertEqual(errors, expected_errors)
def test_badzipfile_import(self): total, success, errors = import_instances_from_zip( os.path.join(CUR_DIR, "Water_Translated_2011_03_10.xml"), self.user) self.assertEqual(total, 0) self.assertEqual(success, 0) expected_errors = [u'File is not a zip file'] self.assertEqual(errors, expected_errors)
def handle(self, *args, **kwargs): path = args[0] debug = False if debug: print (_(u"[Importing XForm Instances from %(path)s]\n") % {'path': path}) im_count = len(glob.glob(os.path.join(IMAGES_DIR, '*'))) instance_count = Instance.objects.count() print _(u"Before Parse:") print _(u" --> Images: %(nb)d") % {'nb': im_count} print (_(u" --> Instances: %(nb)d") % {'nb': Instance.objects.count()}) import_instances_from_zip(path) if debug: im_count2 = len(glob.glob(os.path.join(IMAGES_DIR, '*'))) print _(u"After Parse:") print _(u" --> Images: %(nb)d") % {'nb': im_count2} print (_(u" --> Instances: %(nb)d") % {'nb': Instance.objects.count()})
def bulksubmission(request, username): # puts it in a temp directory. # runs "import_tools(temp_directory)" # deletes posting_user = get_object_or_404(User, username=username) # request.FILES is a django.utils.datastructures.MultiValueDict # for each key we have a list of values try: temp_postfile = request.FILES.pop("zip_submission_file", []) except IOError: return HttpResponseBadRequest( _(u"There was a problem receiving your " u"ODK submission. [Error: IO Error " u"reading data]")) if len(temp_postfile) == 1: postfile = temp_postfile[0] tempdir = tempfile.gettempdir() our_tfpath = os.path.join(tempdir, postfile.name) our_tempfile = open(our_tfpath, 'wb') our_tempfile.write(postfile.read()) our_tempfile.close() our_tf = open(our_tfpath, 'rb') total_count, success_count, errors = \ import_instances_from_zip(our_tf, user=posting_user) # chose the try approach as suggested by the link below # http://stackoverflow.com/questions/82831 try: os.remove(our_tfpath) except IOError: # TODO: log this Exception somewhere pass json_msg = { 'message': _(u"Submission complete. Out of %(total)d " u"survey instances, %(success)d were imported, " u"(%(rejected)d were rejected as duplicates, " u"missing forms, etc.)") % { 'total': total_count, 'success': success_count, 'rejected': total_count - success_count }, 'errors': u"%d %s" % (len(errors), errors) } audit = {"bulk_submission_log": json_msg} audit_log(Actions.USER_BULK_SUBMISSION, request.user, posting_user, _("Made bulk submissions."), audit, request) response = HttpResponse(json.dumps(json_msg)) response.status_code = 200 response['Location'] = request.build_absolute_uri(request.path) return response else: return HttpResponseBadRequest( _(u"There was a problem receiving your" u" ODK submission. [Error: multiple " u"submission files (?)]"))
def bulksubmission(request, username): # puts it in a temp directory. # runs "import_tools(temp_directory)" # deletes posting_user = get_object_or_404(User, username=username) # request.FILES is a django.utils.datastructures.MultiValueDict # for each key we have a list of values try: temp_postfile = request.FILES.pop("zip_submission_file", []) except IOError: return HttpResponseBadRequest(_(u"There was a problem receiving your " u"ODK submission. [Error: IO Error " u"reading data]")) if len(temp_postfile) == 1: postfile = temp_postfile[0] tempdir = tempfile.gettempdir() our_tfpath = os.path.join(tempdir, postfile.name) our_tempfile = open(our_tfpath, 'wb') our_tempfile.write(postfile.read()) our_tempfile.close() our_tf = open(our_tfpath, 'rb') total_count, success_count, errors = \ import_instances_from_zip(our_tf, user=posting_user) # chose the try approach as suggested by the link below # http://stackoverflow.com/questions/82831 try: os.remove(our_tfpath) except IOError: # TODO: log this Exception somewhere pass json_msg = { 'message': _(u"Submission complete. Out of %(total)d " u"survey instances, %(success)d were imported, " u"(%(rejected)d were rejected as duplicates, " u"missing forms, etc.)") % {'total': total_count, 'success': success_count, 'rejected': total_count - success_count}, 'errors': u"%d %s" % (len(errors), errors) } audit = { "bulk_submission_log": json_msg } audit_log(Actions.USER_BULK_SUBMISSION, request.user, posting_user, _("Made bulk submissions."), audit, request) response = HttpResponse(json.dumps(json_msg)) response.status_code = 200 response['Location'] = request.build_absolute_uri(request.path) return response else: return HttpResponseBadRequest(_(u"There was a problem receiving your" u" ODK submission. [Error: multiple " u"submission files (?)]"))
def handle(self, *args, **kwargs): if len(args) < 2: raise CommandError(_("Usage: <command> username file/path.")) username = args[0] path = args[1] try: user = User.objects.get(username=username) except User.DoesNotExist: raise CommandError( _("The specified user '%s' does not exist.") % username) # make sure path exists if not os.path.exists(path): raise CommandError( _("The specified path '%s' does not exist.") % path) for dir, subdirs, files in os.walk(path): # check if the dir has an odk directory if "odk" in subdirs: # dont walk further down this dir subdirs.remove("odk") self.stdout.write(_("Importing from dir %s..\n") % dir) (total_count, success_count, errors) = import_instances_from_path(dir, user) self.stdout.write( _("Total: %(total)d, Imported: %(imported)d, Errors: %(errors)s\n------------------------------\n" ) % { 'total': total_count, 'imported': success_count, 'errors': errors }) for file in files: filepath = os.path.join(path, file) if os.path.isfile(filepath) and os.path.splitext( filepath)[1].lower() == ".zip": self.stdout.write( _("Importing from zip at %s..\n") % filepath) (total_count, success_count, errors) = import_instances_from_zip(filepath, user) self.stdout.write( _("Total: %(total)d, Imported: %(imported)d, Errors: %(errors)s\n------------------------------\n" ) % { 'total': total_count, 'imported': success_count, 'errors': errors })
def bulksubmission(request, username): # puts it in a temp directory. # runs "import_tools(temp_directory)" # deletes try: posting_user = User.objects.get(username=username) except User.DoesNotExist: return HttpResponseBadRequest(_(u"User %s not found") % username) # request.FILES is a django.utils.datastructures.MultiValueDict # for each key we have a list of values try: temp_postfile = request.FILES.pop("zip_submission_file", []) except IOError: return HttpResponseBadRequest(_(u"There was a problem receiving your " u"ODK submission. [Error: IO Error " u"reading data]")) if len(temp_postfile) == 1: postfile = temp_postfile[0] tempdir = tempfile.gettempdir() our_tfpath = os.path.join(tempdir, postfile.name) our_tempfile = open(our_tfpath, 'wb') our_tempfile.write(postfile.read()) our_tempfile.close() our_tf = open(our_tfpath, 'rb') total_count, success_count, errors = \ import_instances_from_zip(our_tf, user=posting_user) os.remove(our_tfpath) json_msg = { 'message': _(u"Submission successful. Out of %(total)d " u"survey instances, %(success)d were imported " u"(%(rejected)d were rejected--duplicates, " u"missing forms, etc.)") % {'total': total_count, 'success': success_count, 'rejected': total_count - success_count}, 'errors': u"%d %s" % (len(errors), errors) } response = HttpResponse(json.dumps(json_msg)) response.status_code = 200 response['Location'] = request.build_absolute_uri(request.path) return response else: return HttpResponseBadRequest(_(u"There was a problem receiving your" u" ODK submission. [Error: multiple " u"submission files (?)]"))
def bulksubmission(request, username): # puts it in a temp directory. # runs "import_tools(temp_directory)" # deletes try: posting_user = User.objects.get(username=username) except User.DoesNotExist: return HttpResponseBadRequest("User %s not found" % username) # request.FILES is a django.utils.datastructures.MultiValueDict # for each key we have a list of values try: temp_postfile = request.FILES.pop("zip_submission_file", []) except IOError: return HttpResponseBadRequest( "There was a problem receiving your ODK submission. [Error: IO Error reading data]" ) if len(temp_postfile) == 1: postfile = temp_postfile[0] tempdir = tempfile.gettempdir() our_tfpath = os.path.join(tempdir, postfile.name) our_tempfile = open(our_tfpath, 'wb') our_tempfile.write(postfile.read()) our_tempfile.close() our_tf = open(our_tfpath, 'rb') total_count, success_count, errors = import_instances_from_zip( our_tf, user=posting_user) os.remove(our_tfpath) json_msg = { 'message': "Submission successful. Out of %d survey instances, %d were imported (%d were rejected--duplicates, missing forms, etc.)" % \ (total_count, success_count, total_count - success_count), 'errors': "%d %s" % (len(errors), errors) } response = HttpResponse(json.dumps(json_msg)) response.status_code = 200 response['Location'] = request.build_absolute_uri(request.path) return response else: return HttpResponseBadRequest( "There was a problem receiving your ODK submission. [Error: multiple submission files (?)]" )