def handle(self, *args, **kwargs): for i, pi in enumerate(queryset_iterator(ParsedInstance.objects.all())): pi.update_mongo() if (i + 1) % 1000 == 0: print (_('Updated %(nb)d records, flushing MongoDB...') % {'nb': i}) settings._MONGO_CONNECTION.admin.command({'fsync': 1})
def create_zip_backup(zip_output_file, user, xform): # create a temp dir that we;ll create our structure within and zip it # when we are done tmp_dir_path = tempfile.mkdtemp() instances_path = os.path.join(tmp_dir_path, "instances") # get the xls file from storage # for each submission in the database - create an xml file in this # form # /<id_string>/YYYY/MM/DD/YYYY-MM-DD-HH-MM-SS.xml qs = Instance.objects.filter(xform=xform) num_instances = qs.count() done = 0 sys.stdout.write("Creating XML Instances\n") for instance in queryset_iterator(qs, 100): # get submission time date_time_str = instance.date_created.strftime(DATE_FORMAT) date_parts = date_time_str.split("-") sub_dirs = os.path.join(*date_parts[:3]) # create the directories full_path = os.path.join(instances_path, sub_dirs) if not os.path.exists(full_path): try: os.makedirs(full_path) except OSError as e: if e.errno != errno.EEXIST: raise full_xml_path = os.path.join(full_path, date_time_str + ".xml") # check for duplicate file names file_index = 1 while os.path.exists(full_xml_path): full_xml_path = os.path.join(full_path, "%s-%d.xml" % (date_time_str, file_index)) file_index += 1 # create the instance xml with codecs.open(full_xml_path, "wb", "utf-8") as f: f.write(instance.xml) done += 1 sys.stdout.write("\r%.2f %% done" % (float(done) / float(num_instances) * 100)) sys.stdout.flush() sleep(0) # write zip file sys.stdout.write("\nWriting to ZIP arhive.\n") zf = zipfile.ZipFile(zip_output_file, "w") done = 0 for dir_path, dir_names, file_names in os.walk(tmp_dir_path): for file_name in file_names: archive_path = dir_path.replace(tmp_dir_path + os.path.sep, "", 1) zf.write(os.path.join(dir_path, file_name), os.path.join(archive_path, file_name)) done += 1 sys.stdout.write("\r%.2f %% done" % (float(done) / float(num_instances) * 100)) sys.stdout.flush() sleep(0) zf.close() # removed dir tree shutil.rmtree(tmp_dir_path) sys.stdout.write("\nBackup saved to %s\n" % zip_output_file)
def handle(self, *args, **kwargs): for i, pi in enumerate(queryset_iterator( ParsedInstance.objects.all())): pi.update_mongo() if (i + 1) % 1000 == 0: print 'Updated %d records, flushing MongoDB...' % i settings._MONGO_CONNECTION.admin.command({'fsync': 1})
def handle(self, *args, **kwargs): print "%d XForms to update" % DataDictionary.objects.count() for i, dd in enumerate(queryset_iterator(DataDictionary.objects.all())): if dd.xls: dd._set_uuid_in_xml() super(DataDictionary, dd).save() if (i + 1) % 10 == 0: print "Updated %d XForms..." % i
def handle(self, *args, **kwargs): print (_('%(nb)d XForms to update') % {'nb': DataDictionary.objects.count()}) for i, dd in enumerate(queryset_iterator(DataDictionary.objects.all())): if dd.xls: dd._set_uuid_in_xml() super(DataDictionary, dd).save() if (i + 1) % 10 == 0: print _('Updated %(nb)d XForms...') % {'nb': i}
def mongo_sync_status(remongo=False, update_all=False, user=None, xform=None): qs = XForm.objects.only('id_string').select_related('user') if user and not xform: qs = qs.filter(user=user) elif user and xform: qs = qs.filter(user=user, id_string=xform.id_string) else: qs = qs.all() total = qs.count() found = 0 done = 0 total_to_remongo = 0 report_string = "" for xform in queryset_iterator(qs, 100): # get the count user = xform.user instance_count = Instance.objects.filter(xform=xform).count() userform_id = "%s_%s" % (user.username, xform.id_string) mongo_count = mongo_instances.find( {common_tags.USERFORM_ID: userform_id}).count() if instance_count != mongo_count or update_all: line = "user: %s, id_string: %s\nInstance count: %d\t"\ "Mongo count: %d\n---------------------------------"\ "-----\n" % ( user.username, xform.id_string, instance_count, mongo_count) report_string += line found += 1 total_to_remongo += (instance_count - mongo_count) # should we remongo if remongo or (remongo and update_all): if update_all: sys.stdout.write( "Updating all records for %s\n--------------------" "---------------------------\n" % xform.id_string) else: sys.stdout.write( "Updating missing records for %s\n----------------" "-------------------------------\n" % xform.id_string) update_mongo_for_xform( xform, only_update_missing=not update_all) done += 1 sys.stdout.write( "%.2f %% done ...\r" % ((float(done) / float(total)) * 100)) # only show stats if we are not updating mongo, the update function # will show progress if not remongo: line = "Total # of forms out of sync: %d\n" \ "Total # of records to remongo: %d\n" % ( found, total_to_remongo) report_string += line return report_string
def mongo_sync_status(remongo=False, update_all=False, user=None, xform=None): qs = XForm.objects.only('id_string', 'user').select_related('user') if user and not xform: qs = qs.filter(user=user) elif user and xform: qs = qs.filter(user=user, id_string=xform.id_string) else: qs = qs.all() total = qs.count() found = 0 done = 0 total_to_remongo = 0 report_string = "" for xform in queryset_iterator(qs, 100): # get the count user = xform.user instance_count = Instance.objects.filter(xform=xform).count() userform_id = "%s_%s" % (user.username, xform.id_string) mongo_count = mongo_instances.find({ common_tags.USERFORM_ID: userform_id }).count() if instance_count != mongo_count or update_all: line = "user: %s, id_string: %s\nInstance count: %d\t"\ "Mongo count: %d\n---------------------------------"\ "-----\n" % ( user.username, xform.id_string, instance_count, mongo_count) report_string += line found += 1 total_to_remongo += (instance_count - mongo_count) # should we remongo if remongo or (remongo and update_all): if update_all: sys.stdout.write( "Updating all records for %s\n--------------------" "---------------------------\n" % xform.id_string) else: sys.stdout.write( "Updating missing records for %s\n----------------" "-------------------------------\n" % xform.id_string) update_mongo_for_xform(xform, only_update_missing=not update_all) done += 1 sys.stdout.write("%.2f %% done ...\r" % ((float(done) / float(total)) * 100)) # only show stats if we are not updating mongo, the update function # will show progress if not remongo: line = "Total # of forms out of sync: %d\n" \ "Total # of records to remongo: %d\n" % ( found, total_to_remongo) report_string += line return report_string
def handle(self, *args, **kwargs): attachments_qs = Attachment.objects.select_related( 'instance', 'instance__xform') if kwargs.get('username'): username = kwargs.get('username') try: user = User.objects.get(username=username) except User.DoesNotExist: raise CommandError( "Error: username %(username)s does not exist" % {'username': username}) attachments_qs = attachments_qs.filter(instance__user=user) if kwargs.get('id_string'): id_string = kwargs.get('id_string') try: xform = XForm.objects.get(id_string=id_string) except XForm.DoesNotExist: raise CommandError( "Error: Form with id_string %(id_string)s does not exist" % {'id_string': id_string}) attachments_qs = attachments_qs.filter(instance__xform=xform) fs = get_storage_class('django.core.files.storage.FileSystemStorage')() for att in queryset_iterator(attachments_qs): filename = att.media_file.name default_storage = get_storage_class()() full_path = get_path(filename, settings.THUMB_CONF['small']['suffix']) if kwargs.get('force') is not None: for s in ['small', 'medium', 'large']: fp = get_path(filename, settings.THUMB_CONF[s]['suffix']) if default_storage.exists(fp): default_storage.delete(fp) if not default_storage.exists(full_path): try: if default_storage.__class__ != fs.__class__: resize(filename) else: resize_local_env(filename) if default_storage.exists( get_path( filename, '%s' % settings.THUMB_CONF['small']['suffix'])): print( _(u'Thumbnails created for %(file)s') % {'file': filename}) else: print( _(u'Problem with the file %(file)s') % {'file': filename}) except (IOError, OSError), e: print _(u'Error on %(filename)s: %(error)s') \ % {'filename': filename, 'error': e}
def handle(self, *args, **kwargs): attachments_qs = Attachment.objects.select_related( 'instance', 'instance__xform') if kwargs.get('username'): username = kwargs.get('username') try: user = User.objects.get(username=username) except User.DoesNotExist: raise CommandError( "Error: username %(username)s does not exist" % {'username': username} ) attachments_qs = attachments_qs.filter(instance__user=user) if kwargs.get('id_string'): id_string = kwargs.get('id_string') try: xform = XForm.objects.get(id_string=id_string) except XForm.DoesNotExist: raise CommandError( "Error: Form with id_string %(id_string)s does not exist" % {'id_string': id_string} ) attachments_qs = attachments_qs.filter(instance__xform=xform) fs = get_storage_class('django.core.files.storage.FileSystemStorage')() for att in queryset_iterator(attachments_qs): filename = att.media_file.name default_storage = get_storage_class()() full_path = get_path(filename, settings.THUMB_CONF['small']['suffix']) if kwargs.get('force') is not None: for s in ['small', 'medium', 'large']: fp = get_path(filename, settings.THUMB_CONF[s]['suffix']) if default_storage.exists(fp): default_storage.delete(fp) if not default_storage.exists(full_path): try: if default_storage.__class__ != fs.__class__: resize(filename) else: resize_local_env(filename) if default_storage.exists(get_path( filename, '%s' % settings.THUMB_CONF['small']['suffix'])): print (_(u'Thumbnails created for %(file)s') % {'file': filename}) else: print (_(u'Problem with the file %(file)s') % {'file': filename}) except (IOError, OSError), e: print _(u'Error on %(filename)s: %(error)s') \ % {'filename': filename, 'error': e}
def handle(self, *args, **kwargs): xforms = XForm.objects.all() total = xforms.count() count = 0 for xform in queryset_iterator(XForm.objects.all()): has_geo = ParsedInstance.objects.filter( instance__xform=xform, lat__isnull=False).count() > 0 try: xform.surveys_with_geopoints = has_geo xform.save() except Exception as e: print e else: count += 1 print "%d of %d forms processed." % (count, total)
def handle(self, *args, **kwargs): fs = get_storage_class('django.core.files.storage.FileSystemStorage')() for att in queryset_iterator(Attachment.objects.select_related( 'instance', 'instance__xform').all()): filename = att.media_file.name default_storage = get_storage_class()() if not default_storage.exists(get_path(filename, settings.THUMB_CONF['smaller']['suffix'])): try: if default_storage.__class__ != fs.__class__: resize(filename) else: resize_local_env(filename) if default_storage.exists(get_path(filename, settings.THUMB_CONF['smaller']['suffix'])): print 'Thumbnails created for %s' % filename else: print 'Something didn\'t go right for %s' % filename except (IOError, OSError), e: print 'Error on %s: %s' % (filename, e)
def handle(self, *args, **kwargs): fs = get_storage_class('django.core.files.storage.FileSystemStorage')() for att in queryset_iterator(Attachment.objects.select_related( 'instance', 'instance__xform').all()): filename = att.media_file.name default_storage = get_storage_class()() if not default_storage.exists(get_path(filename, settings.THUMB_CONF['smaller']['suffix'])): try: if default_storage.__class__ != fs.__class__: resize(filename) else: resize_local_env(filename) if default_storage.exists(get_path(filename, '%s.%s' % (settings.THUMB_CONF['smaller']['suffix'], settings.IMG_FILE_TYPE))): print _(u'Thumbnails created for %s') % filename else: print _(u'Problem with the file %s') % filename except (IOError, OSError), e: print _(u'Error on %(filename)s: %(error)s') \ % {'filename': filename, 'error': e}
def handle(self, *args, **kwargs): fs = get_storage_class('django.core.files.storage.FileSystemStorage')() for att in queryset_iterator( Attachment.objects.select_related('instance', 'instance__xform').all()): filename = att.media_file.name default_storage = get_storage_class()() if not default_storage.exists( get_path(filename, settings.THUMB_CONF['smaller']['suffix'])): try: if default_storage.__class__ != fs.__class__: resize(filename) else: resize_local_env(filename) if default_storage.exists( get_path( filename, settings.THUMB_CONF['smaller']['suffix'])): print 'Thumbnails created for %s' % filename else: print 'Something didn\'t go right for %s' % filename except (IOError, OSError), e: print 'Error on %s: %s' % (filename, e)
def dicts(cls, xform): qs = cls.objects.filter(instance__xform=xform) for parsed_instance in queryset_iterator(qs): yield parsed_instance.to_dict()
def handle(self, *args, **kwargs): user = xform = None if len(args) > 0: username = args[0] try: user = User.objects.get(username=username) except User.DoesNotExist: raise CommandError("User %s does not exist" % username) if len(args) > 1: id_string = args[1] try: xform = XForm.objects.get(user=user, id_string=id_string) except XForm.DoesNotExist: raise CommandError( "Xform %s does not exist for user %s" %\ (id_string, user.username)) remongo = kwargs["remongo"] update_all = kwargs["update_all"] qs = XForm.objects.only('id_string').select_related('user') if user and not xform: qs = qs.filter(user=user) elif user and xform: qs = qs.filter(user=user, id_string=xform.id_string) else: qs = qs.all() total = qs.count() found = 0 done = 0 total_to_remongo = 0 for xform in queryset_iterator(qs, 100): # get the count user = xform.user instance_count = Instance.objects.filter(xform=xform).count() userform_id = "%s_%s" % (user.username, xform.id_string) mongo_count = xform_instances.find( {"_userform_id": userform_id}).count() if instance_count != mongo_count or update_all: line = "user: %s, id_string: %s\nInstance count: %d\t" \ "Mongo count: %d\n---------------------------------" \ "-----\n" % ( user.username, xform.id_string, instance_count, mongo_count) self.stdout.write(line) found += 1 total_to_remongo += (instance_count - mongo_count) # should we remongo if remongo or (remongo and update_all): if update_all: self.stdout.write( "Updating all records for %s\n--------------------" "---------------------------\n" % xform.id_string) else: self.stdout.write( "Updating missing records for %s\n----------------" "-------------------------------\n" %\ xform.id_string) update_mongo_for_xform( xform, only_update_missing=not update_all) done += 1 self.stdout.write( "%.2f %% done ...\r" % ((float(done)/float(total)) * 100)) # only show stats if we are not updating mongo, the update function # will show progress if not remongo: line = "Total Forms out of sync: %d\nTotal to remongo: %d\n" % ( found, total_to_remongo) self.stdout.write(line)
def get_list_of_parsed_instances(self, flat=True): for i in queryset_iterator(self.surveys_for_export(self)): # TODO: there is information we want to add in parsed xforms. yield i.get_dict(flat=flat)
def mongo_sync_status(remongo=False, update_all=False, user=None, xform=None): """Check the status of records in the mysql db versus mongodb. At a minimum, return a report (string) of the results. Optionally, take action to correct the differences, based on these parameters, if present and defined: remongo -> if True, update the records missing in mongodb (default: False) update_all -> if True, update all the relevant records (default: False) user -> if specified, apply only to the forms for the given user (default: None) xform -> if specified, apply only to the given form (default: None) """ qs = XForm.objects.only('id_string', 'user').select_related('user') if user and not xform: qs = qs.filter(user=user) elif user and xform: qs = qs.filter(user=user, id_string=xform.id_string) else: qs = qs.all() total = qs.count() found = 0 done = 0 total_to_remongo = 0 report_string = "" for xform in queryset_iterator(qs, 100): # get the count user = xform.user instance_count = Instance.objects.filter(xform=xform).count() userform_id = "%s_%s" % (user.username, xform.id_string) mongo_count = mongo_instances.find({ common_tags.USERFORM_ID: userform_id }).count() if instance_count != mongo_count or update_all: line = "user: %s, id_string: %s\nInstance count: %d\t"\ "Mongo count: %d\n---------------------------------"\ "-----\n" % ( user.username, xform.id_string, instance_count, mongo_count) report_string += line found += 1 total_to_remongo += (instance_count - mongo_count) # should we remongo if remongo or (remongo and update_all): if update_all: sys.stdout.write( "Updating all records for %s\n--------------------" "---------------------------\n" % xform.id_string) else: sys.stdout.write( "Updating missing records for %s\n----------------" "-------------------------------\n" % xform.id_string) update_mongo_for_xform(xform, only_update_missing=not update_all) done += 1 sys.stdout.write("%.2f %% done ...\r" % ((float(done) / float(total)) * 100)) # only show stats if we are not updating mongo, the update function # will show progress if not remongo: line = "Total # of forms out of sync: %d\n" \ "Total # of records to remongo: %d\n" % ( found, total_to_remongo) report_string += line return report_string
def create_zip_backup(zip_output_file, user, xform): # create a temp dir that we;ll create our structure within and zip it # when we are done tmp_dir_path = tempfile.mkdtemp() instances_path = os.path.join(tmp_dir_path, "instances") # get the xls file from storage # for each submission in the database - create an xml file in this # form # /<id_string>/YYYY/MM/DD/YYYY-MM-DD-HH-MM-SS.xml qs = Instance.objects.filter(xform=xform) num_instances = qs.count() done = 0 sys.stdout.write("Creating XML Instances\n") for instance in queryset_iterator(qs, 100): # get submission time date_time_str = instance.date_created.strftime(DATE_FORMAT) date_parts = date_time_str.split("-") sub_dirs = os.path.join(*date_parts[:3]) # create the directories full_path = os.path.join(instances_path, sub_dirs) if not os.path.exists(full_path): try: os.makedirs(full_path) except OSError as e: if e.errno != errno.EEXIST: raise full_xml_path = os.path.join(full_path, date_time_str + ".xml") # check for duplicate file names file_index = 1 while os.path.exists(full_xml_path): full_xml_path = os.path.join( full_path, "%s-%d.xml" % (date_time_str, file_index)) file_index += 1 # create the instance xml with codecs.open(full_xml_path, "wb", "utf-8") as f: f.write(instance.xml) done += 1 sys.stdout.write("\r%.2f %% done" % (float(done) / float(num_instances) * 100)) sys.stdout.flush() sleep(0) # write zip file sys.stdout.write("\nWriting to ZIP arhive.\n") zf = zipfile.ZipFile(zip_output_file, "w") done = 0 for dir_path, dir_names, file_names in os.walk(tmp_dir_path): for file_name in file_names: archive_path = dir_path.replace(tmp_dir_path + os.path.sep, "", 1) zf.write(os.path.join(dir_path, file_name), os.path.join(archive_path, file_name)) done += 1 sys.stdout.write("\r%.2f %% done" % (float(done) / float(num_instances) * 100)) sys.stdout.flush() sleep(0) zf.close() # removed dir tree shutil.rmtree(tmp_dir_path) sys.stdout.write("\nBackup saved to %s\n" % zip_output_file)
def mongo_sync_status(remongo=False, update_all=False, user=None, xform=None): """Check the status of records in the mysql db versus mongodb. At a minimum, return a report (string) of the results. Optionally, take action to correct the differences, based on these parameters, if present and defined: remongo -> if True, update the records missing in mongodb (default: False) update_all -> if True, update all the relevant records (default: False) user -> if specified, apply only to the forms for the given user (default: None) xform -> if specified, apply only to the given form (default: None) """ qs = XForm.objects.only('id_string', 'user').select_related('user') if user and not xform: qs = qs.filter(user=user) elif user and xform: qs = qs.filter(user=user, id_string=xform.id_string) else: qs = qs.all() total = qs.count() found = 0 done = 0 total_to_remongo = 0 report_string = "" for xform in queryset_iterator(qs, 100): # get the count user = xform.user instance_count = Instance.objects.filter(xform=xform).count() userform_id = "%s_%s" % (user.username, xform.id_string) mongo_count = mongo_instances.find( {common_tags.USERFORM_ID: userform_id}).count() if instance_count != mongo_count or update_all: line = "user: %s, id_string: %s\nInstance count: %d\t"\ "Mongo count: %d\n---------------------------------"\ "-----\n" % ( user.username, xform.id_string, instance_count, mongo_count) report_string += line found += 1 total_to_remongo += (instance_count - mongo_count) # should we remongo if remongo or (remongo and update_all): if update_all: sys.stdout.write( "Updating all records for %s\n--------------------" "---------------------------\n" % xform.id_string) else: sys.stdout.write( "Updating missing records for %s\n----------------" "-------------------------------\n" % xform.id_string) update_mongo_for_xform( xform, only_update_missing=not update_all) done += 1 sys.stdout.write( "%.2f %% done ...\r" % ((float(done) / float(total)) * 100)) # only show stats if we are not updating mongo, the update function # will show progress if not remongo: line = "Total # of forms out of sync: %d\n" \ "Total # of records to remongo: %d\n" % ( found, total_to_remongo) report_string += line return report_string