def handle(self, *args, **kwargs): xforms = XForm.objects.filter(instances_with_osm=True) # username if args: xforms = xforms.filter(user__username=args[0]) for xform in queryset_iterator(xforms): attachments = Attachment.objects.filter( extension=Attachment.OSM, instance__xform=xform).distinct('instance') count = attachments.count() c = 0 for a in queryset_iterator(attachments): pk = a.instance.parsed_instance.pk save_osm_data(pk) c += 1 if c % 1000 == 0: self.stdout.write( "%s:%s: Processed %s of %s." % (xform.user.username, xform.id_string, c, count)) self.stdout.write("%s:%s: processed %s of %s submissions." % (xform.user.username, xform.id_string, c, count))
def handle(self, *args, **options): deleted = 0 self.stdout.write("Starting UserObject") for perm in queryset_iterator( UserObjectPermission.objects.select_related()): try: perm.content_object except AttributeError: perm.delete() deleted += 1 self.stdout.write( "deleted {} stale permission".format(deleted) ) self.stdout.write("Starting GroupObject") for perm in queryset_iterator( GroupObjectPermission.objects.select_related()): try: perm.content_object except AttributeError: perm.delete() deleted += 1 self.stdout.write( "deleted {} stale permission".format(deleted) ) self.stdout.write( "Total removed orphan object permissions instances: %d" % deleted )
def handle(self, *args, **options): self.stdout.write("Re-assigining started", ending='\n') if not args: raise CommandError('Param not set. <app model [created_perm]>') if len(args) < 3: raise CommandError('Param not set. <app model [created_perm]>') app = args[0] model = args[1] username = args[2] new_perms = list(args[3:]) if username == "all": users = User.objects.exclude( username__iexact=settings.ANONYMOUS_DEFAULT_USERNAME ) teams = Team.objects.all() else: users = User.objects.filter(username=username) teams = Team.objects.filter(organization__username=username) # Get all the users for user in queryset_iterator(users): self.reassign_perms(user, app, model, new_perms) for team in queryset_iterator(teams): self.reassign_perms(team, app, model, new_perms) self.stdout.write("Re-assigining finished", ending='\n')
def handle(self, *args, **options): # XForms for xform in queryset_iterator(XForm.objects.all()): OwnerRole.add(xform.user, xform) # UserProfile for profile in queryset_iterator(UserProfile.objects.all()): set_api_permissions_for_user(profile.user) OwnerRole.add(profile.user, profile) if profile.created_by is not None: OwnerRole.add(profile.created_by, profile) # OrganizationProfile for profile in queryset_iterator(OrganizationProfile.objects.all()): OwnerRole.add(profile.user, profile) if profile.created_by is not None: OwnerRole.add(profile.created_by, profile) if profile.creator is not None: OwnerRole.add(profile.creator, profile) # Project for project in queryset_iterator(Project.objects.all()): OwnerRole.add(project.organization, project) OwnerRole.add(project.created_by, project)
def handle(self, *args, **kwargs): xforms = XForm.objects.filter(instances_with_osm=True) # username if args: xforms = xforms.filter(user__username=args[0]) for xform in queryset_iterator(xforms): attachments = Attachment.objects.filter( extension=Attachment.OSM, instance__xform=xform ).distinct('instance') count = attachments.count() c = 0 for a in queryset_iterator(attachments): pk = a.instance.parsed_instance.pk save_osm_data(pk) c += 1 if c % 1000 == 0: self.stdout.write("%s:%s: Processed %s of %s." % (xform.user.username, xform.id_string, c, count)) self.stdout.write("%s:%s: processed %s of %s submissions." % (xform.user.username, xform.id_string, c, count))
def handle(self, *args, **options): print "Task started ..." # Get all the users for user in queryset_iterator(User.objects.all()): # For each user get the forms which are projectless for form in queryset_iterator( XForm.objects.select_related('projectxform').filter( projectxform=None, user=user)): # Create the default project self.create_and_assign_project(user, form) print "Task completed ..."
def handle(self, *args, **options): print "Task started ..." # Get all the users for user in queryset_iterator(User.objects.all()): # For each user get the forms which are projectless for form in queryset_iterator(XForm.objects .select_related('projectxform') .filter(projectxform=None, user=user)): # Create the default project self.create_and_assign_project(user, form) print "Task completed ..."
def destroy(self, request, *args, **kwargs): instance_ids = request.data.get('instance_ids') self.object = self.get_object() if isinstance(self.object, XForm): if not instance_ids: raise ParseError(_(u"Data id(s) not provided.")) else: instance_ids = [ x for x in instance_ids.split(',') if x.isdigit() ] if not instance_ids: raise ParseError(_(u"Invalid data ids were provided.")) initial_count = self.object.submission_count() queryset = Instance.objects.filter( id__in=instance_ids, xform=self.object, # do not update this timestamp when the record have # already been deleted. deleted_at__isnull=True) # loop through queryset # then call delete_instance that calls .save() # to allow emitting post_save signal for instance in queryset_iterator(queryset): delete_instance(instance, request.user) # updates the num_of_submissions for the form. after_count = self.object.submission_count(force_update=True) number_of_records_deleted = initial_count - after_count # send message send_message(instance_id=instance_ids, target_id=self.object.id, target_type=XFORM, user=request.user, message_verb=SUBMISSION_DELETED) return Response(data={ "message": "%d records were deleted" % number_of_records_deleted }, status=status.HTTP_200_OK) elif isinstance(self.object, Instance): if request.user.has_perm(CAN_DELETE_SUBMISSION, self.object.xform): delete_instance(self.object, request.user) # send message send_message(instance_id=self.object, target_id=self.object.xform.id, target_type=XFORM, user=request.user, message_verb=SUBMISSION_DELETED) else: raise PermissionDenied( _(u"You do not have delete " u"permissions.")) return Response(status=status.HTTP_204_NO_CONTENT)
def _get_streaming_response(self): """ Get a StreamingHttpResponse response object """ # use queryset_iterator. Will need to change this to the Django # native .iterator() method when we upgrade to django version 2 # because in Django 2 .iterator() has support for chunk size queryset = queryset_iterator(self.object_list, chunksize=2000) def get_json_string(item): return json.dumps(XFormBaseSerializer( instance=item, context={'request': self.request} ).data) response = StreamingHttpResponse( json_stream(queryset, get_json_string), content_type="application/json" ) # calculate etag value and add it to response headers if hasattr(self, 'etag_data'): self.set_etag_header(None, self.etag_data) self.set_cache_control(response) # set headers on streaming response for k, v in self.headers.items(): response[k] = v return response
def handle(self, *args, **options): self.stdout.write("Assign permission to the member team", ending='\n') count = 0 fail = 0 total = 0 if args: try: org_name = args[0] org = OrganizationProfile.objects.get(user__username=org_name) team = Team.objects.get(organization=org.user, name=u'%s#%s' % ( org.user.username, 'members')) self.assign_perm(team, org) count += 1 total += 1 except ObjectDoesNotExist as e: fail += 1 self.stdout.write(str(e), ending='\n') else: # Get all the teams for team in queryset_iterator( Team.objects.filter(name__contains='members')): self.assign_perm(team, team.organization) count += 1 total += 1 self.stdout.write("Assigned {} of {} records. failed: {}". format(count, total, fail), ending='\n')
def handle(self, *args, **options): self.stdout.write("Assign permission to the member team", ending='\n') count = 0 fail = 0 total = 0 if args: try: org_name = args[0] org = OrganizationProfile.objects.get(user__username=org_name) team = Team.objects.get(organization=org.user, name=u'%s#%s' % (org.user.username, 'members')) self.assign_perm(team, org) count += 1 total += 1 except ObjectDoesNotExist as e: fail += 1 self.stdout.write(str(e), ending='\n') else: # Get all the teams for team in queryset_iterator( Team.objects.filter(name__contains='members')): self.assign_perm(team, team.organization) count += 1 total += 1 self.stdout.write("Assigned {} of {} records. failed: {}".format( count, total, fail), ending='\n')
def _get_streaming_response(self): """ Get a StreamingHttpResponse response object """ # use queryset_iterator. Will need to change this to the Django # native .iterator() method when we upgrade to django version 2 # because in Django 2 .iterator() has support for chunk size queryset = queryset_iterator(self.object_list, chunksize=2000) def get_json_string(item): return json.dumps( XFormBaseSerializer(instance=item, context={ 'request': self.request }).data) response = StreamingHttpResponse(json_stream(queryset, get_json_string), content_type="application/json") # calculate etag value and add it to response headers if hasattr(self, 'etag_data'): self.set_etag_header(None, self.etag_data) self.set_cache_control(response) # set headers on streaming response for k, v in self.headers.items(): response[k] = v return response
def handle(self, *args, **options): print "Task started ..." # Get all the users for user in queryset_iterator( User.objects.exclude(pk=settings.ANONYMOUS_USER_ID)): self.set_project_to_user_forms(user) print "Task completed ..."
def process_attachments(self, user): """ Process attachments for submissions where media_all_received is False. """ xforms = XForm.objects.filter(user=user, deleted_at__isnull=True, downloadable=True) for xform in queryset_iterator(xforms): submissions = xform.instances.filter(media_all_received=False) to_process = submissions.count() if to_process: for submission in queryset_iterator(submissions): update_attachments(submission) not_processed = xform.instances.filter( media_all_received=False).count() self.stdout.write("%s to process %s - %s = %s processed" % (xform, to_process, not_processed, (to_process - not_processed)))
def forwards(self, orm): """Add parsed JSON to JSON instance column.""" for instance in queryset_iterator(orm["odk_logger.Instance"].objects.all()): obj = Instance.objects.get(pk=instance.pk) json = obj.get_dict() json[SUBMISSION_TIME] = instance.date_created.strftime(MONGO_STRFTIME) json[XFORM_ID_STRING] = obj._parser.get_xform_id_string() instance.json = json instance.save()
def handle(self, *args, **kwargs): self.stdout.write(_('%(nb)d XForms to update') % {'nb': DataDictionary.objects.count()}) for i, dd in enumerate( queryset_iterator(DataDictionary.objects.all())): if dd.xls: dd._set_uuid_in_xml() super(DataDictionary, dd).save() if (i + 1) % 10 == 0: self.stdout.write(_('Updated %(nb)d XForms...') % {'nb': i})
def handle(self, *args, **options): print "Task started ..." users = User.objects.exclude( username__iexact=settings.ANONYMOUS_DEFAULT_USERNAME) # Get all the users for user in queryset_iterator(users): self.set_project_to_user_forms(user) print "Task completed ..."
def handle(self, *args, **kwargs): print('%(nb)d XForms to update' % {'nb': DataDictionary.objects.count()}) for i, dd in enumerate(queryset_iterator( DataDictionary.objects.all())): if dd.xls: dd.set_uuid_in_xml(id_string=dd.id_string) super(DataDictionary, dd).save() if (i + 1) % 10 == 0: print('Updated %(nb)d XForms...' % {'nb': i})
def handle(self, *args, **options): self.stdout.write("Migrate permissions started", ending='\n') if len(args) < 2: self.stdout.write("This command takes two argument -m and -p " "Example: " "-m logger.Team " "-p logger.TeamUserObjectPermission") exit() if options['app_model']: app_model = args[0] else: self.stdout.write("-m , should be set as the first argument") exit() if options['perms_tbl']: perms_tbl = args[1] else: self.stdout.write("-p , should be set as the second argument") exit() model = get_model(app_model) perms_model = get_model(perms_tbl) if not issubclass(perms_model, UserObjectPermissionBase): self.stdout.write("-p , should be a model of a class that is " "a subclass of UserObjectPermissionBase") exit() ct = ContentType.objects.get(model=model.__name__.lower(), app_label=model._meta.app_label) # Get all the users users = User.objects.exclude( username__iexact=settings.ANONYMOUS_DEFAULT_USERNAME ).order_by('username') for user in queryset_iterator(users): self.stdout.write( "Processing: {} - {}".format(user.pk, user.username) ) for uop in user.userobjectpermission_set.filter(content_type=ct)\ .select_related('permission', 'content_type')\ .prefetch_related('permission', 'content_type'): try: perms_model( content_object=uop.content_object, user=user, permission=uop.permission ).save() except IntegrityError: continue except ValueError: pass
def handle(self, *args, **options): print "Task started ..." # Delete old bamboo rest service RestService.objects.filter(name='bamboo').delete() # Get all the rest services for rest in queryset_iterator(RestService.objects.all()): rest.save() print "Task ended ..."
def handle(self, *args, **options): self.stdout.write("Task started ...") users = User.objects.exclude( username__iexact=settings.ANONYMOUS_DEFAULT_USERNAME ) # Get all the users for user in queryset_iterator(users): self.set_project_to_user_forms(user) self.stdout.write("Task completed ...")
def handle(self, *args, **kwargs): self.stdout.write( '"username","email","first_name","last_name","name","organization"' ) for p in queryset_iterator(UserProfile.objects.all()): self.stdout.write( u'"{}","{}","{}","{}","{}","{}"'.format( p.user.username, p.user.email, p.user.first_name, p.user.last_name, p.name, p.organization ) )
def forwards(self, orm): """Add parsed JSON to JSON instance column.""" for instance in queryset_iterator( orm['odk_logger.Instance'].objects.all()): obj = Instance.objects.get(pk=instance.pk) json = obj.get_dict() json[SUBMISSION_TIME] = instance.date_created.strftime( MONGO_STRFTIME) json[XFORM_ID_STRING] = obj._parser.get_xform_id_string() instance.json = json instance.save()
def handle(self, *args, **options): self.stdout.write("Task started ...") # Delete old bamboo rest service RestService.objects.filter(name='bamboo').delete() # Get all the rest services for rest in queryset_iterator(RestService.objects.all()): rest.save() self.stdout.write("Task ended ...")
def handle(self, *args, **kwargs): self.stdout.write( _('%(nb)d XForms to update') % {'nb': DataDictionary.objects.count()}) for i, dd in enumerate(queryset_iterator( DataDictionary.objects.all())): if dd.xls: dd._set_uuid_in_xml() super(DataDictionary, dd).save() if (i + 1) % 10 == 0: self.stdout.write(_('Updated %(nb)d XForms...') % {'nb': i})
def handle(self, *args, **options): attachments_qs = Attachment.objects.select_related( 'instance', 'instance__xform') if options.get('username'): username = options.get('username') try: user = User.objects.get(username=username) except User.DoesNotExist: raise CommandError( "Error: username %(username)s does not exist" % {'username': username}) attachments_qs = attachments_qs.filter(instance__user=user) if options.get('id_string'): id_string = options.get('id_string') try: xform = XForm.objects.get(id_string=id_string) except XForm.DoesNotExist: raise CommandError( "Error: Form with id_string %(id_string)s does not exist" % {'id_string': id_string}) attachments_qs = attachments_qs.filter(instance__xform=xform) fs = get_storage_class('django.core.files.storage.FileSystemStorage')() for att in queryset_iterator(attachments_qs): filename = att.media_file.name default_storage = get_storage_class()() full_path = get_path(filename, settings.THUMB_CONF['small']['suffix']) if options.get('force') is not None: for s in ['small', 'medium', 'large']: fp = get_path(filename, settings.THUMB_CONF[s]['suffix']) if default_storage.exists(fp): default_storage.delete(fp) if not default_storage.exists(full_path): try: if default_storage.__class__ != fs.__class__: resize(filename, att.extension) else: resize_local_env(filename, att.extension) path = get_path(filename, '%s' % THUMB_CONF['small']['suffix']) if default_storage.exists(path): self.stdout.write( _(u'Thumbnails created for %(file)s') % {'file': filename}) else: self.stdout.write( _(u'Problem with the file %(file)s') % {'file': filename}) except (IOError, OSError) as e: self.stderr.write( _(u'Error on %(filename)s: %(error)s') % { 'filename': filename, 'error': e })
def handle(self, *args, **options): self.stdout.write("Migrate permissions started", ending='\n') if len(args) < 2: self.stdout.write("This command takes two argument -m and -p " "Example: " "-m logger.Team " "-p logger.TeamUserObjectPermission") exit() if options['app_model']: app_model = args[0] else: self.stdout.write("-m , should be set as the first argument") exit() if options['perms_tbl']: perms_tbl = args[1] else: self.stdout.write("-p , should be set as the second argument") exit() model = get_model(app_model) perms_model = get_model(perms_tbl) if not issubclass(perms_model, UserObjectPermissionBase): self.stdout.write("-p , should be a model of a class that is " "a subclass of UserObjectPermissionBase") exit() ct = ContentType.objects.get( model=model.__name__.lower(), app_label=model._meta.app_label) # Get all the users users = User.objects.exclude( username__iexact=settings.ANONYMOUS_DEFAULT_USERNAME).order_by( 'username') for user in queryset_iterator(users): self.stdout.write( "Processing: {} - {}".format(user.pk, user.username)) for uop in user.userobjectpermission_set.filter(content_type=ct)\ .select_related('permission', 'content_type')\ .prefetch_related('permission', 'content_type'): try: perms_model( content_object=uop.content_object, user=user, permission=uop.permission).save() except IntegrityError: continue except ValueError: pass
def handle(self, *args, **kwargs): attachments_qs = Attachment.objects.select_related( 'instance', 'instance__xform') if kwargs.get('username'): username = kwargs.get('username') try: user = User.objects.get(username=username) except User.DoesNotExist: raise CommandError( "Error: username %(username)s does not exist" % {'username': username}) attachments_qs = attachments_qs.filter(instance__user=user) if kwargs.get('id_string'): id_string = kwargs.get('id_string') try: xform = XForm.objects.get(id_string=id_string) except XForm.DoesNotExist: raise CommandError( "Error: Form with id_string %(id_string)s does not exist" % {'id_string': id_string}) attachments_qs = attachments_qs.filter(instance__xform=xform) for att in queryset_iterator(attachments_qs): filename = att.media_file.name default_storage = get_storage_class()() full_path = get_path(filename, settings.THUMB_CONF['small']['suffix']) if kwargs.get('force') is not None: for s in settings.THUMB_CONF.keys(): fp = get_path(filename, settings.THUMB_CONF[s]['suffix']) if default_storage.exists(fp): default_storage.delete(fp) if not default_storage.exists(full_path): try: resize(filename) if default_storage.exists( get_path( filename, '%s' % settings.THUMB_CONF['small']['suffix'])): print( _('Thumbnails created for %(file)s') % {'file': filename}) else: print( _('Problem with the file %(file)s') % {'file': filename}) except (IOError, OSError), e: print( _('Error on %(filename)s: %(error)s') % { 'filename': filename, 'error': e })
def kml_export_data(id_string, user, xform=None): """ KML export data from form submissions. """ def cached_get_labels(xpath): """ Get and Cache labels for the XForm. """ if xpath in list(labels): return labels[xpath] labels[xpath] = xform.get_label(xpath) return labels[xpath] xform = xform or XForm.objects.get(id_string=id_string, user=user) data_kwargs = {'geom__isnull': False} if xform.is_merged_dataset: data_kwargs.update({ 'xform_id__in': [i for i in xform.mergedxform.xforms.filter( deleted_at__isnull=True).values_list('id', flat=True)] }) else: data_kwargs.update({'xform_id': xform.pk}) instances = Instance.objects.filter(**data_kwargs).order_by('id') data_for_template = [] labels = {} for instance in queryset_iterator(instances): # read the survey instances data_for_display = instance.get_dict() xpaths = list(data_for_display) xpaths.sort(key=cmp_to_key(instance.xform.get_xpath_cmp())) table_rows = [ '<tr><td>%s</td><td>%s</td></tr>' % (cached_get_labels(xpath), data_for_display[xpath]) for xpath in xpaths if not xpath.startswith(u"_")] img_urls = image_urls(instance) if instance.point: data_for_template.append({ 'name': instance.xform.id_string, 'id': instance.id, 'lat': instance.point.y, 'lng': instance.point.x, 'image_urls': img_urls, 'table': '<table border="1"><a href="#"><img width="210" ' 'class="thumbnail" src="%s" alt=""></a>%s' '</table>' % (img_urls[0] if img_urls else "", ''.join(table_rows))}) return data_for_template
def handle(self, *args, **options): attachments_qs = Attachment.objects.select_related( 'instance', 'instance__xform') if options.get('username'): username = options.get('username') try: user = User.objects.get(username=username) except User.DoesNotExist: raise CommandError( "Error: username %(username)s does not exist" % {'username': username}) attachments_qs = attachments_qs.filter(instance__user=user) if options.get('id_string'): id_string = options.get('id_string') try: xform = XForm.objects.get(id_string=id_string) except XForm.DoesNotExist: raise CommandError( "Error: Form with id_string %(id_string)s does not exist" % {'id_string': id_string}) attachments_qs = attachments_qs.filter(instance__xform=xform) fs = get_storage_class('django.core.files.storage.FileSystemStorage')() for att in queryset_iterator(attachments_qs): filename = att.media_file.name default_storage = get_storage_class()() full_path = get_path(filename, settings.THUMB_CONF['small']['suffix']) if options.get('force') is not None: for s in ['small', 'medium', 'large']: fp = get_path(filename, settings.THUMB_CONF[s]['suffix']) if default_storage.exists(fp): default_storage.delete(fp) if not default_storage.exists(full_path): try: if default_storage.__class__ != fs.__class__: resize(filename, att.extension) else: resize_local_env(filename, att.extension) path = get_path( filename, '%s' % THUMB_CONF['small']['suffix']) if default_storage.exists(path): self.stdout.write( _(u'Thumbnails created for %(file)s') % {'file': filename}) else: self.stdout.write( _(u'Problem with the file %(file)s') % {'file': filename}) except (IOError, OSError) as e: self.stderr.write(_( u'Error on %(filename)s: %(error)s') % {'filename': filename, 'error': e})
def kml_export_data(id_string, user, xform=None): if xform is None: xform = XForm.objects.get(id_string=id_string, user=user) instances = Instance.objects.filter(xform__user=user, xform__id_string=id_string, geom__isnull=False).order_by('id') data_for_template = [] labels = {} def cached_get_labels(xpath): if xpath in labels.keys(): return labels[xpath] labels[xpath] = xform.get_label(xpath) return labels[xpath] for instance in queryset_iterator(instances): # read the survey instances data_for_display = instance.get_dict() xpaths = data_for_display.keys() xpaths.sort(cmp=instance.xform.get_xpath_cmp()) label_value_pairs = [(cached_get_labels(xpath), data_for_display[xpath]) for xpath in xpaths if not xpath.startswith(u"_")] table_rows = [ '<tr><td>%s</td><td>%s</td></tr>' % (k, v) for k, v in label_value_pairs ] img_urls = image_urls(instance) img_url = img_urls[0] if img_urls else "" point = instance.point if point: data_for_template.append({ 'name': id_string, 'id': instance.id, 'lat': point.y, 'lng': point.x, 'image_urls': img_urls, 'table': '<table border="1"><a href="#"><img width="210" ' 'class="thumbnail" src="%s" alt=""></a>%s' '</table>' % (img_url, ''.join(table_rows)) }) return data_for_template
def handle(self, *args, **kwargs): # username if args: users = User.objects.filter(username__contains=args[0]) else: # All the accounts self.stdout.write("Fetching all the account {}", ending='\n') users = queryset_iterator( User.objects.exclude(pk=settings.ANONYMOUS_USER_ID)) for user in users: self.add_id(user)
def handle(self, *args, **options): self.stdout.write("Migrate group permissions started", ending='\n') if len(args) < 2: self.stdout.write("This command takes two argument -m and -p " "Example: " "-m logger.Team " "-p logger.TeamUserObjectPermission") exit() if options['app_model']: app_model = args[0] else: self.stdout.write("-m , should be set as the first argument") exit() if options['perms_tbl']: perms_tbl = args[1] else: self.stdout.write("-p , should be set as the second argument") exit() model = get_model(app_model) perms_model = get_model(perms_tbl) if not issubclass(perms_model, GroupObjectPermissionBase): self.stdout.write("-p , should be a model of a class that is " "a subclass of GroupObjectPermissionBase") exit() ct = ContentType.objects.get( model=model.__name__.lower(), app_label=model._meta.app_label) teams = Team.objects.filter().annotate( c=Count('groupobjectpermission')).filter(c__gt=0) for team in queryset_iterator(teams): self.stdout.write("Processing: {} - {}".format(team.pk, team.name)) for gop in team.groupobjectpermission_set.filter(content_type=ct)\ .select_related('permission', 'content_type')\ .prefetch_related('permission', 'content_type'): try: perms_model( content_object=gop.content_object, group=team, permission=gop.permission).save() except IntegrityError: continue except ValueError: pass self.stdout.write("Group permissions migration done.")
def handle(self, *args, **kwargs): with open('gps_points_export.csv', 'w') as csvfile: fieldnames = ['longitude', 'latitude', 'date_created'] writer = csv.writer(csvfile) writer.writerow(fieldnames) for instance in queryset_iterator( Instance.objects.exclude(geom__isnull=True)): if hasattr(instance, 'point') and instance.point is not None: longitude = instance.point.coords[0] latitude = instance.point.coords[1] writer.writerow( [longitude, latitude, instance.date_created]) self.stdout.write("Export of gps files has completed!!!!")
def handle(self, *args, **kwargs): xforms = XForm.objects.all() total = xforms.count() count = 0 for xform in queryset_iterator(XForm.objects.all()): has_geo = xform.geocoded_submission_count() > 0 try: xform.instances_with_geopoints = has_geo xform.save() except Exception as e: self.stderr.write(e) else: count += 1 self.stdout.write("%d of %d forms processed." % (count, total))
def handle(self, *args, **options): self.stdout.write("Migrate group permissions started", ending='\n') if len(args) < 2: self.stdout.write("This command takes two argument -m and -p " "Example: " "-m logger.Team " "-p logger.TeamUserObjectPermission") exit() if options['app_model']: app_model = args[0] else: self.stdout.write("-m , should be set as the first argument") exit() if options['perms_tbl']: perms_tbl = args[1] else: self.stdout.write("-p , should be set as the second argument") exit() model = get_model(app_model) perms_model = get_model(perms_tbl) if not issubclass(perms_model, GroupObjectPermissionBase): self.stdout.write("-p , should be a model of a class that is " "a subclass of GroupObjectPermissionBase") exit() ct = ContentType.objects.get(model=model.__name__.lower(), app_label=model._meta.app_label) teams = Team.objects.filter().annotate( c=Count('groupobjectpermission')).filter(c__gt=0) for team in queryset_iterator(teams): self.stdout.write("Processing: {} - {}".format(team.pk, team.name)) for gop in team.groupobjectpermission_set.filter(content_type=ct)\ .select_related('permission', 'content_type')\ .prefetch_related('permission', 'content_type'): try: perms_model(content_object=gop.content_object, group=team, permission=gop.permission).save() except IntegrityError: continue except ValueError: pass self.stdout.write("Group permissions migration done.")
def test_queryset_iterator(self): user_model = get_user_model() user_model.objects.create_user(username='******', password='******', email='*****@*****.**') user_model.objects.create_user(username='******', password='******', email='*****@*****.**') user_model.objects.create_user(username='******', password='******', email='test@test_3.com') self.assertEquals( 'generator', queryset_iterator(user_model.objects.all(), chunksize=1).__class__.__name__)
def handle(self, *args, **options): deleted = 0 self.stdout.write("Starting UserObject") for perm in queryset_iterator( UserObjectPermission.objects.select_related()): try: perm.content_object except AttributeError: perm.delete() deleted += 1 self.stdout.write( "deleted {} stale permission".format(deleted)) self.stdout.write("Starting GroupObject") for perm in queryset_iterator( GroupObjectPermission.objects.select_related()): try: perm.content_object except AttributeError: perm.delete() deleted += 1 self.stdout.write( "deleted {} stale permission".format(deleted)) self.stdout.write( "Total removed orphan object permissions instances: %d" % deleted)
def set_project_to_user_forms(self, user): default_project_name = user.username + '\'s Project' try: project = Project.objects.get(name=default_project_name) except Project.DoesNotExist: metadata = {'description': 'Default Project'} project = Project.objects.create(name=default_project_name, organization=user, created_by=user, metadata=metadata) self.stdout.write("Created project %s" % project.name) finally: xforms = user.xforms.filter(project=XFORM_DEFAULT_PROJECT_ID) for xform in queryset_iterator(xforms): xform.project = project xform.save()
def handle(self, *args, **kwargs): pks = Attachment.objects.filter( extension=Attachment.OSM, instance__xform__instances_with_osm=False)\ .values_list('instance__xform', flat=True).distinct() xforms = XForm.objects.filter(pk__in=pks) total = xforms.count() count = 0 for xform in queryset_iterator(xforms): try: xform.instances_with_osm = True xform.save() except Exception as e: print e else: count += 1 print "%d of %d forms processed." % (count, total)
def set_project_to_user_forms(self, user): default_project_name = user.username + '\'s Project' try: project = Project.objects.get(name=default_project_name) except Project.DoesNotExist: metadata = {'description': 'Default Project'} project = Project.objects.create(name=default_project_name, organization=user, created_by=user, metadata=metadata) print "Created project " + project.name finally: xforms = user.xforms.filter(project=XFORM_DEFAULT_PROJECT_ID) for xform in queryset_iterator(xforms): xform.project = project xform.save()
def recalculate_xform_hash(apps, schema_editor): # pylint: disable=W0613 """ Recalculate all XForm hashes. """ XForm = apps.get_model('logger', 'XForm') # pylint: disable=C0103 xforms = XForm.objects.filter(downloadable=True, deleted_at__isnull=True).only('xml') count = xforms.count() counter = 0 for xform in queryset_iterator(xforms, 500): xform.hash = u'md5:%s' % md5(xform.xml.encode('utf8')).hexdigest() xform.save(update_fields=['hash']) counter += 1 if counter % 500 == 0: print("Processed %d of %d forms." % (counter, count)) print("Processed %dforms." % counter)
def handle(self, *args, **kwargs): self.stdout.write("Updating forms owner", ending='\n') for project in queryset_iterator(Project.objects.all()): for xform in project.xform_set.all(): try: if xform.user != project.organization: self.stdout.write( "Processing: {} - {}".format(xform.id_string, xform.user.username) ) xform.user = project.organization xform.save() except: self.stdout.write( "Error processing: {} - {}".format(xform.id_string, xform.user.username) ) pass
def handle(self, *args, **kwargs): self.stdout.write("Updating forms owner", ending='\n') for project in queryset_iterator(Project.objects.all()): for xform in project.xform_set.all(): try: if xform.user != project.organization: self.stdout.write( "Processing: {} - {}".format(xform.id_string, xform.user.username) ) xform.user = project.organization xform.save() except Exception: self.stdout.write( "Error processing: {} - {}".format(xform.id_string, xform.user.username) ) pass
def handle(self, *args, **options): self.stdout.write("Re-assigining started", ending='\n') if not args: raise CommandError('Param not set. <app model [created_perm]>') if len(args) < 3: raise CommandError('Param not set. <app model [created_perm]>') app = args[0] model = args[1] new_perm = args[2] from onadata.libs.utils.model_tools import queryset_iterator # Get all the users for user in queryset_iterator( User.objects.exclude(pk=settings.ANONYMOUS_USER_ID)): self.reassign_perms(user, app, model, new_perm) self.stdout.write("Re-assigining finished", ending='\n')
def _reapply_perms(self, username=None, days=None, seconds=None): xforms = XForm.objects.none() the_past = None if username: xforms = XForm.objects.filter(user__username=username) else: if days: the_past = timezone.now() - timedelta(days=days) if seconds: the_past = timezone.now() - timedelta(seconds=seconds) if the_past: xforms = XForm.objects.filter( Q(date_created__gte=the_past) | Q(date_modified__gte=the_past)) self.stdout.write(_("{} to be updated").format(xforms.count())) with use_master: for xform in queryset_iterator(xforms): set_project_perms_to_xform(xform, xform.project) self.stdout.write(gc.collect())
def get_list_of_parsed_instances(self, flat=True): for i in queryset_iterator(self.instances_for_export(self)): # TODO: there is information we want to add in parsed xforms. yield i.get_dict(flat=flat)
def dicts(cls, xform): qs = cls.objects.filter(instance__xform=xform) for parsed_instance in queryset_iterator(qs): yield parsed_instance.to_dict()
def forwards(self, orm): "Parse all instance to add geoms." for obj in queryset_iterator(orm['odk_logger.Instance'].objects.all()): instance = Instance.objects.get(pk=obj.pk) instance.save(force=True)
def mongo_sync_status(remongo=False, update_all=False, user=None, xform=None): """Check the status of records in the mysql db versus mongodb. At a minimum, return a report (string) of the results. Optionally, take action to correct the differences, based on these parameters, if present and defined: remongo -> if True, update the records missing in mongodb (default: False) update_all -> if True, update all the relevant records (default: False) user -> if specified, apply only to the forms for the given user (default: None) xform -> if specified, apply only to the given form (default: None) """ qs = XForm.objects.only('id_string', 'user').select_related('user') if user and not xform: qs = qs.filter(user=user) elif user and xform: qs = qs.filter(user=user, id_string=xform.id_string) else: qs = qs.all() total = qs.count() found = 0 done = 0 total_to_remongo = 0 report_string = "" for xform in queryset_iterator(qs, 100): # get the count user = xform.user instance_count = Instance.objects.filter(xform=xform).count() userform_id = "%s_%s" % (user.username, xform.id_string) mongo_count = mongo_instances.find( {common_tags.USERFORM_ID: userform_id}).count() if instance_count != mongo_count or update_all: line = "user: %s, id_string: %s\nInstance count: %d\t"\ "Mongo count: %d\n---------------------------------"\ "-----\n" % ( user.username, xform.id_string, instance_count, mongo_count) report_string += line found += 1 total_to_remongo += (instance_count - mongo_count) # should we remongo if remongo or (remongo and update_all): if update_all: sys.stdout.write( "Updating all records for %s\n--------------------" "---------------------------\n" % xform.id_string) else: sys.stdout.write( "Updating missing records for %s\n----------------" "-------------------------------\n" % xform.id_string) update_mongo_for_xform( xform, only_update_missing=not update_all) done += 1 sys.stdout.write( "%.2f %% done ...\r" % ((float(done) / float(total)) * 100)) # only show stats if we are not updating mongo, the update function # will show progress if not remongo: line = "Total # of forms out of sync: %d\n" \ "Total # of records to remongo: %d\n" % (found, total_to_remongo) report_string += line return report_string