def save_species(migration_rules, migration_event, species_dict, species_obj, instance, **kwargs): non_migrated_species = Species.objects.raw(""" SELECT * FROM treemap_species WHERE instance_id=%(instance_id)s AND id not in (SELECT otm2_model_id FROM otm1_migrator_otm1modelrelic WHERE otm2_model_name='species' AND instance_id=%(instance_id)s) """ % {'instance_id': instance.pk}) if len(list(non_migrated_species)) > 0: raise MigrationException("You cannot migrate species, at all, " "if any species for this instance are " "not the result of a migration. This is " "necessary to avoid record duplication.") species_obj.save_with_user_without_verifying_authorization( User.system_user()) OTM1ModelRelic.objects.create( instance=instance, migration_event=migration_event, otm1_model_id=species_dict['pk'], otm2_model_name='species', otm2_model_id=species_obj.pk) return species_obj
def merge_species(request, instance): species_to_delete_id = request.REQUEST['species_to_delete'] species_to_replace_with_id = request.REQUEST['species_to_replace_with'] species_to_delete = get_object_or_404(Species, instance=instance, pk=species_to_delete_id) species_to_replace_with = get_object_or_404(Species, instance=instance, pk=species_to_replace_with_id) if species_to_delete.pk == species_to_replace_with.pk: return HttpResponse(json.dumps( {"error": "Must pick different species"}), content_type='application/json', status=400) # TODO: .update_with_user()? trees_to_update = Tree.objects\ .filter(instance=instance)\ .filter(species=species_to_delete) for tree in trees_to_update: tree.species = species_to_replace_with tree.save_with_system_user_bypass_auth() species_to_delete.delete_with_user(request.user) # Force a tree count update species_to_replace_with.tree_count = 0 species_to_replace_with.save_with_user(User.system_user()) return HttpResponse(json.dumps({"status": "ok"}), content_type='application/json')
def save_treephoto(migration_rules, migration_event, treephoto_path, model_dict, treephoto_obj, instance, **kwargs): if model_dict['fields']['tree'] == models.UNBOUND_MODEL_ID: treephoto_obj = None pk = models.UNBOUND_MODEL_ID else: image = open(os.path.join(treephoto_path, model_dict['fields']['photo'])) treephoto_obj.set_image(image) treephoto_obj.map_feature_id = (Tree .objects .values_list('plot__id', flat=True) .get(pk=treephoto_obj.tree_id)) del model_dict['fields']['photo'] treephoto_obj.save_with_user_without_verifying_authorization( User.system_user()) pk = treephoto_obj.pk OTM1ModelRelic.objects.create( instance=instance, migration_event=migration_event, otm1_model_id=model_dict['pk'], otm2_model_name='treephoto', otm2_model_id=pk) return treephoto_obj
def merge_species(request, instance): species_to_delete_id = request.REQUEST['species_to_delete'] species_to_replace_with_id = request.REQUEST['species_to_replace_with'] species_to_delete = get_object_or_404( Species, instance=instance, pk=species_to_delete_id) species_to_replace_with = get_object_or_404( Species, instance=instance, pk=species_to_replace_with_id) if species_to_delete.pk == species_to_replace_with.pk: return HttpResponse( json.dumps({"error": "Must pick different species"}), content_type='application/json', status=400) # TODO: .update_with_user()? trees_to_update = Tree.objects\ .filter(instance=instance)\ .filter(species=species_to_delete) for tree in trees_to_update: tree.species = species_to_replace_with tree.save_with_system_user_bypass_auth() species_to_delete.delete_with_user(request.user) # Force a tree count update species_to_replace_with.tree_count = 0 species_to_replace_with.save_with_user(User.system_user()) return HttpResponse( json.dumps({"status": "ok"}), content_type='application/json')
def create_override(species_obj, species_dict): for region in ['NoEastXXX', 'PiedmtCLT']: override = ITreeCodeOverride( instance_species_id=species_obj.pk, region=ITreeRegion.objects.get(code=region), itree_code=species_dict['fields']['itree_code']) override.save_with_user(User.system_user()) return species_obj
def setUp(self): instance = make_instance() user = make_admin_user(instance) species = Species(instance=instance, genus="g1", species="", cultivar="", max_diameter=50.0, max_height=100.0) species.save_with_user(User.system_user()) login(self.client, user.username)
def setUp(self): instance = make_instance() user = make_admin_user(instance) species = Species(instance=instance, genus='g1', species='', cultivar='', max_diameter=50.0, max_height=100.0) species.save_with_user(User.system_user()) login(self.client, user.username)
def setup_env(self, *args, **options): """ Create some seed data """ instance = Instance.objects.get(pk=options['instance']) try: user = User.system_user() except User.DoesNotExist: self.stdout.write('Error: Could not find a superuser to use') return 1 instance_user = user.get_instance_user(instance) if instance_user is None: r = Role(name='global', rep_thresh=0, instance=instance) r.save() instance_user = InstanceUser(instance=instance, user=user, role=r) instance_user.save_with_user(user) self.stdout.write('Added system user to instance with global role') for field in Plot._meta.get_all_field_names(): _, c = FieldPermission.objects.get_or_create( model_name='Plot', field_name=field, role=instance_user.role, instance=instance, permission_level=FieldPermission.WRITE_DIRECTLY) if c: self.stdout.write('Created plot permission for field "%s"' % field) for field in Tree._meta.get_all_field_names(): _, c = FieldPermission.objects.get_or_create( model_name='Tree', field_name=field, role=instance_user.role, instance=instance, permission_level=FieldPermission.WRITE_DIRECTLY) if c: self.stdout.write('Created tree permission for field "%s"' % field) dt = 0 dp = 0 if options.get('delete', False): for t in Tree.objects.all(): t.delete_with_user(user) dt += 1 for p in Plot.objects.all(): p.delete_with_user(user) dp += 1 self.stdout.write("Deleted %s trees and %s plots" % (dt, dp)) return instance, user
def create_override(species_obj, species_dict): itree_code = species_dict['fields'].get('itree_code', None) if not itree_code: raise MigrationException("species_dict missing itree_code: " + str(species_dict)) override = ITreeCodeOverride( instance_species_id=species_obj.pk, region=ITreeRegion.objects.get(code=TREEZILLA_ITREE_REGION_CODE), itree_code=itree_code) override.save_with_user(User.system_user()) return species_obj
def setup_env(self, *args, **options): """ Create some seed data """ if options['instance']: instance = Instance.objects.get(pk=options['instance']) elif options['instance_url_name']: instance = Instance.objects.get( url_name=options['instance_url_name']) else: raise Exception("must provide instance") try: user = User.system_user() except User.DoesNotExist: self.stdout.write('Error: Could not find a superuser to use') return 1 instance_user = user.get_instance_user(instance) if instance_user is None: r = Role.objects.get_or_create(name=Role.ADMINISTRATOR, rep_thresh=0, instance=instance, default_permission=3) instance_user = InstanceUser(instance=instance, user=user, role=r[0]) instance_user.save_with_user(user) self.stdout.write( 'Added system user to instance with ADMINISTRATOR role') add_default_permissions(instance) dt = 0 dp = 0 if options.get('delete', False): for t in Tree.objects.all(): t.delete_with_user(user) dt += 1 for p in Plot.objects.all(): p.delete_with_user(user) dp += 1 self.stdout.write("Deleted %s trees and %s plots" % (dt, dp)) dr = 0 if options.get('delete_resources', False): for f in MapFeature.objects.all(): if f.feature_type != 'Plot': f.delete_with_user(user) dr += 1 self.stdout.write("Deleted %s resources" % dr) return instance, user
def create_override(species_obj, species_dict): itree_code = species_dict['fields'].get('itree_code', None) if not itree_code: sci_name = species_dict['fields'].get('scientific_name', '').lower() print('No itree_code for "%d: %s"' % (species_dict['pk'], sci_name)) itree_code = meta_species.get(sci_name, '') print('Looked up meta species "%s"' % itree_code) override = ITreeCodeOverride( instance_species_id=species_obj.pk, region=ITreeRegion.objects.get(code=TAMPA_ITREE_REGION_CODE), itree_code=itree_code) override.save_with_user(User.system_user()) return species_obj
def save_species(migration_rules, migration_event, species_dict, species_obj, instance): species_obj.otm_code = otm_code_search(species_dict['fields']) or '' species_obj.save_with_user_without_verifying_authorization( User.system_user()) OTM1ModelRelic.objects.create( instance=instance, migration_event=migration_event, otm1_model_id=species_dict['pk'], otm2_model_name='species', otm2_model_id=species_obj.pk) return species_obj
def _map_feature_audits(user, instance, feature, filters=None, cudf_filters=None): if filters is None: filters = [] if cudf_filters is None: cudf_filters = [] readable_plot_fields = feature.visible_fields(user) feature_filter = Q( model=feature.feature_type, model_id=feature.pk, field__in=readable_plot_fields) filters.append(feature_filter) feature_collection_udfs_filter = Q( model__in=feature.visible_collection_udfs_audit_names(user), model_id__in=feature.collection_udfs_audit_ids()) cudf_filters.append(feature_collection_udfs_filter) # Seems to be much faster to do three smaller # queries here instead of ORing them together # (about a 50% inprovement!) # TODO: Verify this is still the case now that we are also getting # collection udf audits iaudit = Audit.objects\ .filter(instance=instance)\ .exclude(user=User.system_user()) audits = [] for afilter in filters: audits += list(iaudit.filter(afilter).order_by('-created')[:5]) # UDF collection audits have some fields which aren't very useful to show udf_collection_exclude_filter = Q( field__in=['model_id', 'field_definition']) for afilter in cudf_filters: audits += list( iaudit.filter(afilter).exclude(udf_collection_exclude_filter). order_by('-created')[:5]) audits = sorted(audits, key=lambda audit: audit.updated, reverse=True)[:5] return audits
def _map_feature_audits(user, instance, feature, filters=None, cudf_filters=None): if filters is None: filters = [] if cudf_filters is None: cudf_filters = [] readable_plot_fields = feature.visible_fields(user) feature_filter = Q(model=feature.feature_type, model_id=feature.pk, field__in=readable_plot_fields) filters.append(feature_filter) feature_collection_udfs_filter = Q( model__in=feature.visible_collection_udfs_audit_names(user), model_id__in=feature.collection_udfs_audit_ids()) cudf_filters.append(feature_collection_udfs_filter) # Seems to be much faster to do three smaller # queries here instead of ORing them together # (about a 50% inprovement!) # TODO: Verify this is still the case now that we are also getting # collection udf audits iaudit = Audit.objects\ .filter(instance=instance)\ .exclude(user=User.system_user()) audits = [] for afilter in filters: audits += list(iaudit.filter(afilter).order_by('-created')[:5]) # UDF collection audits have some fields which aren't very useful to show udf_collection_exclude_filter = Q( field__in=['model_id', 'field_definition']) for afilter in cudf_filters: audits += list( iaudit.filter(afilter).exclude( udf_collection_exclude_filter).order_by('-created')[:5]) audits = sorted(audits, key=lambda audit: audit.updated, reverse=True)[:5] return audits
def find_user_to_save_with(migration_rules, model): model_name = model.__class__.__name__.lower() user_field_to_try = (migration_rules[model_name].get('dependencies', {}).get('user', None)) if user_field_to_try: potential_user_id = getattr(model, user_field_to_try, None) else: potential_user_id = None try: user = User.objects.get(pk=potential_user_id) except User.DoesNotExist: user = User.system_user() return user
def save_plot(migration_rules, migration_event, plot_dict, plot_obj, instance): if plot_dict['fields']['present'] is False: plot_obj = None pk = models.UNBOUND_MODEL_ID else: plot_obj.save_with_user_without_verifying_authorization( User.system_user()) pk = plot_obj.pk OTM1ModelRelic.objects.create( instance=instance, migration_event=migration_event, otm1_model_id=plot_dict['pk'], otm2_model_name='plot', otm2_model_id=pk) return plot_obj
def find_user_to_save_with(migration_rules, model): model_name = model.__class__.__name__.lower() user_field_to_try = (migration_rules[model_name] .get('dependencies', {}) .get('user', None)) if user_field_to_try: potential_user_id = getattr(model, user_field_to_try, None) else: potential_user_id = None try: user = User.objects.get(pk=potential_user_id) except User.DoesNotExist: user = User.system_user() return user
def save_tree(migration_rules, migration_event, tree_dict, tree_obj, instance, **kwargs): if ((tree_dict['fields']['present'] is False or tree_dict['fields']['plot'] == models.UNBOUND_MODEL_ID)): tree_obj = None pk = models.UNBOUND_MODEL_ID else: tree_obj.save_with_user_without_verifying_authorization( User.system_user()) pk = tree_obj.pk OTM1ModelRelic.objects.create( instance=instance, migration_event=migration_event, otm1_model_id=tree_dict['pk'], otm2_model_name='tree', otm2_model_id=pk) return tree_obj
def tests_works_when_normal_save_fails(self): self.plot = self.plot self.plot.width = 444 with self.assertRaises(AuthorizeException): self.plot.save_with_user(User.system_user()) self.plot.save_with_system_user_bypass_auth()
def tearDown(self): self.user.delete_with_user(User.system_user())
def get_audits(logged_in_user, instance, query_vars, user=None, models=ALLOWED_MODELS, model_id=None, start_id=None, prev_start_ids=[], page_size=PAGE_DEFAULT, exclude_pending=True, should_count=False): if instance: if instance.is_accessible_by(logged_in_user): instances = Instance.objects.filter(pk=instance.pk) else: instances = Instance.objects.none() # If we didn't specify an instance we only want to # show audits where the user has permission else: instances = Instance.objects\ .filter(user_accessible_instance_filter(logged_in_user)) if user: instances = instances.filter(pk__in=_instance_ids_edited_by(user)) instances = instances.distinct() if not instances.exists(): # Force no results return { 'audits': Audit.objects.none(), 'total_count': 0, 'next_page': None, 'prev_page': None } map_feature_models = set(MapFeature.subclass_dict().keys()) model_filter = Q() # We only want to show the TreePhoto's image, not other fields # and we want to do it automatically if 'Tree' was specified as # a model. The same goes for MapFeature(s) <-> MapFeaturePhoto # There is no need to check permissions, because photos are always visible if 'Tree' in models: model_filter = model_filter | Q(model='TreePhoto', field='image') if map_feature_models.intersection(models): model_filter = model_filter | Q(model='MapFeaturePhoto', field='image') for inst in instances: eligible_models = ({'Tree', 'TreePhoto', 'MapFeaturePhoto'} | set(inst.map_feature_types)) & set(models) if logged_in_user == user: eligible_udfs = { 'udf:%s' % udf.id for udf in udf_defs(inst) if udf.model_type in eligible_models and udf.iscollection } # The logged-in user can see all their own edits model_filter = model_filter | Q( instance=inst, model__in=(eligible_models | eligible_udfs)) else: # Filter other users' edits by their visibility to the # logged-in user for model in eligible_models: ModelClass = get_auditable_class(model) fake_model = ModelClass(instance=inst) if issubclass(ModelClass, Authorizable): visible_fields = fake_model.visible_fields(logged_in_user) model_filter = model_filter |\ Q(model=model, field__in=visible_fields, instance=inst) else: model_filter = model_filter | Q(model=model, instance=inst) if issubclass(ModelClass, UDFModel): model_collection_udfs_audit_names = ( fake_model.visible_collection_udfs_audit_names( logged_in_user)) model_filter = model_filter | (Q( model__in=model_collection_udfs_audit_names)) udf_bookkeeping_fields = Q(model__startswith='udf:', field__in=('id', 'model_id', 'field_definition')) audits = (Audit.objects.filter(model_filter).filter( instance__in=instances).select_related('instance').exclude( udf_bookkeeping_fields).exclude( user=User.system_user()).order_by('-pk')) if user: audits = audits.filter(user=user) if model_id: audits = audits.filter(model_id=model_id) if exclude_pending: audits = audits.exclude(requires_auth=True, ref__isnull=True) # Slicing the QuerySet uses a SQL Limit, which has proven to be quite slow. # By relying on the fact the our list is ordered by primary key from newest # to oldest, we can rely on the index on the primary key, which is faster. if start_id is not None: audits = audits.filter(pk__lte=start_id) total_count = audits.count() if should_count else 0 audits = audits[:page_size] # Coerce the queryset into a list so we can get the last audit row on the # current page audits = list(audits) # We are using len(audits) instead of audits.count() because we # have already realized the queryset at this point if len(audits) == page_size: query_vars.setlist('prev', prev_start_ids + [audits[0].pk]) query_vars['start'] = audits[-1].pk - 1 next_page = "?" + query_vars.urlencode() else: next_page = None if prev_start_ids: if len(prev_start_ids) == 1: del query_vars['prev'] del query_vars['start'] else: prev_start_id = prev_start_ids.pop() query_vars.setlist('prev', prev_start_ids) query_vars['start'] = prev_start_id prev_page = "?" + query_vars.urlencode() else: prev_page = None return { 'audits': audits, 'total_count': total_count, 'next_page': next_page, 'prev_page': prev_page }
def setup_env(self, *args, **options): """ Create some seed data """ if options['instance']: instance = Instance.objects.get(pk=options['instance']) elif options['instance_url_name']: instance = Instance.objects.get( url_name=options['instance_url_name']) else: raise Exception("must provide instance") try: user = User.system_user() except User.DoesNotExist: self.stdout.write('Error: Could not find a superuser to use') return 1 instance_user = user.get_instance_user(instance) if instance_user is None: r = Role.objects.get_or_create(name=Role.ADMINISTRATOR, rep_thresh=0, instance=instance, default_permission_level=3) instance_user = InstanceUser(instance=instance, user=user, role=r[0]) instance_user.save_with_user(user) self.stdout.write( 'Added system user to instance with ADMINISTRATOR role') add_default_permissions(instance) if options.get('delete', False): # Can't delete through the ORM because it will pull all the data # into memory for signal handlers, then run out of memory and crash # BUT... cascading delete is not handled at the DB level, so we # need to delete from all related tables in the right order n_photo = MapFeaturePhoto.objects.filter(instance=instance).count() with connection.cursor() as cursor: cursor.execute( """ DELETE FROM treemap_treephoto t WHERE t.mapfeaturephoto_ptr_id IN (SELECT id FROM treemap_mapfeaturephoto p WHERE p.instance_id = %s) """, (instance.pk, )) with connection.cursor() as cursor: cursor.execute( 'DELETE FROM treemap_mapfeaturephoto t WHERE t.instance_id = %s', # NOQA (instance.pk, )) self.stdout.write("Deleted %s photos" % n_photo) n_trees = Tree.objects.filter(instance=instance).count() with connection.cursor() as cursor: cursor.execute( 'DELETE FROM treemap_tree t WHERE t.instance_id = %s', (instance.pk, )) self.stdout.write("Deleted %s trees" % n_trees) n_favorites = Favorite.objects \ .filter(map_feature__instance=instance).count() with connection.cursor() as cursor: cursor.execute( """ DELETE FROM treemap_favorite f WHERE f.map_feature_id IN (SELECT id FROM treemap_mapfeature m WHERE m.instance_id = %s) """, (instance.pk, )) self.stdout.write("Deleted %s favorites" % n_favorites) n_comments = EnhancedThreadedComment.objects \ .filter(instance=instance).count() with connection.cursor() as cursor: cursor.execute( """ DELETE FROM otm_comments_enhancedthreadedcommentflag f WHERE f.comment_id IN (SELECT threadedcomment_ptr_id FROM otm_comments_enhancedthreadedcomment c WHERE c.instance_id = %s) """, (instance.pk, )) with connection.cursor() as cursor: cursor.execute( """ DELETE FROM otm_comments_enhancedthreadedcomment c WHERE c.instance_id = %s """, (instance.pk, )) self.stdout.write("Deleted %s comments" % n_comments) n_rows = TreeImportRow.objects \ .filter(plot__instance=instance).count() with connection.cursor() as cursor: cursor.execute( """ UPDATE importer_treeimportrow r SET plot_id = NULL WHERE r.import_event_id IN (SELECT id FROM importer_treeimportevent e WHERE e.instance_id = %s) """, (instance.pk, )) self.stdout.write("Nulled out plot in %s import rows" % n_rows) n_features = MapFeature.objects.filter(instance=instance).count() with connection.cursor() as cursor: cursor.execute( """ DELETE FROM treemap_plot p WHERE p.mapfeature_ptr_id IN (SELECT id FROM treemap_mapfeature f WHERE f.instance_id = %s) """, (instance.pk, )) with connection.cursor() as cursor: cursor.execute( """ DELETE FROM stormwater_bioswale b WHERE b.polygonalmapfeature_ptr_id IN (SELECT id FROM treemap_mapfeature f WHERE f.instance_id = %s) """, (instance.pk, )) with connection.cursor() as cursor: cursor.execute( """ DELETE FROM stormwater_raingarden b WHERE b.polygonalmapfeature_ptr_id IN (SELECT id FROM treemap_mapfeature f WHERE f.instance_id = %s) """, (instance.pk, )) with connection.cursor() as cursor: cursor.execute( """ DELETE FROM stormwater_rainbarrel b WHERE b.mapfeature_ptr_id IN (SELECT id FROM treemap_mapfeature f WHERE f.instance_id = %s) """, (instance.pk, )) with connection.cursor() as cursor: cursor.execute( """ DELETE FROM stormwater_polygonalmapfeature b WHERE b.mapfeature_ptr_id IN (SELECT id FROM treemap_mapfeature f WHERE f.instance_id = %s) """, (instance.pk, )) with connection.cursor() as cursor: cursor.execute( 'DELETE FROM treemap_mapfeature f WHERE f.instance_id = %s', # NOQA (instance.pk, )) self.stdout.write("Deleted %s map features" % n_features) n_audits = Audit.objects.filter(instance=instance).count() with connection.cursor() as cursor: cursor.execute( """ DELETE FROM treemap_audit a WHERE a.instance_id = %s AND a.model NOT IN ('InstanceUser', 'Species', 'ITreeCodeOverride', 'EnhancedInstance') """, (instance.pk, )) self.stdout.write("Deleted %s audits" % n_audits) instance.update_revs('geo_rev', 'eco_rev', 'universal_rev') return instance, user
def setup_env(self, *args, **options): """ Create some seed data """ if options['instance']: instance = Instance.objects.get(pk=options['instance']) elif options['instance_url_name']: instance = Instance.objects.get( url_name=options['instance_url_name']) else: raise Exception("must provide instance") try: user = User.system_user() except User.DoesNotExist: self.stdout.write('Error: Could not find a superuser to use') return 1 instance_user = user.get_instance_user(instance) if instance_user is None: r = Role.objects.get_or_create(name=Role.ADMINISTRATOR, rep_thresh=0, instance=instance, default_permission=3) instance_user = InstanceUser(instance=instance, user=user, role=r[0]) instance_user.save_with_user(user) self.stdout.write( 'Added system user to instance with ADMINISTRATOR role') add_default_permissions(instance) if options.get('delete', False): # Can't delete through the ORM because it will pull all the data # into memory for signal handlers, then run out of memory and crash # BUT... cascading delete is not handled at the DB level, so we # need to delete from all related tables in the right order n_photo = MapFeaturePhoto.objects.filter(instance=instance).count() with connection.cursor() as cursor: cursor.execute( """ DELETE FROM treemap_treephoto t WHERE t.mapfeaturephoto_ptr_id IN (SELECT id FROM treemap_mapfeaturephoto p WHERE p.instance_id = %s) """, (instance.pk,)) with connection.cursor() as cursor: cursor.execute( 'DELETE FROM treemap_mapfeaturephoto t WHERE t.instance_id = %s', # NOQA (instance.pk,)) self.stdout.write("Deleted %s photos" % n_photo) n_trees = Tree.objects.filter(instance=instance).count() with connection.cursor() as cursor: cursor.execute( 'DELETE FROM treemap_tree t WHERE t.instance_id = %s', (instance.pk,)) self.stdout.write("Deleted %s trees" % n_trees) n_favorites = Favorite.objects \ .filter(map_feature__instance=instance).count() with connection.cursor() as cursor: cursor.execute( """ DELETE FROM treemap_favorite f WHERE f.map_feature_id IN (SELECT id FROM treemap_mapfeature m WHERE m.instance_id = %s) """, (instance.pk,)) self.stdout.write("Deleted %s favorites" % n_favorites) n_comments = EnhancedThreadedComment.objects \ .filter(instance=instance).count() with connection.cursor() as cursor: cursor.execute( """ DELETE FROM otm_comments_enhancedthreadedcommentflag f WHERE f.comment_id IN (SELECT threadedcomment_ptr_id FROM otm_comments_enhancedthreadedcomment c WHERE c.instance_id = %s) """, (instance.pk,)) with connection.cursor() as cursor: cursor.execute( """ DELETE FROM otm_comments_enhancedthreadedcomment c WHERE c.instance_id = %s """, (instance.pk,)) self.stdout.write("Deleted %s comments" % n_comments) n_rows = TreeImportRow.objects \ .filter(plot__instance=instance).count() with connection.cursor() as cursor: cursor.execute( """ UPDATE importer_treeimportrow r SET plot_id = NULL WHERE r.import_event_id IN (SELECT id FROM importer_treeimportevent e WHERE e.instance_id = %s) """, (instance.pk,)) self.stdout.write("Nulled out plot in %s import rows" % n_rows) n_features = MapFeature.objects.filter(instance=instance).count() with connection.cursor() as cursor: cursor.execute( """ DELETE FROM treemap_plot p WHERE p.mapfeature_ptr_id IN (SELECT id FROM treemap_mapfeature f WHERE f.instance_id = %s) """, (instance.pk,)) with connection.cursor() as cursor: cursor.execute( """ DELETE FROM stormwater_bioswale b WHERE b.polygonalmapfeature_ptr_id IN (SELECT id FROM treemap_mapfeature f WHERE f.instance_id = %s) """, (instance.pk,)) with connection.cursor() as cursor: cursor.execute( """ DELETE FROM stormwater_raingarden b WHERE b.polygonalmapfeature_ptr_id IN (SELECT id FROM treemap_mapfeature f WHERE f.instance_id = %s) """, (instance.pk,)) with connection.cursor() as cursor: cursor.execute( """ DELETE FROM stormwater_rainbarrel b WHERE b.mapfeature_ptr_id IN (SELECT id FROM treemap_mapfeature f WHERE f.instance_id = %s) """, (instance.pk,)) with connection.cursor() as cursor: cursor.execute( """ DELETE FROM stormwater_polygonalmapfeature b WHERE b.mapfeature_ptr_id IN (SELECT id FROM treemap_mapfeature f WHERE f.instance_id = %s) """, (instance.pk,)) with connection.cursor() as cursor: cursor.execute( 'DELETE FROM treemap_mapfeature f WHERE f.instance_id = %s', # NOQA (instance.pk,)) self.stdout.write("Deleted %s map features" % n_features) n_audits = Audit.objects.filter(instance=instance).count() with connection.cursor() as cursor: cursor.execute( """ DELETE FROM treemap_audit a WHERE a.instance_id = %s AND a.model NOT IN ('InstanceUser', 'Species', 'ITreeCodeOverride', 'EnhancedInstance') """, (instance.pk,)) self.stdout.write("Deleted %s audits" % n_audits) return instance, user
def get_audits(logged_in_user, instance, query_vars, user, models, model_id, page=0, page_size=20, exclude_pending=True, should_count=False): start_pos = page * page_size end_pos = start_pos + page_size if instance: if instance.is_accessible_by(logged_in_user): instances = Instance.objects.filter(pk=instance.pk) else: instances = [] # If we didn't specify an instance we only want to # show audits where the user has permission else: instances = Instance.objects.filter( user_accessible_instance_filter(logged_in_user)) if len(instances) == 0: # Force no results return {'audits': [], 'total_count': 0, 'next_page': None, 'prev_page': None} map_feature_models = set(MapFeature.subclass_dict().keys()) model_filter = Q() # We only want to show the TreePhoto's image, not other fields # and we want to do it automatically if 'Tree' was specified as # a model. The same goes for MapFeature(s) <-> MapFeaturePhoto # There is no need to check permissions, because photos are always visible if 'Tree' in models: model_filter = model_filter | Q(model='TreePhoto', field='image') if map_feature_models.intersection(models): model_filter = model_filter | Q(model='MapFeaturePhoto', field='image') if logged_in_user == user: # The logged-in user can see all their own edits model_filter = model_filter | \ Q(model__in=models) | Q(model__startswith='udf:') else: # Filter other users' edits by their visibility to the logged-in user for inst in instances: for model in models: ModelClass = get_auditable_class(model) if issubclass(ModelClass, Authorizable): fake_model = ModelClass(instance=inst) visible_fields = fake_model.visible_fields(logged_in_user) model_filter = model_filter |\ Q(model=model, field__in=visible_fields, instance=inst) else: model_filter = model_filter | Q(model=model, instance=inst) # Add UDF collections related to model if model == 'Tree': fake_model = Tree(instance=inst) elif model == 'Plot': fake_model = Plot(instance=inst) else: continue model_collection_udfs_audit_names =\ fake_model.visible_collection_udfs_audit_names( logged_in_user) model_filter = model_filter |\ Q(model__in=model_collection_udfs_audit_names) udf_bookkeeping_fields = Q( model__startswith='udf:', field__in=('id', 'model_id', 'field_definition')) audits = Audit.objects \ .filter(model_filter) \ .filter(instance__in=instances) \ .exclude(udf_bookkeeping_fields) \ .exclude(user=User.system_user()) \ .order_by('-created', 'id') if user: audits = audits.filter(user=user) if model_id: audits = audits.filter(model_id=model_id) if exclude_pending: audits = audits.exclude(requires_auth=True, ref__isnull=True) total_count = audits.count() if should_count else 0 audits = audits[start_pos:end_pos] query_vars = {k: v for (k, v) in query_vars.iteritems() if k != 'page'} next_page = None prev_page = None if len(audits) == page_size: query_vars['page'] = page + 1 next_page = "?" + urllib.urlencode(query_vars) if page > 0: query_vars['page'] = page - 1 prev_page = "?" + urllib.urlencode(query_vars) return {'audits': audits, 'total_count': total_count, 'next_page': next_page, 'prev_page': prev_page}
def commit_row(self): is_valid = self.validate_row() if not is_valid or not self.merged: return # not ready to commit if self.status == SpeciesImportRow.SUCCESS: return # nothing changed so no need to commit # Get our data data = self.cleaned species_edited = False # Initially grab species from row if it exists and edit it species = self.species # If not specified create a new one if species is None: species = Species(instance=self.import_event.instance) # Convert units self.convert_units(data, { fields.species.MAX_DIAMETER: self.import_event.max_diameter_conversion_factor, fields.species.MAX_HEIGHT: self.import_event.max_tree_height_conversion_factor }) for modelkey, datakey in SpeciesImportRow.SPECIES_MAP.iteritems(): importdata = data.get(datakey, None) if importdata is not None: species_edited = True setattr(species, modelkey, importdata) # Set OTM code if missing and available if not species.otm_code: species_dict = species_for_scientific_name( species.genus, species.species, species.cultivar, species.other_part_of_name) if species_dict: species_edited = True species.otm_code = species_dict['otm_code'] if species_edited: species.save_with_system_user_bypass_auth() # Make i-Tree code override(s) if necessary if fields.species.ITREE_PAIRS in data: for region_code, itree_code in data[fields.species.ITREE_PAIRS]: if itree_code != species.get_itree_code(region_code): override = ITreeCodeOverride.objects.get_or_create( instance_species=species, region=ITreeRegion.objects.get(code=region_code), )[0] override.itree_code = itree_code override.save_with_user(User.system_user()) self.species = species self.status = SpeciesImportRow.SUCCESS self.save()
def get_audits(logged_in_user, instance, query_vars, user, models, model_id, page=0, page_size=20, exclude_pending=True, should_count=False): start_pos = page * page_size end_pos = start_pos + page_size if instance: if instance.is_accessible_by(logged_in_user): instances = Instance.objects.filter(pk=instance.pk) else: instances = Instance.objects.none() # If we didn't specify an instance we only want to # show audits where the user has permission else: instances = Instance.objects\ .filter(pk__in=_instance_ids_edited_by(user))\ .filter(user_accessible_instance_filter( logged_in_user))\ .distinct() if not instances.exists(): # Force no results return {'audits': Audit.objects.none(), 'total_count': 0, 'next_page': None, 'prev_page': None} map_feature_models = set(MapFeature.subclass_dict().keys()) model_filter = Q() # We only want to show the TreePhoto's image, not other fields # and we want to do it automatically if 'Tree' was specified as # a model. The same goes for MapFeature(s) <-> MapFeaturePhoto # There is no need to check permissions, because photos are always visible if 'Tree' in models: model_filter = model_filter | Q(model='TreePhoto', field='image') if map_feature_models.intersection(models): model_filter = model_filter | Q(model='MapFeaturePhoto', field='image') for inst in instances: eligible_models = ({'Tree', 'TreePhoto', 'MapFeaturePhoto'} | set(inst.map_feature_types)) & set(models) if logged_in_user == user: eligible_udfs = {'udf:%s' % udf.id for udf in udf_defs(inst) if udf.model_type in eligible_models and udf.iscollection} # The logged-in user can see all their own edits model_filter = model_filter | Q( instance=inst, model__in=(eligible_models | eligible_udfs)) else: # Filter other users' edits by their visibility to the # logged-in user for model in eligible_models: ModelClass = get_auditable_class(model) fake_model = ModelClass(instance=inst) if issubclass(ModelClass, Authorizable): visible_fields = fake_model.visible_fields(logged_in_user) model_filter = model_filter |\ Q(model=model, field__in=visible_fields, instance=inst) else: model_filter = model_filter | Q(model=model, instance=inst) if issubclass(ModelClass, UDFModel): model_collection_udfs_audit_names = ( fake_model.visible_collection_udfs_audit_names( logged_in_user)) model_filter = model_filter | ( Q(model__in=model_collection_udfs_audit_names)) udf_bookkeeping_fields = Q( model__startswith='udf:', field__in=('id', 'model_id', 'field_definition')) audits = (Audit.objects .filter(model_filter) .filter(instance__in=instances) .select_related('instance') .exclude(udf_bookkeeping_fields) .exclude(user=User.system_user()) .order_by('-created')) if user: audits = audits.filter(user=user) if model_id: audits = audits.filter(model_id=model_id) if exclude_pending: audits = audits.exclude(requires_auth=True, ref__isnull=True) total_count = audits.count() if should_count else 0 audits = audits[start_pos:end_pos] query_vars = {k: v for (k, v) in query_vars.iteritems() if k != 'page'} next_page = None prev_page = None # We are using len(audits) instead of audits.count() because we # have already realized the queryset at this point if len(audits) == page_size: query_vars['page'] = page + 1 next_page = "?" + urllib.urlencode(query_vars) if page > 0: query_vars['page'] = page - 1 prev_page = "?" + urllib.urlencode(query_vars) return {'audits': audits, 'total_count': total_count, 'next_page': next_page, 'prev_page': prev_page}
def get_audits(logged_in_user, instance, query_vars, user, models, model_id, page=0, page_size=20, exclude_pending=True, should_count=False): start_pos = page * page_size end_pos = start_pos + page_size if instance: if instance.is_accessible_by(logged_in_user): instances = Instance.objects.filter(pk=instance.pk) else: instances = Instance.objects.none() # If we didn't specify an instance we only want to # show audits where the user has permission else: instances = Instance.objects.filter( user_accessible_instance_filter(logged_in_user)) if not instances.exists(): # Force no results return { 'audits': Audit.objects.none(), 'total_count': 0, 'next_page': None, 'prev_page': None } map_feature_models = set(MapFeature.subclass_dict().keys()) model_filter = Q() # We only want to show the TreePhoto's image, not other fields # and we want to do it automatically if 'Tree' was specified as # a model. The same goes for MapFeature(s) <-> MapFeaturePhoto # There is no need to check permissions, because photos are always visible if 'Tree' in models: model_filter = model_filter | Q(model='TreePhoto', field='image') if map_feature_models.intersection(models): model_filter = model_filter | Q(model='MapFeaturePhoto', field='image') if logged_in_user == user: # The logged-in user can see all their own edits model_filter = model_filter | \ Q(model__in=models) | Q(model__startswith='udf:') else: # Filter other users' edits by their visibility to the logged-in user for inst in instances: for model in models: ModelClass = get_auditable_class(model) if issubclass(ModelClass, Authorizable): fake_model = ModelClass(instance=inst) visible_fields = fake_model.visible_fields(logged_in_user) model_filter = model_filter |\ Q(model=model, field__in=visible_fields, instance=inst) else: model_filter = model_filter | Q(model=model, instance=inst) # Add UDF collections related to model if model == 'Tree': fake_model = Tree(instance=inst) elif model == 'Plot': fake_model = Plot(instance=inst) else: continue model_collection_udfs_audit_names =\ fake_model.visible_collection_udfs_audit_names( logged_in_user) model_filter = model_filter |\ Q(model__in=model_collection_udfs_audit_names) udf_bookkeeping_fields = Q(model__startswith='udf:', field__in=('id', 'model_id', 'field_definition')) audits = (Audit.objects.filter(model_filter).filter( instance__in=instances).select_related('instance').exclude( udf_bookkeeping_fields).exclude(user=User.system_user()).order_by( '-created', 'id')) if user: audits = audits.filter(user=user) if model_id: audits = audits.filter(model_id=model_id) if exclude_pending: audits = audits.exclude(requires_auth=True, ref__isnull=True) total_count = audits.count() if should_count else 0 audits = audits[start_pos:end_pos] query_vars = {k: v for (k, v) in query_vars.iteritems() if k != 'page'} next_page = None prev_page = None if audits.count() == page_size: query_vars['page'] = page + 1 next_page = "?" + urllib.urlencode(query_vars) if page > 0: query_vars['page'] = page - 1 prev_page = "?" + urllib.urlencode(query_vars) return { 'audits': audits, 'total_count': total_count, 'next_page': next_page, 'prev_page': prev_page }
def get_audits(logged_in_user, instance, query_vars, user=None, models=ALLOWED_MODELS, model_id=None, start_id=None, prev_start_ids=[], page_size=PAGE_DEFAULT, exclude_pending=True, should_count=False): if instance: if instance.is_accessible_by(logged_in_user): instances = Instance.objects.filter(pk=instance.pk) else: instances = Instance.objects.none() # If we didn't specify an instance we only want to # show audits where the user has permission else: instances = Instance.objects\ .filter(user_accessible_instance_filter(logged_in_user)) if user: instances = instances.filter(pk__in=_instance_ids_edited_by(user)) instances = instances.distinct() if not instances.exists(): # Force no results return {'audits': Audit.objects.none(), 'total_count': 0, 'next_page': None, 'prev_page': None} map_feature_models = set(MapFeature.subclass_dict().keys()) model_filter = Q() # We only want to show the TreePhoto's image, not other fields # and we want to do it automatically if 'Tree' was specified as # a model. The same goes for MapFeature(s) <-> MapFeaturePhoto # There is no need to check permissions, because photos are always visible if 'Tree' in models: model_filter = model_filter | Q(model='TreePhoto', field='image') if map_feature_models.intersection(models): model_filter = model_filter | Q(model='MapFeaturePhoto', field='image') for inst in instances: eligible_models = ({'Tree', 'TreePhoto', 'MapFeaturePhoto'} | set(inst.map_feature_types)) & set(models) if logged_in_user == user: eligible_udfs = {'udf:%s' % udf.id for udf in udf_defs(inst) if udf.model_type in eligible_models and udf.iscollection} # The logged-in user can see all their own edits model_filter = model_filter | Q( instance=inst, model__in=(eligible_models | eligible_udfs)) else: # Filter other users' edits by their visibility to the # logged-in user for model in eligible_models: ModelClass = get_auditable_class(model) fake_model = ModelClass(instance=inst) if issubclass(ModelClass, Authorizable): visible_fields = fake_model.visible_fields(logged_in_user) model_filter = model_filter |\ Q(model=model, field__in=visible_fields, instance=inst) else: model_filter = model_filter | Q(model=model, instance=inst) if issubclass(ModelClass, UDFModel): model_collection_udfs_audit_names = ( fake_model.visible_collection_udfs_audit_names( logged_in_user)) model_filter = model_filter | ( Q(model__in=model_collection_udfs_audit_names)) udf_bookkeeping_fields = Q( model__startswith='udf:', field__in=('id', 'model_id', 'field_definition')) audits = (Audit.objects .filter(model_filter) .filter(instance__in=instances) .select_related('instance') .exclude(udf_bookkeeping_fields) .exclude(user=User.system_user()) .order_by('-pk')) if user: audits = audits.filter(user=user) if model_id: audits = audits.filter(model_id=model_id) if exclude_pending: audits = audits.exclude(requires_auth=True, ref__isnull=True) # Slicing the QuerySet uses a SQL Limit, which has proven to be quite slow. # By relying on the fact the our list is ordered by primary key from newest # to oldest, we can rely on the index on the primary key, which is faster. if start_id is not None: audits = audits.filter(pk__lte=start_id) total_count = audits.count() if should_count else 0 audits = audits[:page_size] # Coerce the queryset into a list so we can get the last audit row on the # current page audits = list(audits) # We are using len(audits) instead of audits.count() because we # have already realized the queryset at this point if len(audits) == page_size: query_vars.setlist('prev', prev_start_ids + [audits[0].pk]) query_vars['start'] = audits[-1].pk - 1 next_page = "?" + query_vars.urlencode() else: next_page = None if prev_start_ids: if len(prev_start_ids) == 1: del query_vars['prev'] del query_vars['start'] else: prev_start_id = prev_start_ids.pop() query_vars.setlist('prev', prev_start_ids) query_vars['start'] = prev_start_id prev_page = "?" + query_vars.urlencode() else: prev_page = None return {'audits': audits, 'total_count': total_count, 'next_page': next_page, 'prev_page': prev_page}
def commit_row(self): is_valid = self.validate_row() if not is_valid or not self.merged: return # not ready to commit if self.status == SpeciesImportRow.SUCCESS: return # nothing changed so no need to commit # Get our data data = self.cleaned species_edited = False # Initially grab species from row if it exists and edit it species = self.species # If not specified create a new one if species is None: species = Species(instance=self.import_event.instance) # Convert units self.convert_units( data, { fields.species.MAX_DIAMETER: self.import_event.max_diameter_conversion_factor, fields.species.MAX_HEIGHT: self.import_event.max_tree_height_conversion_factor }) for modelkey, datakey in SpeciesImportRow.SPECIES_MAP.iteritems(): importdata = data.get(datakey, None) if importdata is not None: species_edited = True setattr(species, modelkey, importdata) # Set OTM code if missing and available if not species.otm_code: species_dict = species_for_scientific_name( species.genus, species.species, species.cultivar, species.other_part_of_name) if species_dict: species_edited = True species.otm_code = species_dict['otm_code'] if species_edited: species.save_with_system_user_bypass_auth() # Make i-Tree code override(s) if necessary if fields.species.ITREE_PAIRS in data: for region_code, itree_code in data[fields.species.ITREE_PAIRS]: if itree_code != species.get_itree_code(region_code): override = ITreeCodeOverride.objects.get_or_create( instance_species=species, region=ITreeRegion.objects.get(code=region_code), )[0] override.itree_code = itree_code override.save_with_user(User.system_user()) self.species = species self.status = SpeciesImportRow.SUCCESS self.save()
def user_roles_list(request, instance): page = int(request.GET.get('page', '1')) user_sort = request.GET.get('user_sort', 'user__username') invite_sort = request.GET.get('invite_sort', 'email') query = request.GET.get('query', '') def invite_context(invites): for invite in invites: yield { 'id': str(invite.pk), 'username': invite.email, 'role_id': invite.role.pk, 'role_name': invite.role.name, 'admin': invite.admin, } def instance_user_context(instance_users): for instance_user in paged_instance_users: user = instance_user.user yield { 'id': str(instance_user.pk), 'username': user.username, 'role_id': instance_user.role.pk, 'role_name': instance_user.role.name, 'admin': instance_user.admin, 'is_owner': does_user_own_instance(instance, user) } # The secondary sort on username/email is needed to ensure consistent # ordering within groupings. Testing shows that things work correctly when # the supplied sort order is also username/email invited = instance.instanceinvitation_set \ .select_related('role') \ .filter(accepted=False) \ .order_by(invite_sort, 'email') instance_users = instance.instanceuser_set \ .select_related('role', 'user') \ .exclude(user=User.system_user()) \ .order_by(user_sort, 'user__username') if query: instance_users = instance_users.filter(user__username__icontains=query) paginator = Paginator(instance_users, 15) urlizer = UrlParams('user_roles_partial', instance.url_name, page=page, invite_sort=invite_sort, user_sort=user_sort, query=query) try: paged_instance_users = paginator.page(page) except EmptyPage: # If the page number is out of bounds, return the last page paged_instance_users = paginator.page(paginator.num_pages) return { 'instance': instance, 'instance_users': instance_user_context(paged_instance_users), 'paged_instance_users': paged_instance_users, 'invited_users': invite_context(invited), 'instance_roles': Role.objects.filter(instance_id=instance.pk), 'page_url': urlizer.url('invite_sort', 'user_sort', 'query'), 'invite_sort_url': urlizer.url('page', 'user_sort', 'query'), 'user_sort_url': urlizer.url('invite_sort', 'query'), 'search_url': urlizer.url('invite_sort', 'user_sort'), 'invite_sort': invite_sort, 'user_sort': user_sort, }
def tearDown(self): self.instance.delete() self.user.delete_with_user(User.system_user())
def tearDown(self): self.instance.delete() self.user.delete_with_user(User.system_user()) super(TreemapUITestCase, self).tearDown()