def import_locations(config, file_name='locations.csv', overwrite=False, dry_run=False, quiet=False): """Import locations from CSV file provided by client.""" from arctasks.django import setup; setup(config) # noqa from ecoroofs.locations.importer import Importer location_importer = Importer(file_name, overwrite=overwrite, dry_run=dry_run, quiet=quiet) location_importer.run()
def copy_records(ctx, recreate_db=False): """Copy database records from old site. This is messy because only certain records are copied from the old site while others are loaded from fixtures. """ setup() from django.db import connections from django.contrib.auth import get_user_model from elasticmodels import suspended_updates settings = get_settings() settings.DATABASES['old'] = { 'ENGINE': 'django.contrib.gis.db.backends.postgis', 'HOST': 'pgsql.rc.pdx.edu', 'NAME': 'invhotline', 'USER': '******', 'PASSWORD': getpass( 'Old database password (get from ' '/vol/www/invasivespecieshotline/invasivespecieshotline/config/database.yml): ' ), } User = get_user_model() if recreate_db: createdb(ctx, drop=True) migrate(ctx) loaddata(ctx) with suspended_updates(): _copy_records(settings) default = connections['default'] print('Updating sequences...', end='') tables = ( 'category', 'comment', 'county', 'image', 'invite', 'notification', 'report', 'species', 'user', 'user_notification_query', ) statement = """ SELECT setval( pg_get_serial_sequence('{table}', '{table}_id'), coalesce(max({table}_id), 0) + 1, false ) FROM "{table}"; """ statements = ' '.join(statement.format(table=table) for table in tables) default.cursor().execute(statements) print('Done') print('Updating expert contact name...', end='') expert = User.objects.filter(first_name='EXPERT', last_name='CONTACT') expert.update(first_name='', last_name='') print('Done')
def import_neighborhoods(config, path='rlis/nbo_hood', from_srid=None, overwrite=True, dry_run=False, quiet=False): """Import neighborhoods from RLIS shapefile. We overwrite by default because doing so should be safe. The neighborhoods shapefile can be downloaded from Metro's RLIS Discovery site:: http://rlisdiscovery.oregonmetro.gov/?action=viewDetail&layerID=237 This task expects the shapefile directory to be at ``rlis/nbo_hood`` by default, but it can be located anywhere if you pass the corresponding ``--path`` option. """ from arctasks.django import setup; setup(config) # noqa from ecoroofs.neighborhoods.importer import Importer location_importer = Importer( path, from_srid=from_srid, overwrite=overwrite, dry_run=dry_run, quiet=quiet) location_importer.run()
def remove_duplicate_users(ctx): setup() from django.apps import apps from django.contrib.auth import get_user_model from arcutils.db import will_be_deleted_with Comment = apps.get_model('comments', 'Comment') Image = apps.get_model('images', 'Image') Notification = apps.get_model('notifications', 'Notification') UserNotificationQuery = apps.get_model('notifications', 'UserNotificationQuery') Invite = apps.get_model('reports', 'Invite') Report = apps.get_model('reports', 'Report') user_model = get_user_model() dupes = user_model.objects.raw( 'SELECT * from "user" u1 ' 'WHERE (' ' SELECT count(*) FROM "user" u2 WHERE lower(u2.email) = lower(u1.email )' ') > 1 ' 'ORDER BY lower(email)' ) dupes = [d for d in dupes] print_info('Found {n} duplicates'.format(n=len(dupes))) # Delete any dupes we can. # Active and staff users are never deleted. # Public users with no associated records will be deleted. for user in dupes: email = user.email objects = list(will_be_deleted_with(user)) num_objects = len(objects) f = locals() if user.is_active: print('Skipping active user: {email}.'.format_map(f)) elif user.is_staff: print('Skipping inactive staff user: {email}.'.format_map(f)) elif num_objects == 0: print_warning('Deleting {email} will *not* cascade.'.format_map(f)) if confirm(ctx, 'Delete {email}?'.format_map(f), yes_values=('yes',)): print('Okay, deleting {email}...'.format_map(f), end='') user.delete() dupes.remove(user) print('Deleted') else: print( 'Deleting {email} would cascade to {num_objects} objects. Skipping.'.format_map(f)) # Group the remaining duplicates by email address grouped_dupes = defaultdict(list) for user in dupes: email = user.email.lower() grouped_dupes[email].append(user) grouped_dupes = {email: users for (email, users) in grouped_dupes.items() if len(users) > 1} # For each group, find the "best" user (staff > active > inactive). # The other users' associated records will be associated with this # "winner". for email, users in grouped_dupes.items(): winner = None for user in users: if user.is_staff: winner = user break if winner is None: for user in users: if user.is_active: winner = user break if winner is None: for user in users: if user.full_name: winner = user if winner is None: winner = users[0] losers = [user for user in users if user != winner] print('Winner:', winner.full_name, '<{0.email}>'.format(winner)) for loser in losers: print('Loser:', loser.full_name, '<{0.email}>'.format(loser)) print('Re-associating loser objects...', end='') Comment.objects.filter(created_by=loser).update(created_by=winner) Image.objects.filter(created_by=loser).update(created_by=winner) Invite.objects.filter(user=loser).update(user=winner) Invite.objects.filter(created_by=loser).update(created_by=winner) Notification.objects.filter(user=loser).update(user=winner) Report.objects.filter(claimed_by=loser).update(claimed_by=winner) Report.objects.filter(created_by=loser).update(created_by=winner) UserNotificationQuery.objects.filter(user=loser).update(user=winner) print('Done')