def test_merge_model_with_o2m_relationship_and_raise_unique_validation( self): primary_object, alias_object = RestaurantFactory.create_batch(2) report = EarningsReportFactory(restaurant=primary_object) EarningsReportFactory(date=report.date, restaurant=alias_object) with pytest.raises(ValidationError): MergedModelInstance.create(primary_object, [alias_object], raise_validation_exception=True)
def handle(self, *args, **options): """ Handle the command (ie. execute it). """ primary_username = options["primary_username"] secondary_usernames = options["secondary_usernames"] self.stdout.write( self.style.WARNING( "Are you sure you want to merge users {secondary} into user {primary}? " "This will delete {secondary}. (y/n)".format( primary=primary_username, secondary=", ".join(secondary_usernames)))) if input("> ") != "y": self.stdout.write(self.style.WARNING("Cancelled.")) return # To avoid clashes in project names (which will cause them to be dropped) # check for duplicate project names attached to the users personal account # and append a unique string to any duplicates existing_names = Project.objects.filter( account__user__username=primary_username).values_list("name", flat=True) secondary_projects = Project.objects.filter( account__user__username__in=secondary_usernames) for project in secondary_projects: if project.name in existing_names: project.name += "-" + shortuuid.ShortUUID().random(length=8) project.save() # Merge the users' personal accounts primary_account = Account.objects.get(user__username=primary_username) secondary_accounts = Account.objects.filter( user__username__in=secondary_usernames) MergedModelInstance.create( primary_account, secondary_accounts, keep_old=False, ) # To avoid a user having more than one primary email, set all emails # for secondary users to primary=False EmailAddress.objects.filter( user__username__in=secondary_usernames).update(primary=False) # Merge the users primary_user = User.objects.get(username=primary_username) secondary_users = User.objects.filter(username__in=secondary_usernames) MergedModelInstance.create(primary_user, secondary_users, keep_old=False) self.stdout.write(self.style.SUCCESS("Succeeded."))
def test_merge_basic_model(self): primary_object = PlaceFactory.create(address=None) alias_object = PlaceFactory.create() merged_object = MergedModelInstance.create(primary_object, [alias_object]) assert merged_object.address == alias_object.address
def test_merge_model_with_multi_table_inheritance(self): primary_object = NewsAgencyFactory.create(address=None, website=None) alias_object = NewsAgencyFactory.create() merged_object = MergedModelInstance.create(primary_object, [alias_object]) assert merged_object.address == alias_object.address assert merged_object.website == alias_object.website
def test_dedupe_basic_model_no_merge(self): primary_object = PlaceFactory.create(address=None) alias_object = PlaceFactory.create() merged_object = MergedModelInstance.create(primary_object, [alias_object], merge_field_values=False) assert merged_object.address == primary_object.address
def test_merge_model_with_reverse_m2m_relationsip(self): primary_object = PublicationFactory.create() alias_object = PublicationFactory.create(number_of_articles=3) assert primary_object.article_set.count() == 0 merged_object = MergedModelInstance.create(primary_object, [alias_object]) assert merged_object.article_set.count() == 3
def test_merge_model_with_o2m_relationship(self): primary_object = NewsAgencyFactory.create() alias_object = NewsAgencyFactory.create() related_object = ReporterFactory.create(news_agency=alias_object) merged_object = MergedModelInstance.create(primary_object, [alias_object]) assert related_object.news_agency != merged_object related_object.refresh_from_db() assert related_object.news_agency == merged_object
def test_merge_generic_foreign_keys(self): primary_object = ArticleFactory() alias_object = ArticleFactory() primary_object.tags.create(content_object=primary_object, tag='django') alias_object.tags.create(content_object=alias_object, tag='python') merged_object = MergedModelInstance.create(primary_object, [alias_object], keep_old=False) assert merged_object.tags.count() == 2
def test_dedupe_model_with_o2o_relationship_no_merge(self): primary_object = RestaurantFactory.create(place=None, serves_hot_dogs=True, serves_pizza=False) alias_object = RestaurantFactory.create(serves_hot_dogs=False, serves_pizza=True) merged_object = MergedModelInstance.create(primary_object, [alias_object], merge_field_values=False) assert not merged_object.place assert merged_object.serves_hot_dogs and not merged_object.serves_pizza
def test_merge_model_with_m2m_relationship(self): primary_object = ArticleFactory.create(reporter=None) related_object = ReporterFactory.create() alias_object = ArticleFactory.create(number_of_publications=3, reporter=related_object) assert primary_object.reporter is None assert primary_object.publications.count() == 0 merged_object = MergedModelInstance.create(primary_object, [alias_object]) assert merged_object.reporter == related_object assert merged_object.publications.count() == 3
def test_merge_model_with_o2m_relationship_and_unique_validation_set_null( self): primary_object, alias_object = RestaurantFactory.create_batch(2) waiter = WaiterFactory(restaurant=primary_object) duplicate_waiter = WaiterFactory(name=waiter.name, restaurant=alias_object) merged_object = MergedModelInstance.create(primary_object, [alias_object]) waiter.refresh_from_db() assert waiter.restaurant == merged_object duplicate_waiter.refresh_from_db() assert duplicate_waiter.restaurant is None
def test_merge_model_with_o2o_relationship(self): primary_object = RestaurantFactory.create(place=None, serves_hot_dogs=True, serves_pizza=False) alias_object = RestaurantFactory.create(serves_hot_dogs=False, serves_pizza=True) alias_address = alias_object.place.address alias_name = alias_object.place.name merged_object = MergedModelInstance.create(primary_object, [alias_object]) assert merged_object.place.address == alias_address assert merged_object.place.name == alias_name assert merged_object.serves_hot_dogs and not merged_object.serves_pizza
def test_merge_deletes_alias_objects(self): primary_object = PlaceFactory.create(address=None) alias_object = PlaceFactory.create() assert primary_object.address is None assert PlaceFactory._meta.model.objects.filter( pk=alias_object.pk).exists() merged_object = MergedModelInstance.create(primary_object, [alias_object], keep_old=False) assert merged_object.address == alias_object.address assert not PlaceFactory._meta.model.objects.filter( pk=alias_object.pk).exists()
def test_merge_model_with_o2m_relationship_and_unique_validation_delete( self): primary_object, alias_object = RestaurantFactory.create_batch(2) report = EarningsReportFactory(restaurant=primary_object) other_report = EarningsReportFactory(date=report.date, restaurant=alias_object) merged_object = MergedModelInstance.create(primary_object, [alias_object]) report.refresh_from_db() assert report.restaurant == merged_object with pytest.raises(EarningsReportFactory._meta.model.DoesNotExist): other_report.refresh_from_db()
def handle(self, *args, **options): self.stdout.write(str(options)) args = AttrDict(options) has_param = args.resources or args.query or args.limit if args.resources: filt = Q() for r in args.resources: filt |= Q(host__iregex=r) resources = Resource.objects.filter(filt) else: resources = Resource.objects.filter(module__has_accounts_infos_update=True) countrier = Countrier() now = timezone.now() for resource in resources: with transaction.atomic(): plugin = self._get_plugin(resource.module) accounts = resource.account_set if args.query: accounts = accounts.filter(key__iregex=args.query) else: accounts = accounts.filter(Q(updated__isnull=True) | Q(updated__lte=now)) total = accounts.count() accounts = list(accounts[:args.limit]) if not accounts: continue try: with tqdm(total=len(accounts), desc=f'getting {resource.host} (total = {total})') as pbar: infos = plugin.Statistic.get_users_infos( users=[a.key for a in accounts], resource=resource, accounts=accounts, pbar=pbar, ) for account, data in zip(accounts, infos): info = data['info'] if info is None: _, info = account.delete() info = {k: v for k, v in info.items() if v} pbar.set_postfix(warning=f'Remove user {account} = {info}') continue contest_addition_update = data.pop('contest_addition_update', {}) contest_addition_update_by = data.pop('contest_addition_update_by', None) if contest_addition_update: account_update_contest_additions( account, contest_addition_update, timedelta_limit=timedelta(days=31) if account.info and not has_param else None, by=contest_addition_update_by, ) coders = data.pop('coders', []) if coders: qs = Coder.objects \ .filter(account__resource=resource, account__key__in=coders) \ .exclude(account=account) for c in qs: c.account_set.add(account) if 'rename' in data: other, created = Account.objects.get_or_create(resource=account.resource, key=data['rename']) if not created: new = MergedModelInstance.create(other, [account]) account.delete() else: new = MergedModelInstance.create(account, [other]) other.delete() account = new if info.get('country'): account.country = countrier.get(info['country']) if info.get('rating'): info['rating_ts'] = int(now.timestamp()) delta = info.pop('delta', timedelta(days=365)) account.info.update(info) account.updated = now + delta account.save() except Exception: if not has_param: for account in tqdm(accounts, desc='changing update time'): account.updated = now + timedelta(days=1) account.save() self.logger.error(format_exc()) self.logger.error(f'resource = {resource}')
def test_prevent_self_merge(self): primary_object = PlaceFactory.create(address=None) alias_object = primary_object with pytest.raises(ValueError): MergedModelInstance.create(primary_object, [alias_object])
def test_merge_different_models(self): primary_object = ArticleFactory.create() alias_object = ReporterFactory.create() with pytest.raises(TypeError): MergedModelInstance.create(primary_object, [alias_object])
def handle(self, *args, **options): self.stdout.write(str(options)) args = AttrDict(options) has_param = args.resources or args.query or args.limit if args.resources: filt = Q() for r in args.resources: filt |= Q(host__iregex=r) resources = Resource.objects.filter(filt) else: resources = Resource.objects.filter( module__has_accounts_infos_update=True) countrier = Countrier() now = timezone.now() for resource in resources: accounts = resource.account_set if args.query: accounts = accounts.filter(key__iregex=args.query) elif args.force: accounts = accounts.order_by('updated') else: accounts = accounts.filter( Q(updated__isnull=True) | Q(updated__lte=now)) count, total = 0, accounts.count() resource_info = resource.info.get('accounts', {}) if args.limit or not resource_info.get( 'nolimit', False) or resource_info.get('limit'): limit = resource_info.get('limit') or args.limit or 1000 accounts = accounts[:limit] accounts = list(accounts) if not accounts: continue try: with tqdm(total=len(accounts), desc=f'getting {resource.host} (total = {total})' ) as pbar: infos = resource.plugin.Statistic.get_users_infos( users=[a.key for a in accounts], resource=resource, accounts=accounts, pbar=pbar, ) for account, data in zip(accounts, infos): with transaction.atomic(): if data.get('skip'): continue count += 1 info = data['info'] if info is None: _, info = account.delete() info = {k: v for k, v in info.items() if v} pbar.set_postfix( warning=f'Remove user {account} = {info}') continue params = data.pop('contest_addition_update_params', {}) contest_addition_update = data.pop( 'contest_addition_update', params.pop('update', {})) contest_addition_update_by = data.pop( 'contest_addition_update_by', params.pop('by', None)) if contest_addition_update: account_update_contest_additions( account, contest_addition_update, timedelta_limit=timedelta(days=31) if account.info and not has_param else None, by=contest_addition_update_by, **params, ) if 'rename' in data: other, created = Account.objects.get_or_create( resource=account.resource, key=data['rename']) new = MergedModelInstance.create( other, [account]) account.delete() account = new account.save() coders = data.pop('coders', []) if coders: qs = Coder.objects \ .filter(account__resource=resource, account__key__in=coders) \ .exclude(account=account) for c in qs: account.coders.add(c) if info.get('country'): account.country = countrier.get( info['country']) if info.get('name'): account.name = info['name'] if 'rating' in info: info['rating_ts'] = int(now.timestamp()) delta = info.pop('delta', timedelta(days=365)) if data.get('replace_info'): for k, v in account.info.items(): if k.endswith('_') and k not in info: info[k] = v account.info = info else: account.info.update(info) account.updated = now + delta account.save() except Exception: if not has_param: for account in tqdm(accounts, desc='changing update time'): account.updated = now + timedelta(days=1) account.save() self.logger.error(format_exc()) self.logger.error(f'resource = {resource}') self.logger.info( f'Parsed accounts infos (resource = {resource}): {count} of {total}' )