def test_merge_model_with_o2m_relationship_and_raise_unique_validation(
self):
primary_object, alias_object = RestaurantFactory.create_batch(2)
report = EarningsReportFactory(restaurant=primary_object)
EarningsReportFactory(date=report.date, restaurant=alias_object)
with pytest.raises(ValidationError):
MergedModelInstance.create(primary_object, [alias_object],
raise_validation_exception=True)
def handle(self, *args, **options):
"""
Handle the command (ie. execute it).
"""
primary_username = options["primary_username"]
secondary_usernames = options["secondary_usernames"]
self.stdout.write(
self.style.WARNING(
"Are you sure you want to merge users {secondary} into user {primary}? "
"This will delete {secondary}. (y/n)".format(
primary=primary_username,
secondary=", ".join(secondary_usernames))))
if input("> ") != "y":
self.stdout.write(self.style.WARNING("Cancelled."))
return
# To avoid clashes in project names (which will cause them to be dropped)
# check for duplicate project names attached to the users personal account
# and append a unique string to any duplicates
existing_names = Project.objects.filter(
account__user__username=primary_username).values_list("name",
flat=True)
secondary_projects = Project.objects.filter(
account__user__username__in=secondary_usernames)
for project in secondary_projects:
if project.name in existing_names:
project.name += "-" + shortuuid.ShortUUID().random(length=8)
project.save()
# Merge the users' personal accounts
primary_account = Account.objects.get(user__username=primary_username)
secondary_accounts = Account.objects.filter(
user__username__in=secondary_usernames)
MergedModelInstance.create(
primary_account,
secondary_accounts,
keep_old=False,
)
# To avoid a user having more than one primary email, set all emails
# for secondary users to primary=False
EmailAddress.objects.filter(
user__username__in=secondary_usernames).update(primary=False)
# Merge the users
primary_user = User.objects.get(username=primary_username)
secondary_users = User.objects.filter(username__in=secondary_usernames)
MergedModelInstance.create(primary_user,
secondary_users,
keep_old=False)
self.stdout.write(self.style.SUCCESS("Succeeded."))
def merge_objects(request):
go_back = request.META.get('HTTP_REFERER')
if request.method == 'POST':
keep = request.POST.get("keep", None)
remove = request.POST.getlist("remove", None)
model_name = request.POST.get("model_name", None)
app_name = request.POST.get("app_name", None)
print('##############################')
print(keep, remove, model_name, app_name)
if keep and remove and model_name and app_name:
print('all good')
try:
ct = ContentType.objects.get(app_label=app_name,
model=model_name).model_class()
except ObjectDoesNotExist:
ct = None
if ct:
try:
keep_obj = ct.objects.filter(pk=keep)[0]
except IndexError:
print("No matching object to keep found")
return HttpResponseRedirect(go_back)
remove_objs = ct.objects.filter(pk__in=remove)
if len(remove_objs) > 0:
for x in remove_objs:
merged_object = MergedModelInstance(keep_obj,
x).merge(x)
x.delete()
print("merged {} into {}".format(x, keep_obj))
return HttpResponseRedirect(go_back)
else:
return HttpResponseRedirect(go_back)
def test_merge_basic_model(self):
primary_object = PlaceFactory.create(address=None)
alias_object = PlaceFactory.create()
merged_object = MergedModelInstance.create(primary_object,
[alias_object])
assert merged_object.address == alias_object.address
def test_reverse_m2m_merge_with_audit_trail(self):
primary_object = PublicationFactory.create()
alias_object = PublicationFactory.create(number_of_articles=3)
related_objects = set(alias_object.article_set.all())
_, audit_trail = MergedModelInstance.create_with_audit_trail(
primary_object, [alias_object])
assert set(audit_trail) == related_objects
def test_dedupe_basic_model_no_merge(self):
primary_object = PlaceFactory.create(address=None)
alias_object = PlaceFactory.create()
merged_object = MergedModelInstance.create(primary_object,
[alias_object],
merge_field_values=False)
assert merged_object.address == primary_object.address
def test_merge_model_with_multi_table_inheritance(self):
primary_object = NewsAgencyFactory.create(address=None, website=None)
alias_object = NewsAgencyFactory.create()
merged_object = MergedModelInstance.create(primary_object,
[alias_object])
assert merged_object.address == alias_object.address
assert merged_object.website == alias_object.website
def test_o2m_merge_with_audit_trail(self):
primary_object = NewsAgencyFactory.create()
alias_object = NewsAgencyFactory.create()
related_objects = set(
ReporterFactory.create_batch(3, news_agency=alias_object))
_, audit_trail = MergedModelInstance.create_with_audit_trail(
primary_object, [alias_object])
assert set(audit_trail) == related_objects
def test_merge_model_with_reverse_m2m_relationsip(self):
primary_object = PublicationFactory.create()
alias_object = PublicationFactory.create(number_of_articles=3)
assert primary_object.article_set.count() == 0
merged_object = MergedModelInstance.create(primary_object,
[alias_object])
assert merged_object.article_set.count() == 3
def test_merge_generic_foreign_keys(self):
primary_object = ArticleFactory()
alias_object = ArticleFactory()
primary_object.tags.create(content_object=primary_object, tag='django')
alias_object.tags.create(content_object=alias_object, tag='python')
merged_object = MergedModelInstance.create(primary_object,
[alias_object],
keep_old=False)
assert merged_object.tags.count() == 2
def test_m2m_merge_with_audit_trail(self):
primary_object = ArticleFactory.create(reporter=None)
related_object = ReporterFactory.create()
alias_object = ArticleFactory.create(number_of_publications=3,
reporter=related_object)
related_objects = set(alias_object.publications.all())
_, audit_trail = MergedModelInstance.create_with_audit_trail(
primary_object, [alias_object])
assert set(audit_trail) == related_objects
def test_o2o_merge_with_audit_trail(self):
primary_object = RestaurantFactory.create(place=None,
serves_hot_dogs=True,
serves_pizza=False)
alias_objects = RestaurantFactory.create_batch(3)
related_object = set([alias_objects[0].place])
_, audit_trail = MergedModelInstance.create_with_audit_trail(
primary_object, alias_objects)
assert set(audit_trail) == related_object
def test_merge_model_with_o2m_relationship(self):
primary_object = NewsAgencyFactory.create()
alias_object = NewsAgencyFactory.create()
related_object = ReporterFactory.create(news_agency=alias_object)
merged_object = MergedModelInstance.create(primary_object,
[alias_object])
assert related_object.news_agency != merged_object
related_object.refresh_from_db()
assert related_object.news_agency == merged_object
def test_dedupe_model_with_o2o_relationship_no_merge(self):
primary_object = RestaurantFactory.create(place=None,
serves_hot_dogs=True,
serves_pizza=False)
alias_object = RestaurantFactory.create(serves_hot_dogs=False,
serves_pizza=True)
merged_object = MergedModelInstance.create(primary_object,
[alias_object],
merge_field_values=False)
assert not merged_object.place
assert merged_object.serves_hot_dogs and not merged_object.serves_pizza
def test_merge_model_with_m2m_relationship(self):
primary_object = ArticleFactory.create(reporter=None)
related_object = ReporterFactory.create()
alias_object = ArticleFactory.create(number_of_publications=3,
reporter=related_object)
assert primary_object.reporter is None
assert primary_object.publications.count() == 0
merged_object = MergedModelInstance.create(primary_object,
[alias_object])
assert merged_object.reporter == related_object
assert merged_object.publications.count() == 3
def test_merge_model_with_o2m_relationship_and_unique_validation_set_null(
self):
primary_object, alias_object = RestaurantFactory.create_batch(2)
waiter = WaiterFactory(restaurant=primary_object)
duplicate_waiter = WaiterFactory(name=waiter.name,
restaurant=alias_object)
merged_object = MergedModelInstance.create(primary_object,
[alias_object])
waiter.refresh_from_db()
assert waiter.restaurant == merged_object
duplicate_waiter.refresh_from_db()
assert duplicate_waiter.restaurant is None
def test_merge_model_with_o2o_relationship(self):
primary_object = RestaurantFactory.create(place=None,
serves_hot_dogs=True,
serves_pizza=False)
alias_object = RestaurantFactory.create(serves_hot_dogs=False,
serves_pizza=True)
alias_address = alias_object.place.address
alias_name = alias_object.place.name
merged_object = MergedModelInstance.create(primary_object,
[alias_object])
assert merged_object.place.address == alias_address
assert merged_object.place.name == alias_name
assert merged_object.serves_hot_dogs and not merged_object.serves_pizza
def test_merge_deletes_alias_objects(self):
primary_object = PlaceFactory.create(address=None)
alias_object = PlaceFactory.create()
assert primary_object.address is None
assert PlaceFactory._meta.model.objects.filter(
pk=alias_object.pk).exists()
merged_object = MergedModelInstance.create(primary_object,
[alias_object],
keep_old=False)
assert merged_object.address == alias_object.address
assert not PlaceFactory._meta.model.objects.filter(
pk=alias_object.pk).exists()
def test_merge_model_with_o2m_relationship_and_unique_validation_delete(
self):
primary_object, alias_object = RestaurantFactory.create_batch(2)
report = EarningsReportFactory(restaurant=primary_object)
other_report = EarningsReportFactory(date=report.date,
restaurant=alias_object)
merged_object = MergedModelInstance.create(primary_object,
[alias_object])
report.refresh_from_db()
assert report.restaurant == merged_object
with pytest.raises(EarningsReportFactory._meta.model.DoesNotExist):
other_report.refresh_from_db()
def handle(self, *args, **options):
self.stdout.write(str(options))
args = AttrDict(options)
has_param = args.resources or args.query or args.limit
if args.resources:
filt = Q()
for r in args.resources:
filt |= Q(host__iregex=r)
resources = Resource.objects.filter(filt)
else:
resources = Resource.objects.filter(
module__has_accounts_infos_update=True)
countrier = Countrier()
now = timezone.now()
for resource in resources:
accounts = resource.account_set
if args.query:
accounts = accounts.filter(key__iregex=args.query)
elif args.force:
accounts = accounts.order_by('updated')
else:
accounts = accounts.filter(
Q(updated__isnull=True) | Q(updated__lte=now))
count, total = 0, accounts.count()
resource_info = resource.info.get('accounts', {})
if args.limit or not resource_info.get(
'nolimit', False) or resource_info.get('limit'):
limit = resource_info.get('limit') or args.limit or 1000
accounts = accounts[:limit]
accounts = list(accounts)
if not accounts:
continue
try:
with tqdm(total=len(accounts),
desc=f'getting {resource.host} (total = {total})'
) as pbar:
infos = resource.plugin.Statistic.get_users_infos(
users=[a.key for a in accounts],
resource=resource,
accounts=accounts,
pbar=pbar,
)
for account, data in zip(accounts, infos):
with transaction.atomic():
if data.get('skip'):
continue
count += 1
info = data['info']
if info is None:
_, info = account.delete()
info = {k: v for k, v in info.items() if v}
pbar.set_postfix(
warning=f'Remove user {account} = {info}')
continue
params = data.pop('contest_addition_update_params',
{})
contest_addition_update = data.pop(
'contest_addition_update',
params.pop('update', {}))
contest_addition_update_by = data.pop(
'contest_addition_update_by',
params.pop('by', None))
if contest_addition_update:
account_update_contest_additions(
account,
contest_addition_update,
timedelta_limit=timedelta(days=31) if
account.info and not has_param else None,
by=contest_addition_update_by,
**params,
)
if 'rename' in data:
other, created = Account.objects.get_or_create(
resource=account.resource,
key=data['rename'])
new = MergedModelInstance.create(
other, [account])
account.delete()
account = new
account.save()
coders = data.pop('coders', [])
if coders:
qs = Coder.objects \
.filter(account__resource=resource, account__key__in=coders) \
.exclude(account=account)
for c in qs:
account.coders.add(c)
if info.get('country'):
account.country = countrier.get(
info['country'])
if info.get('name'):
account.name = info['name']
if 'rating' in info:
info['rating_ts'] = int(now.timestamp())
delta = info.pop('delta', timedelta(days=365))
if data.get('replace_info'):
for k, v in account.info.items():
if k.endswith('_') and k not in info:
info[k] = v
account.info = info
else:
account.info.update(info)
account.updated = now + delta
account.save()
except Exception:
if not has_param:
for account in tqdm(accounts, desc='changing update time'):
account.updated = now + timedelta(days=1)
account.save()
self.logger.error(format_exc())
self.logger.error(f'resource = {resource}')
self.logger.info(
f'Parsed accounts infos (resource = {resource}): {count} of {total}'
)
def handle(self, *args, **options):
self.stdout.write(str(options))
args = AttrDict(options)
has_param = args.resources or args.query or args.limit
if args.resources:
filt = Q()
for r in args.resources:
filt |= Q(host__iregex=r)
resources = Resource.objects.filter(filt)
else:
resources = Resource.objects.filter(module__has_accounts_infos_update=True)
countrier = Countrier()
now = timezone.now()
for resource in resources:
with transaction.atomic():
plugin = self._get_plugin(resource.module)
accounts = resource.account_set
if args.query:
accounts = accounts.filter(key__iregex=args.query)
else:
accounts = accounts.filter(Q(updated__isnull=True) | Q(updated__lte=now))
total = accounts.count()
accounts = list(accounts[:args.limit])
if not accounts:
continue
try:
with tqdm(total=len(accounts), desc=f'getting {resource.host} (total = {total})') as pbar:
infos = plugin.Statistic.get_users_infos(
users=[a.key for a in accounts],
resource=resource,
accounts=accounts,
pbar=pbar,
)
for account, data in zip(accounts, infos):
info = data['info']
if info is None:
_, info = account.delete()
info = {k: v for k, v in info.items() if v}
pbar.set_postfix(warning=f'Remove user {account} = {info}')
continue
contest_addition_update = data.pop('contest_addition_update', {})
contest_addition_update_by = data.pop('contest_addition_update_by', None)
if contest_addition_update:
account_update_contest_additions(
account,
contest_addition_update,
timedelta_limit=timedelta(days=31) if account.info and not has_param else None,
by=contest_addition_update_by,
)
coders = data.pop('coders', [])
if coders:
qs = Coder.objects \
.filter(account__resource=resource, account__key__in=coders) \
.exclude(account=account)
for c in qs:
c.account_set.add(account)
if 'rename' in data:
other, created = Account.objects.get_or_create(resource=account.resource,
key=data['rename'])
if not created:
new = MergedModelInstance.create(other, [account])
account.delete()
else:
new = MergedModelInstance.create(account, [other])
other.delete()
account = new
if info.get('country'):
account.country = countrier.get(info['country'])
if info.get('rating'):
info['rating_ts'] = int(now.timestamp())
delta = info.pop('delta', timedelta(days=365))
account.info.update(info)
account.updated = now + delta
account.save()
except Exception:
if not has_param:
for account in tqdm(accounts, desc='changing update time'):
account.updated = now + timedelta(days=1)
account.save()
self.logger.error(format_exc())
self.logger.error(f'resource = {resource}')
def test_merge_different_models(self):
primary_object = ArticleFactory.create()
alias_object = ReporterFactory.create()
with pytest.raises(TypeError):
MergedModelInstance.create(primary_object, [alias_object])
def test_prevent_self_merge(self):
primary_object = PlaceFactory.create(address=None)
alias_object = primary_object
with pytest.raises(ValueError):
MergedModelInstance.create(primary_object, [alias_object])
