def test_obj_id(self):
"""Test filter_unique with objects using id as key, which fails."""
# Two objects which may be equal do not necessary have the same id.
deduped = set()
deduper = tools.filter_unique(self.decs, container=deduped, key=id)
self.assertEqual(len(deduped), 0)
for _ in self.decs:
self.assertEqual(id(next(deduper)), deduped.pop())
self.assertRaises(StopIteration, next, deduper)
# No. of Decimal with distinct ids != no. of Decimal with distinct value.
deduper_ids = list(tools.filter_unique(self.decs, key=id))
self.assertNotEqual(len(deduper_ids), len(set(deduper_ids)))
def test_skip(self):
"""Test filter_unique with a container that skips items."""
deduped = SkipList()
deduper = tools.filter_unique(self.ints, container=deduped)
deduped_out = list(deduper)
self.assertCountEqual(deduped, deduped_out)
self.assertEqual(deduped, set([2, 4]))
def test_process_again(self):
"""Test filter_unique with an ignoring container."""
deduped = ProcessAgainList()
deduper = tools.filter_unique(self.ints, container=deduped)
deduped_out = list(deduper)
self.assertEqual(deduped_out, [1, 3, 2, 1, 1, 4])
self.assertEqual(deduped, set([2, 4]))
def test_for_resumable(self):
"""Test filter_unique is resumable after a for loop."""
gen2 = tools.filter_unique(self.ints)
deduped = []
for item in gen2:
deduped.append(item)
if len(deduped) == 3:
break
self.assertEqual(deduped, [1, 3, 2])
last = next(gen2)
self.assertEqual(last, 4)
self.assertRaises(StopIteration, next, gen2)
def test_for_resumable(self):
"""Test filter_unique is resumable after a for loop."""
gen2 = tools.filter_unique(self.ints)
deduped = []
for item in gen2:
deduped.append(item)
if len(deduped) == 3:
break
self.assertEqual(deduped, [1, 3, 2])
last = next(gen2)
self.assertEqual(last, 4)
self.assertRaises(StopIteration, next, gen2)
def test_stop(self):
"""Test filter_unique with an ignoring container."""
deduped = ContainsStopList()
deduped.stop_list = [2]
deduper = tools.filter_unique(self.ints, container=deduped)
deduped_out = list(deduper)
self.assertCountEqual(deduped, deduped_out)
self.assertEqual(deduped, {1, 3})
# And it should not resume
self.assertRaises(StopIteration, next, deduper)
deduped = AddStopList()
deduped.stop_list = [4]
deduper = tools.filter_unique(self.ints, container=deduped)
deduped_out = list(deduper)
self.assertCountEqual(deduped, deduped_out)
self.assertEqual(deduped, {1, 2, 3})
# And it should not resume
self.assertRaises(StopIteration, next, deduper)
def test_stop(self):
"""Test filter_unique with an ignoring container."""
deduped = ContainsStopList()
deduped.stop_list = [2]
deduper = tools.filter_unique(self.ints, container=deduped)
deduped_out = list(deduper)
self.assertCountEqual(deduped, deduped_out)
self.assertEqual(deduped, set([1, 3]))
# And it should not resume
self.assertRaises(StopIteration, next, deduper)
deduped = AddStopList()
deduped.stop_list = [4]
deduper = tools.filter_unique(self.ints, container=deduped)
deduped_out = list(deduper)
self.assertCountEqual(deduped, deduped_out)
self.assertEqual(deduped, set([1, 2, 3]))
# And it should not resume
self.assertRaises(StopIteration, next, deduper)
def test_OrderedDict(self):
"""Test filter_unique with an OrderedDict."""
deduped = OrderedDict()
deduper = tools.filter_unique(self.ints, container=deduped)
self._test_dedup_int(deduped, deduper)
def test_str_id(self):
"""Test str using id as key."""
deduped = set()
deduper = tools.filter_unique(self.strs, container=deduped, key=id)
self._test_dedup_str(deduped, deduper, id)
def test_dict(self):
"""Test filter_unique with a dict."""
deduped = {}
deduper = tools.filter_unique(self.ints, container=deduped)
self._test_dedup_int(deduped, deduper)
def test_obj_id(self):
"""Test filter_unique with objects using id as key, which fails."""
# Two objects which may be equal do not have the same id.
deduped = set()
deduper = tools.filter_unique(self.decs, container=deduped, key=id)
self._test_dedup_int(deduped, deduper, id)
def test_str_id(self):
"""Test str using id as key fails on Python 3."""
# str in Python 3 behave like objects.
deduped = set()
deduper = tools.filter_unique(self.strs, container=deduped, key=id)
self._test_dedup_str(deduped, deduper, id)
def test_str_hash(self):
"""Test filter_unique with str using hash as key."""
deduped = set()
deduper = tools.filter_unique(self.strs, container=deduped, key=hash)
self._test_dedup_str(deduped, deduper, hash)
def test_OrderedDict(self):
"""Test filter_unique with a OrderedDict."""
deduped = tools.OrderedDict()
deduper = tools.filter_unique(self.ints, container=deduped)
self._test_dedup_int(deduped, deduper)
def test_int_id(self):
"""Test filter_unique with ints using id as key."""
deduped = set()
deduper = tools.filter_unique(self.ints, container=deduped, key=id)
self._test_dedup_int(deduped, deduper, id)
def test_obj_hash(self):
"""Test filter_unique with objects using hash as key."""
deduped = set()
deduper = tools.filter_unique(self.decs, container=deduped, key=hash)
self._test_dedup_int(deduped, deduper, hash)
def test_str_hash(self):
"""Test filter_unique with str using hash as key."""
deduped = set()
deduper = tools.filter_unique(self.strs, container=deduped, key=hash)
self._test_dedup_str(deduped, deduper, hash)
def test_str_id(self):
"""Test str using id as key fails on Python 3."""
# str in Python 3 behave like objects.
deduped = set()
deduper = tools.filter_unique(self.strs, container=deduped, key=id)
self._test_dedup_str(deduped, deduper, id)
def test_str(self):
"""Test filter_unique with str."""
deduped = set()
deduper = tools.filter_unique(self.strs, container=deduped)
self._test_dedup_str(deduped, deduper)
def test_obj_id(self):
"""Test filter_unique with objects using id as key, which fails."""
# Two objects which may be equal do not have the same id.
deduped = set()
deduper = tools.filter_unique(self.decs, container=deduped, key=id)
self._test_dedup_int(deduped, deduper, id)
def test_obj_hash(self):
"""Test filter_unique with objects using hash as key."""
deduped = set()
deduper = tools.filter_unique(self.decs, container=deduped, key=hash)
self._test_dedup_int(deduped, deduper, hash)
def test_obj(self):
"""Test filter_unique with objects."""
deduped = set()
deduper = tools.filter_unique(self.decs, container=deduped)
self._test_dedup_int(deduped, deduper)
def test_int_id(self):
"""Test filter_unique with ints using id as key."""
deduped = set()
deduper = tools.filter_unique(self.ints, container=deduped, key=id)
self._test_dedup_int(deduped, deduper, id)
def test_obj(self):
"""Test filter_unique with objects."""
deduped = set()
deduper = tools.filter_unique(self.decs, container=deduped)
self._test_dedup_int(deduped, deduper)
def test_str_id(self):
"""Test str using id as key."""
deduped = set()
deduper = tools.filter_unique(self.strs, container=deduped, key=id)
self._test_dedup_str(deduped, deduper, id)
def test_str(self):
"""Test filter_unique with str."""
deduped = set()
deduper = tools.filter_unique(self.strs, container=deduped)
self._test_dedup_str(deduped, deduper)
def main(*args):
"""
Process command line arguments and invoke bot.
If args is an empty list, sys.argv is used.
@param args: command line arguments
@type args: str
"""
template_names = []
templates = {}
options = {}
# If xmlfilename is None, references will be loaded from the live wiki.
xmlfilename = None
user = None
skip = False
timestamp = None
# read command line parameters
local_args = pywikibot.handle_args(args)
# Avoid conflicts with pagegenerators.py parameters.
if any(arg.startswith('-category:') for arg in local_args):
warn('-category (to append a category to each edited page) has been'
' renamed to -addcat; make sure you are using the correct param.',
ArgumentDeprecationWarning)
site = pywikibot.Site()
gen_factory = pagegenerators.GeneratorFactory()
for arg in local_args:
if arg == '-remove':
options['remove'] = True
elif arg == '-subst':
options['subst'] = True
elif arg == '-assubst':
options['subst'] = options['remove'] = True
elif arg == '-always':
options['always'] = True
elif arg.startswith('-xml'):
if len(arg) == 4:
xmlfilename = pywikibot.input(
"Please enter the XML dump's filename: ")
else:
xmlfilename = arg[5:]
elif arg.startswith('-addcat:'):
options['addedCat'] = arg[len('-addcat:'):]
elif arg.startswith('-summary:'):
options['summary'] = arg[len('-summary:'):]
elif arg.startswith('-onlyuser:'******'-onlyuser:'******'-skipuser:'******'-skipuser:'******'-timestamp:'):
timestamp = arg[len('-timestamp:'):]
else:
if not gen_factory.handleArg(arg):
template_name = pywikibot.Page(site, arg, ns=10)
template_names.append(template_name.title(with_ns=False))
if not template_names:
pywikibot.bot.suggest_help(missing_parameters=['templates'])
return False
if options.get('subst', False) ^ options.get('remove', False):
for template_name in template_names:
templates[template_name] = None
else:
try:
for i in range(0, len(template_names), 2):
templates[template_names[i]] = template_names[i + 1]
except IndexError:
pywikibot.output('Unless using solely -subst or -remove, '
'you must give an even number of template names.')
return
old_templates = []
for template_name in templates.keys():
old_template = pywikibot.Page(site, template_name, ns=10)
old_templates.append(old_template)
if xmlfilename:
builder = textlib._MultiTemplateMatchBuilder(site)
predicate = builder.search_any_predicate(old_templates)
gen = XMLDumpPageGenerator(
xmlfilename, site=site, text_predicate=predicate)
else:
gen = gen_factory.getCombinedGenerator()
if not gen:
gens = (
pagegenerators.ReferringPageGenerator(t,
onlyTemplateInclusion=True)
for t in old_templates
)
gen = chain(*gens)
gen = filter_unique(gen, key=lambda p: '{}:{}:{}'.format(*p._cmpkey()))
if user:
gen = pagegenerators.UserEditFilterGenerator(gen, user, timestamp,
skip,
max_revision_depth=100,
show_filtered=True)
if not gen_factory.gens:
# make sure that proper namespace filtering etc. is handled
gen = gen_factory.getCombinedGenerator(gen)
if not gen_factory.nopreload:
gen = pagegenerators.PreloadingGenerator(gen)
bot = TemplateRobot(gen, templates, site=site, **options)
bot.run()
def main(*args: str) -> None:
"""
Process command line arguments and invoke bot.
If args is an empty list, sys.argv is used.
:param args: command line arguments
"""
template_names = []
templates = {}
options = {}
# If xmlfilename is None, references will be loaded from the live wiki.
xmlfilename = None
user = None
skip = False
timestamp = None
# read command line parameters
local_args = pywikibot.handle_args(args)
site = pywikibot.Site()
gen_factory = pagegenerators.GeneratorFactory()
for arg in local_args:
if arg == '-remove':
options['remove'] = True
elif arg.startswith('-subst'):
options['subst'] = arg[len('-subst:'):] + 'subst'
assert options['subst'] in ('subst', 'safesubst')
elif arg == '-assubst':
options['subst'] = 'subst'
options['remove'] = True
elif arg == '-always':
options['always'] = True
elif arg.startswith('-xml'):
if len(arg) == 4:
xmlfilename = pywikibot.input(
"Please enter the XML dump's filename: ")
else:
xmlfilename = arg[5:]
elif arg.startswith('-addcat:'):
options['addcat'] = arg[len('-addcat:'):]
elif arg.startswith('-summary:'):
options['summary'] = arg[len('-summary:'):]
elif arg.startswith('-onlyuser:'******'-onlyuser:'******'-skipuser:'******'-skipuser:'******'-timestamp:'):
timestamp = arg[len('-timestamp:'):]
else:
if not gen_factory.handle_arg(arg):
template_name = pywikibot.Page(site, arg, ns=10)
template_names.append(template_name.title(with_ns=False))
if not template_names:
pywikibot.bot.suggest_help(missing_parameters=['templates'])
return
if bool(options.get('subst', False)) ^ options.get('remove', False):
for template_name in template_names:
templates[template_name] = None
else:
try:
for i in range(0, len(template_names), 2):
templates[template_names[i]] = template_names[i + 1]
except IndexError:
pywikibot.output('Unless using solely -subst or -remove, '
'you must give an even number of template names.')
return
old_templates = [pywikibot.Page(site, template_name, ns=10)
for template_name in templates]
if xmlfilename:
builder = textlib.MultiTemplateMatchBuilder(site)
predicate = builder.search_any_predicate(old_templates)
gen = XMLDumpPageGenerator(
xmlfilename, site=site, text_predicate=predicate)
else:
gen = gen_factory.getCombinedGenerator()
if not gen:
gens = (
t.getReferences(only_template_inclusion=True,
follow_redirects=False)
for t in old_templates
)
gen = roundrobin_generators(*gens)
gen = filter_unique(gen, key=lambda p: '{}:{}:{}'.format(*p._cmpkey()))
if user:
gen = pagegenerators.UserEditFilterGenerator(gen, user, timestamp,
skip,
max_revision_depth=100,
show_filtered=True)
if not gen_factory.gens:
# make sure that proper namespace filtering etc. is handled
gen = gen_factory.getCombinedGenerator(gen)
if not gen_factory.nopreload:
gen = pagegenerators.PreloadingGenerator(gen)
bot = TemplateRobot(gen, templates, site=site, **options)
bot.run()
def test_set(self):
"""Test filter_unique with a set."""
deduped = set()
deduper = tools.filter_unique(self.ints, container=deduped)
self._test_dedup_int(deduped, deduper)
