def __setup_stream(self):
logger.info('Building stream object')
stream = EventStreams(streams=['recentchange', 'revision-create'])
stream.register_filter(server_name='www.wikidata.org',
type='edit',
wiki='wikidatawiki')
logger.debug('Stream object built: {}'.format(stream))
self.__stream = stream
def run(self):
stream = EventStreams(stream='recentchange')
for event in stream:
if self.stop_event.isSet():
raise SystemExit
self.handler(event)
def test_url_parameter(self, key):
"""Test EventStreams with given url."""
e = EventStreams(url=self.sites[key]['hostname'])
self.assertEqual(e._url, self.sites[key]['hostname'])
self.assertEqual(e._url, e.url)
self.assertEqual(e._url, e.sse_kwargs.get('url'))
self.assertIsNone(e._total)
self.assertIsNone(e._stream)
def test_url_with_streams(self):
"""Test EventStreams with url from default site."""
streams = 'recentchange'
e = EventStreams(streams=streams)
self.assertEqual(e._url,
'https://stream.wikimedia.org/v2/stream/' + streams)
self.assertEqual(e._url, e.url)
self.assertEqual(e._url, e.sse_kwargs.get('url'))
self.assertIsNone(e._total)
self.assertEqual(e._streams, streams)
def test_url_from_site(self, key):
"""Test EventStreams with url from site."""
site = self.get_site(key)
stream = 'recentchanges'
e = EventStreams(site=site, stream=stream)
self.assertEqual(e._url,
'https://stream.wikimedia.org/v2/stream/' + stream)
self.assertEqual(e._url, e.url)
self.assertEqual(e._url, e.sse_kwargs.get('url'))
self.assertIsNone(e._total)
self.assertEqual(e._stream, stream)
def test_multiple_streams(self):
"""Test EventStreams with multiple streams."""
streams = ('page-create', 'page-move', 'page-delete')
e = EventStreams(streams=streams)
combined_streams = ','.join(streams)
self.assertEqual(
e._url,
'https://stream.wikimedia.org/v2/stream/' + combined_streams)
self.assertEqual(e._url, e.url)
self.assertEqual(e._url, e.sse_kwargs.get('url'))
self.assertEqual(e._streams, combined_streams)
def test_url_with_stream(self):
"""Test EventStreams with url from default site."""
site = self.get_site()
fam = site.family
if not isinstance(fam, WikimediaFamily):
self.skipTest(
"Family '{0}' of site '{1}' is not a WikimediaFamily.".format(
fam, site))
stream = 'recentchanges'
e = EventStreams(stream=stream)
self.assertEqual(e._url,
'https://stream.wikimedia.org/v2/stream/' + stream)
self.assertEqual(e._url, e.url)
self.assertEqual(e._url, e.sse_kwargs.get('url'))
self.assertIsNone(e._total)
self.assertEqual(e._stream, stream)
def setUp(self):
"""Set up unit test."""
super(TestEventStreamsSettingTests, self).setUp()
with mock.patch('pywikibot.comms.eventstreams.EventSource'):
self.es = EventStreams(url='dummy url')
class TestEventStreamsSettingTests(TestCase):
"""Setting tests for eventstreams module."""
dry = True
def setUp(self):
"""Set up unit test."""
super(TestEventStreamsSettingTests, self).setUp()
with mock.patch('pywikibot.comms.eventstreams.EventSource'):
self.es = EventStreams(url='dummy url')
def test_maximum_items(self):
"""Test EventStreams total value."""
total = 4711
self.es.set_maximum_items(total)
self.assertEqual(self.es._total, total)
def test_timeout_setting(self):
"""Test EventStreams timeout value."""
self.assertEqual(self.es.sse_kwargs.get('timeout'),
config.socket_timeout)
def test_filter_function_settings(self):
"""Test EventStreams filter function settings."""
def foo():
"""Dummy function."""
return True
self.es.register_filter(foo)
self.assertEqual(self.es.filter['all'][0], foo)
self.assertEqual(self.es.filter['any'], [])
self.assertEqual(self.es.filter['none'], [])
self.es.register_filter(foo, ftype='none')
self.assertEqual(self.es.filter['all'][0], foo)
self.assertEqual(self.es.filter['any'], [])
self.assertEqual(self.es.filter['none'][0], foo)
self.es.register_filter(foo, ftype='any')
self.assertEqual(self.es.filter['all'][0], foo)
self.assertEqual(self.es.filter['any'][0], foo)
self.assertEqual(self.es.filter['none'][0], foo)
def test_filter_function_settings_fail(self):
"""Test EventStreams failing filter function settings."""
with self.assertRaises(TypeError):
self.es.register_filter('test')
def test_filter_settings(self):
"""Test EventStreams filter settings."""
self.es.register_filter(foo='bar')
self.assertIsInstance(self.es.filter['all'][0], FunctionType)
self.es.register_filter(bar='baz')
self.assertEqual(len(self.es.filter['all']), 2)
def test_url_missing_stream(self):
"""Test EventStreams with url from site with missing stream."""
with self.assertRaises(NotImplementedError):
EventStreams()
def main_function():
query_timestamp = query_timestamp = (
datetime.datetime.utcnow() - datetime.timedelta(minutes=1)
).strftime(
'%Y%m%d%H%M%S') #change mintnues=1 to 5 for last 5 min wiki-changes
now = int(datetime.datetime.utcnow().strftime('%Y%m%d%H%M%S'))
wiki_data = []
domain_to_data = {}
stream = EventStreams(streams=['revision-create'], since=query_timestamp)
while True:
response = next(iter(stream))
# checking is user is bot or not , if not bot then only considering the response
if (response['performer']['user_is_bot'] != True):
wiki_data.append(response)
if response['meta']['domain'] in domain_to_data:
domain_to_data[response['meta']['domain']].append(response)
else:
domain_to_data[response['meta']['domain']] = [response]
res_time = response['rev_timestamp'].split('T')
res_time = int(''.join(res_time[0].split('-')) +
''.join(res_time[1][:-1].split(':')))
# fetching data since last minute of main_function executed
if res_time >= now:
break
# For displaying Domain Reports
d_t = {}
users_data = []
pages_count = 0
for domain in domain_to_data.keys():
pages = []
user_list = []
for reports in domain_to_data[domain]:
pages.append(reports['page_title'])
name = reports['performer']['user_text']
try:
e_count = reports['performer']['user_edit_count']
except:
# if user_edit_count doesn't exist , we counting as 0
e_count = 0
user_list.append([name, e_count])
users_data.append(user_list)
d_t[domain] = len(set(pages))
d_t = sorted(d_t.items(), key=lambda x: x[1], reverse=True)
print("Sample Reports")
print("Total number of Wikipedia Domains Updated ",
len(domain_to_data.keys()), '\n')
for i in d_t:
print(f'{i[0]}: {i[1]} pages updated ')
# For displaying User report
k = 0
print("\n\t\tUSER Report \n")
for domain_user in users_data:
print(f'\nUsers who updates to {list(domain_to_data.keys())[k]}\n')
user_to_count = {}
for user in domain_user:
if user[0] not in user_to_count:
user_to_count[user[0]] = user[1]
else:
user_to_count[user[0]] = user[1]
for name, count in user_to_count.items():
print(f'{name} : {count}')
k += 1
class TestEventStreamsFilterTests(TestCase):
"""Filter tests for eventstreams module."""
dry = True
data = {'foo': True, 'bar': 'baz'}
def setUp(self):
"""Set up unit test."""
super(TestEventStreamsFilterTests, self).setUp()
with mock.patch('pywikibot.comms.eventstreams.EventSource'):
self.es = EventStreams(url='dummy url')
def test_filter_function_all(self):
"""Test EventStreams filter all function."""
self.es.register_filter(lambda x: True)
self.assertTrue(self.es.streamfilter(self.data))
self.es.register_filter(lambda x: False)
self.assertFalse(self.es.streamfilter(self.data))
def test_filter_function_any(self):
"""Test EventStreams filter any function."""
self.es.register_filter(lambda x: True, ftype='any')
self.assertTrue(self.es.streamfilter(self.data))
self.es.register_filter(lambda x: False, ftype='any')
self.assertTrue(self.es.streamfilter(self.data))
def test_filter_function_none(self):
"""Test EventStreams filter none function."""
self.es.register_filter(lambda x: False, ftype='none')
self.assertTrue(self.es.streamfilter(self.data))
self.es.register_filter(lambda x: True, ftype='none')
self.assertFalse(self.es.streamfilter(self.data))
def _test_filter(self, none_type, all_type, any_type, result):
"""Test a single fixed filter."""
self.es.filter = {'all': [], 'any': [], 'none': []}
self.es.register_filter(lambda x: none_type, ftype='none')
self.es.register_filter(lambda x: all_type, ftype='all')
if any_type is not None:
self.es.register_filter(lambda x: any_type, ftype='any')
self.assertEqual(
self.es.streamfilter(self.data), result,
'Test EventStreams filter mixed function failed for\n'
"'none': {0}, 'all': {1}, 'any': {2}\n"
'(expected {3}, given {4})'.format(none_type, all_type, any_type,
result, not result))
def test_filter_mixed_function(self):
"""Test EventStreams filter mixed function."""
for none_type in (False, True):
for all_type in (False, True):
for any_type in (False, True, None):
if none_type is False and all_type is True and (
any_type is None or any_type is True):
result = True
else:
result = False
self._test_filter(none_type, all_type, any_type, result)
def get_stream(stream_date, stream_server):
stream = EventStreams(streams=["recentchange", "revision-create"],
since=stream_date)
stream.register_filter(server_name=stream_server, type="edit")
return stream
class TestEventStreamsFilter(TestCase):
"""Filter tests for eventstreams module."""
dry = True
data = {'foo': True, 'bar': 'baz'}
def setUp(self):
"""Set up unit test."""
super().setUp()
with mock.patch('pywikibot.comms.eventstreams.EventSource'):
self.es = EventStreams(url='dummy url')
def test_filter_function_all(self):
"""Test EventStreams filter all function."""
self.es.register_filter(lambda x: True)
self.assertTrue(self.es.streamfilter(self.data))
self.es.register_filter(lambda x: False)
self.assertFalse(self.es.streamfilter(self.data))
def test_filter_function_any(self):
"""Test EventStreams filter any function."""
self.es.register_filter(lambda x: True, ftype='any')
self.assertTrue(self.es.streamfilter(self.data))
self.es.register_filter(lambda x: False, ftype='any')
self.assertTrue(self.es.streamfilter(self.data))
def test_filter_function_none(self):
"""Test EventStreams filter none function."""
self.es.register_filter(lambda x: False, ftype='none')
self.assertTrue(self.es.streamfilter(self.data))
self.es.register_filter(lambda x: True, ftype='none')
self.assertFalse(self.es.streamfilter(self.data))
def test_filter_false(self):
"""Test EventStreams filter with assignment of True."""
self.es.register_filter(foo=False)
self.assertFalse(self.es.streamfilter(self.data))
def test_filter_true(self):
"""Test EventStreams filter with assignment of False."""
self.es.register_filter(foo=True)
self.assertTrue(self.es.streamfilter(self.data))
def test_filter_value(self):
"""Test EventStreams filter with assignment of an int value."""
self.es.register_filter(foo=10)
self.assertFalse(self.es.streamfilter(self.data))
def test_filter_sequence_false(self):
"""Test EventStreams filter with assignment of a sequence."""
self.es.register_filter(bar=list('baz'))
self.assertFalse(self.es.streamfilter(self.data))
def test_filter_sequence_true(self):
"""Test EventStreams filter with assignment of a sequence."""
self.es.register_filter(bar=('foo', 'bar', 'baz'))
self.assertTrue(self.es.streamfilter(self.data))
def test_filter_multiple(self):
"""Test EventStreams filter with multiple arguments."""
self.es.register_filter(foo=False, bar='baz')
self.assertFalse(self.es.streamfilter(self.data))
self.es.filter = {'all': [], 'any': [], 'none': []}
self.es.register_filter(foo=True, bar='baz')
self.assertTrue(self.es.streamfilter(self.data))
# check whether filter functions are different
f, g = self.es.filter['all']
c = {'foo': True}
self.assertNotEqual(f(c), g(c))
c = {'bar': 'baz'}
self.assertNotEqual(f(c), g(c))
def _test_filter(self, none_type, all_type, any_type, result):
"""Test a single fixed filter."""
self.es.filter = {'all': [], 'any': [], 'none': []}
self.es.register_filter(lambda x: none_type, ftype='none')
self.es.register_filter(lambda x: all_type, ftype='all')
if any_type is not None:
self.es.register_filter(lambda x: any_type, ftype='any')
self.assertEqual(
self.es.streamfilter(self.data), result,
'Test EventStreams filter mixed function failed for\n'
"'none': {}, 'all': {}, 'any': {}\n"
'(expected {}, given {})'.format(none_type, all_type, any_type,
result, not result))
def test_filter_mixed_function(self):
"""Test EventStreams filter mixed function."""
for none_type in (False, True):
for all_type in (False, True):
for any_type in (False, True, None):
result = none_type is False and all_type is True \
and (any_type is None or any_type is True)
self._test_filter(none_type, all_type, any_type, result)
def setUp(self):
"""Set up unit test."""
super(TestEventStreamsSettingTests, self).setUp()
with mock.patch('pywikibot.comms.eventstreams.EventSource'):
self.es = EventStreams(url='dummy url')
class TestEventStreamsSettingTests(TestCase):
"""Setting tests for eventstreams module."""
dry = True
def setUp(self):
"""Set up unit test."""
super(TestEventStreamsSettingTests, self).setUp()
with mock.patch('pywikibot.comms.eventstreams.EventSource'):
self.es = EventStreams(url='dummy url')
def test_maximum_items(self):
"""Test EventStreams total value."""
total = 4711
self.es.set_maximum_items(total)
self.assertEqual(self.es._total, total)
def test_timeout_setting(self):
"""Test EventStreams timeout value."""
self.assertEqual(self.es.sse_kwargs.get('timeout'),
config.socket_timeout)
def test_filter_function_settings(self):
"""Test EventStreams filter function settings."""
def foo():
"""Dummy function."""
return True
self.es.register_filter(foo)
self.assertEqual(self.es.filter['all'][0], foo)
self.assertEqual(self.es.filter['any'], [])
self.assertEqual(self.es.filter['none'], [])
self.es.register_filter(foo, ftype='none')
self.assertEqual(self.es.filter['all'][0], foo)
self.assertEqual(self.es.filter['any'], [])
self.assertEqual(self.es.filter['none'][0], foo)
self.es.register_filter(foo, ftype='any')
self.assertEqual(self.es.filter['all'][0], foo)
self.assertEqual(self.es.filter['any'][0], foo)
self.assertEqual(self.es.filter['none'][0], foo)
def test_filter_function_settings_fail(self):
"""Test EventStreams failing filter function settings."""
with self.assertRaises(TypeError):
self.es.register_filter('test')
def test_filter_settings(self):
"""Test EventStreams filter settings."""
self.es.register_filter(foo='bar')
self.assertIsInstance(self.es.filter['all'][0], FunctionType)
self.es.register_filter(bar='baz')
self.assertEqual(len(self.es.filter['all']), 2)
class TestEventStreamsFilterTests(TestCase):
"""Filter tests for eventstreams module."""
dry = True
data = {'foo': True, 'bar': 'baz'}
def setUp(self):
"""Set up unit test."""
super(TestEventStreamsFilterTests, self).setUp()
with mock.patch('pywikibot.comms.eventstreams.EventSource'):
self.es = EventStreams(url='dummy url')
def test_filter_function_all(self):
"""Test EventStreams filter all function."""
self.es.register_filter(lambda x: True)
self.assertTrue(self.es.streamfilter(self.data))
self.es.register_filter(lambda x: False)
self.assertFalse(self.es.streamfilter(self.data))
def test_filter_function_any(self):
"""Test EventStreams filter any function."""
self.es.register_filter(lambda x: True, ftype='any')
self.assertTrue(self.es.streamfilter(self.data))
self.es.register_filter(lambda x: False, ftype='any')
self.assertTrue(self.es.streamfilter(self.data))
def test_filter_function_none(self):
"""Test EventStreams filter none function."""
self.es.register_filter(lambda x: False, ftype='none')
self.assertTrue(self.es.streamfilter(self.data))
self.es.register_filter(lambda x: True, ftype='none')
self.assertFalse(self.es.streamfilter(self.data))
def test_filter_false(self):
"""Test EventStreams filter with assignment of True."""
self.es.register_filter(foo=False)
self.assertFalse(self.es.streamfilter(self.data))
def test_filter_true(self):
"""Test EventStreams filter with assignment of False."""
self.es.register_filter(foo=True)
self.assertTrue(self.es.streamfilter(self.data))
def test_filter_value(self):
"""Test EventStreams filter with assignment of a int value."""
self.es.register_filter(foo=10)
self.assertFalse(self.es.streamfilter(self.data))
def _test_filter(self, none_type, all_type, any_type, result):
"""Test a single fixed filter."""
self.es.filter = {'all': [], 'any': [], 'none': []}
self.es.register_filter(lambda x: none_type, ftype='none')
self.es.register_filter(lambda x: all_type, ftype='all')
if any_type is not None:
self.es.register_filter(lambda x: any_type, ftype='any')
self.assertEqual(self.es.streamfilter(self.data), result,
'Test EventStreams filter mixed function failed for\n'
"'none': {0}, 'all': {1}, 'any': {2}\n"
'(expected {3}, given {4})'
.format(none_type, all_type, any_type,
result, not result))
def test_filter_mixed_function(self):
"""Test EventStreams filter mixed function."""
for none_type in (False, True):
for all_type in (False, True):
for any_type in (False, True, None):
if none_type is False and all_type is True and (
any_type is None or any_type is True):
result = True
else:
result = False
self._test_filter(none_type, all_type, any_type, result)
from botbase import *
from pywikibot.comms.eventstreams import EventStreams # pylint: disable=import-error
def sign_diff(old_id, new_id):
"""
Creates diff based on revisions and determines based on diff whether
something needs to be signed, and where to insert sign
"""
# in reality, all namespaces except for main are valid
# need to check category to find wikipedia space pages that need signing
valid_namespaces = (1, 3, 5, 7, 9, 11, 13, 15, 101, 109, 119, 711, 829, 2301, 2303)
stream = EventStreams(streams="recentchange")
stream.register_filter(
type="edit", wiki="enwiki", bot=False, namespace=valid_namespaces
)
for edit in stream:
print(edit)