def __setup_stream(self): logger.info('Building stream object') stream = EventStreams(streams=['recentchange', 'revision-create']) stream.register_filter(server_name='www.wikidata.org', type='edit', wiki='wikidatawiki') logger.debug('Stream object built: {}'.format(stream)) self.__stream = stream
def run(self): stream = EventStreams(stream='recentchange') for event in stream: if self.stop_event.isSet(): raise SystemExit self.handler(event)
def test_url_parameter(self, key): """Test EventStreams with given url.""" e = EventStreams(url=self.sites[key]['hostname']) self.assertEqual(e._url, self.sites[key]['hostname']) self.assertEqual(e._url, e.url) self.assertEqual(e._url, e.sse_kwargs.get('url')) self.assertIsNone(e._total) self.assertIsNone(e._stream)
def test_url_with_streams(self): """Test EventStreams with url from default site.""" streams = 'recentchange' e = EventStreams(streams=streams) self.assertEqual(e._url, 'https://stream.wikimedia.org/v2/stream/' + streams) self.assertEqual(e._url, e.url) self.assertEqual(e._url, e.sse_kwargs.get('url')) self.assertIsNone(e._total) self.assertEqual(e._streams, streams)
def test_url_from_site(self, key): """Test EventStreams with url from site.""" site = self.get_site(key) stream = 'recentchanges' e = EventStreams(site=site, stream=stream) self.assertEqual(e._url, 'https://stream.wikimedia.org/v2/stream/' + stream) self.assertEqual(e._url, e.url) self.assertEqual(e._url, e.sse_kwargs.get('url')) self.assertIsNone(e._total) self.assertEqual(e._stream, stream)
def test_multiple_streams(self): """Test EventStreams with multiple streams.""" streams = ('page-create', 'page-move', 'page-delete') e = EventStreams(streams=streams) combined_streams = ','.join(streams) self.assertEqual( e._url, 'https://stream.wikimedia.org/v2/stream/' + combined_streams) self.assertEqual(e._url, e.url) self.assertEqual(e._url, e.sse_kwargs.get('url')) self.assertEqual(e._streams, combined_streams)
def test_url_with_stream(self): """Test EventStreams with url from default site.""" site = self.get_site() fam = site.family if not isinstance(fam, WikimediaFamily): self.skipTest( "Family '{0}' of site '{1}' is not a WikimediaFamily.".format( fam, site)) stream = 'recentchanges' e = EventStreams(stream=stream) self.assertEqual(e._url, 'https://stream.wikimedia.org/v2/stream/' + stream) self.assertEqual(e._url, e.url) self.assertEqual(e._url, e.sse_kwargs.get('url')) self.assertIsNone(e._total) self.assertEqual(e._stream, stream)
def setUp(self): """Set up unit test.""" super(TestEventStreamsSettingTests, self).setUp() with mock.patch('pywikibot.comms.eventstreams.EventSource'): self.es = EventStreams(url='dummy url')
class TestEventStreamsSettingTests(TestCase): """Setting tests for eventstreams module.""" dry = True def setUp(self): """Set up unit test.""" super(TestEventStreamsSettingTests, self).setUp() with mock.patch('pywikibot.comms.eventstreams.EventSource'): self.es = EventStreams(url='dummy url') def test_maximum_items(self): """Test EventStreams total value.""" total = 4711 self.es.set_maximum_items(total) self.assertEqual(self.es._total, total) def test_timeout_setting(self): """Test EventStreams timeout value.""" self.assertEqual(self.es.sse_kwargs.get('timeout'), config.socket_timeout) def test_filter_function_settings(self): """Test EventStreams filter function settings.""" def foo(): """Dummy function.""" return True self.es.register_filter(foo) self.assertEqual(self.es.filter['all'][0], foo) self.assertEqual(self.es.filter['any'], []) self.assertEqual(self.es.filter['none'], []) self.es.register_filter(foo, ftype='none') self.assertEqual(self.es.filter['all'][0], foo) self.assertEqual(self.es.filter['any'], []) self.assertEqual(self.es.filter['none'][0], foo) self.es.register_filter(foo, ftype='any') self.assertEqual(self.es.filter['all'][0], foo) self.assertEqual(self.es.filter['any'][0], foo) self.assertEqual(self.es.filter['none'][0], foo) def test_filter_function_settings_fail(self): """Test EventStreams failing filter function settings.""" with self.assertRaises(TypeError): self.es.register_filter('test') def test_filter_settings(self): """Test EventStreams filter settings.""" self.es.register_filter(foo='bar') self.assertIsInstance(self.es.filter['all'][0], FunctionType) self.es.register_filter(bar='baz') self.assertEqual(len(self.es.filter['all']), 2)
def test_url_missing_stream(self): """Test EventStreams with url from site with missing stream.""" with self.assertRaises(NotImplementedError): EventStreams()
def main_function(): query_timestamp = query_timestamp = ( datetime.datetime.utcnow() - datetime.timedelta(minutes=1) ).strftime( '%Y%m%d%H%M%S') #change mintnues=1 to 5 for last 5 min wiki-changes now = int(datetime.datetime.utcnow().strftime('%Y%m%d%H%M%S')) wiki_data = [] domain_to_data = {} stream = EventStreams(streams=['revision-create'], since=query_timestamp) while True: response = next(iter(stream)) # checking is user is bot or not , if not bot then only considering the response if (response['performer']['user_is_bot'] != True): wiki_data.append(response) if response['meta']['domain'] in domain_to_data: domain_to_data[response['meta']['domain']].append(response) else: domain_to_data[response['meta']['domain']] = [response] res_time = response['rev_timestamp'].split('T') res_time = int(''.join(res_time[0].split('-')) + ''.join(res_time[1][:-1].split(':'))) # fetching data since last minute of main_function executed if res_time >= now: break # For displaying Domain Reports d_t = {} users_data = [] pages_count = 0 for domain in domain_to_data.keys(): pages = [] user_list = [] for reports in domain_to_data[domain]: pages.append(reports['page_title']) name = reports['performer']['user_text'] try: e_count = reports['performer']['user_edit_count'] except: # if user_edit_count doesn't exist , we counting as 0 e_count = 0 user_list.append([name, e_count]) users_data.append(user_list) d_t[domain] = len(set(pages)) d_t = sorted(d_t.items(), key=lambda x: x[1], reverse=True) print("Sample Reports") print("Total number of Wikipedia Domains Updated ", len(domain_to_data.keys()), '\n') for i in d_t: print(f'{i[0]}: {i[1]} pages updated ') # For displaying User report k = 0 print("\n\t\tUSER Report \n") for domain_user in users_data: print(f'\nUsers who updates to {list(domain_to_data.keys())[k]}\n') user_to_count = {} for user in domain_user: if user[0] not in user_to_count: user_to_count[user[0]] = user[1] else: user_to_count[user[0]] = user[1] for name, count in user_to_count.items(): print(f'{name} : {count}') k += 1
class TestEventStreamsFilterTests(TestCase): """Filter tests for eventstreams module.""" dry = True data = {'foo': True, 'bar': 'baz'} def setUp(self): """Set up unit test.""" super(TestEventStreamsFilterTests, self).setUp() with mock.patch('pywikibot.comms.eventstreams.EventSource'): self.es = EventStreams(url='dummy url') def test_filter_function_all(self): """Test EventStreams filter all function.""" self.es.register_filter(lambda x: True) self.assertTrue(self.es.streamfilter(self.data)) self.es.register_filter(lambda x: False) self.assertFalse(self.es.streamfilter(self.data)) def test_filter_function_any(self): """Test EventStreams filter any function.""" self.es.register_filter(lambda x: True, ftype='any') self.assertTrue(self.es.streamfilter(self.data)) self.es.register_filter(lambda x: False, ftype='any') self.assertTrue(self.es.streamfilter(self.data)) def test_filter_function_none(self): """Test EventStreams filter none function.""" self.es.register_filter(lambda x: False, ftype='none') self.assertTrue(self.es.streamfilter(self.data)) self.es.register_filter(lambda x: True, ftype='none') self.assertFalse(self.es.streamfilter(self.data)) def _test_filter(self, none_type, all_type, any_type, result): """Test a single fixed filter.""" self.es.filter = {'all': [], 'any': [], 'none': []} self.es.register_filter(lambda x: none_type, ftype='none') self.es.register_filter(lambda x: all_type, ftype='all') if any_type is not None: self.es.register_filter(lambda x: any_type, ftype='any') self.assertEqual( self.es.streamfilter(self.data), result, 'Test EventStreams filter mixed function failed for\n' "'none': {0}, 'all': {1}, 'any': {2}\n" '(expected {3}, given {4})'.format(none_type, all_type, any_type, result, not result)) def test_filter_mixed_function(self): """Test EventStreams filter mixed function.""" for none_type in (False, True): for all_type in (False, True): for any_type in (False, True, None): if none_type is False and all_type is True and ( any_type is None or any_type is True): result = True else: result = False self._test_filter(none_type, all_type, any_type, result)
def get_stream(stream_date, stream_server): stream = EventStreams(streams=["recentchange", "revision-create"], since=stream_date) stream.register_filter(server_name=stream_server, type="edit") return stream
class TestEventStreamsFilter(TestCase): """Filter tests for eventstreams module.""" dry = True data = {'foo': True, 'bar': 'baz'} def setUp(self): """Set up unit test.""" super().setUp() with mock.patch('pywikibot.comms.eventstreams.EventSource'): self.es = EventStreams(url='dummy url') def test_filter_function_all(self): """Test EventStreams filter all function.""" self.es.register_filter(lambda x: True) self.assertTrue(self.es.streamfilter(self.data)) self.es.register_filter(lambda x: False) self.assertFalse(self.es.streamfilter(self.data)) def test_filter_function_any(self): """Test EventStreams filter any function.""" self.es.register_filter(lambda x: True, ftype='any') self.assertTrue(self.es.streamfilter(self.data)) self.es.register_filter(lambda x: False, ftype='any') self.assertTrue(self.es.streamfilter(self.data)) def test_filter_function_none(self): """Test EventStreams filter none function.""" self.es.register_filter(lambda x: False, ftype='none') self.assertTrue(self.es.streamfilter(self.data)) self.es.register_filter(lambda x: True, ftype='none') self.assertFalse(self.es.streamfilter(self.data)) def test_filter_false(self): """Test EventStreams filter with assignment of True.""" self.es.register_filter(foo=False) self.assertFalse(self.es.streamfilter(self.data)) def test_filter_true(self): """Test EventStreams filter with assignment of False.""" self.es.register_filter(foo=True) self.assertTrue(self.es.streamfilter(self.data)) def test_filter_value(self): """Test EventStreams filter with assignment of an int value.""" self.es.register_filter(foo=10) self.assertFalse(self.es.streamfilter(self.data)) def test_filter_sequence_false(self): """Test EventStreams filter with assignment of a sequence.""" self.es.register_filter(bar=list('baz')) self.assertFalse(self.es.streamfilter(self.data)) def test_filter_sequence_true(self): """Test EventStreams filter with assignment of a sequence.""" self.es.register_filter(bar=('foo', 'bar', 'baz')) self.assertTrue(self.es.streamfilter(self.data)) def test_filter_multiple(self): """Test EventStreams filter with multiple arguments.""" self.es.register_filter(foo=False, bar='baz') self.assertFalse(self.es.streamfilter(self.data)) self.es.filter = {'all': [], 'any': [], 'none': []} self.es.register_filter(foo=True, bar='baz') self.assertTrue(self.es.streamfilter(self.data)) # check whether filter functions are different f, g = self.es.filter['all'] c = {'foo': True} self.assertNotEqual(f(c), g(c)) c = {'bar': 'baz'} self.assertNotEqual(f(c), g(c)) def _test_filter(self, none_type, all_type, any_type, result): """Test a single fixed filter.""" self.es.filter = {'all': [], 'any': [], 'none': []} self.es.register_filter(lambda x: none_type, ftype='none') self.es.register_filter(lambda x: all_type, ftype='all') if any_type is not None: self.es.register_filter(lambda x: any_type, ftype='any') self.assertEqual( self.es.streamfilter(self.data), result, 'Test EventStreams filter mixed function failed for\n' "'none': {}, 'all': {}, 'any': {}\n" '(expected {}, given {})'.format(none_type, all_type, any_type, result, not result)) def test_filter_mixed_function(self): """Test EventStreams filter mixed function.""" for none_type in (False, True): for all_type in (False, True): for any_type in (False, True, None): result = none_type is False and all_type is True \ and (any_type is None or any_type is True) self._test_filter(none_type, all_type, any_type, result)
def setUp(self): """Set up unit test.""" super(TestEventStreamsSettingTests, self).setUp() with mock.patch('pywikibot.comms.eventstreams.EventSource'): self.es = EventStreams(url='dummy url')
class TestEventStreamsSettingTests(TestCase): """Setting tests for eventstreams module.""" dry = True def setUp(self): """Set up unit test.""" super(TestEventStreamsSettingTests, self).setUp() with mock.patch('pywikibot.comms.eventstreams.EventSource'): self.es = EventStreams(url='dummy url') def test_maximum_items(self): """Test EventStreams total value.""" total = 4711 self.es.set_maximum_items(total) self.assertEqual(self.es._total, total) def test_timeout_setting(self): """Test EventStreams timeout value.""" self.assertEqual(self.es.sse_kwargs.get('timeout'), config.socket_timeout) def test_filter_function_settings(self): """Test EventStreams filter function settings.""" def foo(): """Dummy function.""" return True self.es.register_filter(foo) self.assertEqual(self.es.filter['all'][0], foo) self.assertEqual(self.es.filter['any'], []) self.assertEqual(self.es.filter['none'], []) self.es.register_filter(foo, ftype='none') self.assertEqual(self.es.filter['all'][0], foo) self.assertEqual(self.es.filter['any'], []) self.assertEqual(self.es.filter['none'][0], foo) self.es.register_filter(foo, ftype='any') self.assertEqual(self.es.filter['all'][0], foo) self.assertEqual(self.es.filter['any'][0], foo) self.assertEqual(self.es.filter['none'][0], foo) def test_filter_function_settings_fail(self): """Test EventStreams failing filter function settings.""" with self.assertRaises(TypeError): self.es.register_filter('test') def test_filter_settings(self): """Test EventStreams filter settings.""" self.es.register_filter(foo='bar') self.assertIsInstance(self.es.filter['all'][0], FunctionType) self.es.register_filter(bar='baz') self.assertEqual(len(self.es.filter['all']), 2)
class TestEventStreamsFilterTests(TestCase): """Filter tests for eventstreams module.""" dry = True data = {'foo': True, 'bar': 'baz'} def setUp(self): """Set up unit test.""" super(TestEventStreamsFilterTests, self).setUp() with mock.patch('pywikibot.comms.eventstreams.EventSource'): self.es = EventStreams(url='dummy url') def test_filter_function_all(self): """Test EventStreams filter all function.""" self.es.register_filter(lambda x: True) self.assertTrue(self.es.streamfilter(self.data)) self.es.register_filter(lambda x: False) self.assertFalse(self.es.streamfilter(self.data)) def test_filter_function_any(self): """Test EventStreams filter any function.""" self.es.register_filter(lambda x: True, ftype='any') self.assertTrue(self.es.streamfilter(self.data)) self.es.register_filter(lambda x: False, ftype='any') self.assertTrue(self.es.streamfilter(self.data)) def test_filter_function_none(self): """Test EventStreams filter none function.""" self.es.register_filter(lambda x: False, ftype='none') self.assertTrue(self.es.streamfilter(self.data)) self.es.register_filter(lambda x: True, ftype='none') self.assertFalse(self.es.streamfilter(self.data)) def test_filter_false(self): """Test EventStreams filter with assignment of True.""" self.es.register_filter(foo=False) self.assertFalse(self.es.streamfilter(self.data)) def test_filter_true(self): """Test EventStreams filter with assignment of False.""" self.es.register_filter(foo=True) self.assertTrue(self.es.streamfilter(self.data)) def test_filter_value(self): """Test EventStreams filter with assignment of a int value.""" self.es.register_filter(foo=10) self.assertFalse(self.es.streamfilter(self.data)) def _test_filter(self, none_type, all_type, any_type, result): """Test a single fixed filter.""" self.es.filter = {'all': [], 'any': [], 'none': []} self.es.register_filter(lambda x: none_type, ftype='none') self.es.register_filter(lambda x: all_type, ftype='all') if any_type is not None: self.es.register_filter(lambda x: any_type, ftype='any') self.assertEqual(self.es.streamfilter(self.data), result, 'Test EventStreams filter mixed function failed for\n' "'none': {0}, 'all': {1}, 'any': {2}\n" '(expected {3}, given {4})' .format(none_type, all_type, any_type, result, not result)) def test_filter_mixed_function(self): """Test EventStreams filter mixed function.""" for none_type in (False, True): for all_type in (False, True): for any_type in (False, True, None): if none_type is False and all_type is True and ( any_type is None or any_type is True): result = True else: result = False self._test_filter(none_type, all_type, any_type, result)
from botbase import * from pywikibot.comms.eventstreams import EventStreams # pylint: disable=import-error def sign_diff(old_id, new_id): """ Creates diff based on revisions and determines based on diff whether something needs to be signed, and where to insert sign """ # in reality, all namespaces except for main are valid # need to check category to find wikipedia space pages that need signing valid_namespaces = (1, 3, 5, 7, 9, 11, 13, 15, 101, 109, 119, 711, 829, 2301, 2303) stream = EventStreams(streams="recentchange") stream.register_filter( type="edit", wiki="enwiki", bot=False, namespace=valid_namespaces ) for edit in stream: print(edit)