def run(self):
stream = EventStreams(stream='recentchange')
for event in stream:
if self.stop_event.isSet():
raise SystemExit
self.handler(event)
def test_url_parameter(self, key):
"""Test EventStreams with given url."""
e = EventStreams(url=self.sites[key]['hostname'])
self.assertEqual(e._url, self.sites[key]['hostname'])
self.assertEqual(e._url, e.url)
self.assertEqual(e._url, e.sse_kwargs.get('url'))
self.assertIsNone(e._total)
self.assertIsNone(e._stream)
def __setup_stream(self):
logger.info('Building stream object')
stream = EventStreams(streams=['recentchange', 'revision-create'])
stream.register_filter(server_name='www.wikidata.org',
type='edit',
wiki='wikidatawiki')
logger.debug('Stream object built: {}'.format(stream))
self.__stream = stream
def test_url_with_streams(self):
"""Test EventStreams with url from default site."""
streams = 'recentchange'
e = EventStreams(streams=streams)
self.assertEqual(e._url,
'https://stream.wikimedia.org/v2/stream/' + streams)
self.assertEqual(e._url, e.url)
self.assertEqual(e._url, e.sse_kwargs.get('url'))
self.assertIsNone(e._total)
self.assertEqual(e._streams, streams)
def test_url_from_site(self, key):
"""Test EventStreams with url from site."""
site = self.get_site(key)
stream = 'recentchanges'
e = EventStreams(site=site, stream=stream)
self.assertEqual(e._url,
'https://stream.wikimedia.org/v2/stream/' + stream)
self.assertEqual(e._url, e.url)
self.assertEqual(e._url, e.sse_kwargs.get('url'))
self.assertIsNone(e._total)
self.assertEqual(e._stream, stream)
def test_multiple_streams(self):
"""Test EventStreams with multiple streams."""
streams = ('page-create', 'page-move', 'page-delete')
e = EventStreams(streams=streams)
combined_streams = ','.join(streams)
self.assertEqual(
e._url,
'https://stream.wikimedia.org/v2/stream/' + combined_streams)
self.assertEqual(e._url, e.url)
self.assertEqual(e._url, e.sse_kwargs.get('url'))
self.assertEqual(e._streams, combined_streams)
def test_url_with_stream(self):
"""Test EventStreams with url from default site."""
site = self.get_site()
fam = site.family
if not isinstance(fam, WikimediaFamily):
self.skipTest(
"Family '{0}' of site '{1}' is not a WikimediaFamily.".format(
fam, site))
stream = 'recentchanges'
e = EventStreams(stream=stream)
self.assertEqual(e._url,
'https://stream.wikimedia.org/v2/stream/' + stream)
self.assertEqual(e._url, e.url)
self.assertEqual(e._url, e.sse_kwargs.get('url'))
self.assertIsNone(e._total)
self.assertEqual(e._stream, stream)
def setUp(self):
"""Set up unit test."""
super(TestEventStreamsSettingTests, self).setUp()
with mock.patch('pywikibot.comms.eventstreams.EventSource'):
self.es = EventStreams(url='dummy url')
def test_url_missing_stream(self):
"""Test EventStreams with url from site with missing stream."""
with self.assertRaises(NotImplementedError):
EventStreams()
from botbase import *
from pywikibot.comms.eventstreams import EventStreams # pylint: disable=import-error
def sign_diff(old_id, new_id):
"""
Creates diff based on revisions and determines based on diff whether
something needs to be signed, and where to insert sign
"""
# in reality, all namespaces except for main are valid
# need to check category to find wikipedia space pages that need signing
valid_namespaces = (1, 3, 5, 7, 9, 11, 13, 15, 101, 109, 119, 711, 829, 2301, 2303)
stream = EventStreams(streams="recentchange")
stream.register_filter(
type="edit", wiki="enwiki", bot=False, namespace=valid_namespaces
)
for edit in stream:
print(edit)
def get_stream(stream_date, stream_server):
stream = EventStreams(streams=["recentchange", "revision-create"],
since=stream_date)
stream.register_filter(server_name=stream_server, type="edit")
return stream
def main_function():
query_timestamp = query_timestamp = (
datetime.datetime.utcnow() - datetime.timedelta(minutes=1)
).strftime(
'%Y%m%d%H%M%S') #change mintnues=1 to 5 for last 5 min wiki-changes
now = int(datetime.datetime.utcnow().strftime('%Y%m%d%H%M%S'))
wiki_data = []
domain_to_data = {}
stream = EventStreams(streams=['revision-create'], since=query_timestamp)
while True:
response = next(iter(stream))
# checking is user is bot or not , if not bot then only considering the response
if (response['performer']['user_is_bot'] != True):
wiki_data.append(response)
if response['meta']['domain'] in domain_to_data:
domain_to_data[response['meta']['domain']].append(response)
else:
domain_to_data[response['meta']['domain']] = [response]
res_time = response['rev_timestamp'].split('T')
res_time = int(''.join(res_time[0].split('-')) +
''.join(res_time[1][:-1].split(':')))
# fetching data since last minute of main_function executed
if res_time >= now:
break
# For displaying Domain Reports
d_t = {}
users_data = []
pages_count = 0
for domain in domain_to_data.keys():
pages = []
user_list = []
for reports in domain_to_data[domain]:
pages.append(reports['page_title'])
name = reports['performer']['user_text']
try:
e_count = reports['performer']['user_edit_count']
except:
# if user_edit_count doesn't exist , we counting as 0
e_count = 0
user_list.append([name, e_count])
users_data.append(user_list)
d_t[domain] = len(set(pages))
d_t = sorted(d_t.items(), key=lambda x: x[1], reverse=True)
print("Sample Reports")
print("Total number of Wikipedia Domains Updated ",
len(domain_to_data.keys()), '\n')
for i in d_t:
print(f'{i[0]}: {i[1]} pages updated ')
# For displaying User report
k = 0
print("\n\t\tUSER Report \n")
for domain_user in users_data:
print(f'\nUsers who updates to {list(domain_to_data.keys())[k]}\n')
user_to_count = {}
for user in domain_user:
if user[0] not in user_to_count:
user_to_count[user[0]] = user[1]
else:
user_to_count[user[0]] = user[1]
for name, count in user_to_count.items():
print(f'{name} : {count}')
k += 1