def run(self): stream = EventStreams(stream='recentchange') for event in stream: if self.stop_event.isSet(): raise SystemExit self.handler(event)
def test_url_parameter(self, key): """Test EventStreams with given url.""" e = EventStreams(url=self.sites[key]['hostname']) self.assertEqual(e._url, self.sites[key]['hostname']) self.assertEqual(e._url, e.url) self.assertEqual(e._url, e.sse_kwargs.get('url')) self.assertIsNone(e._total) self.assertIsNone(e._stream)
def __setup_stream(self): logger.info('Building stream object') stream = EventStreams(streams=['recentchange', 'revision-create']) stream.register_filter(server_name='www.wikidata.org', type='edit', wiki='wikidatawiki') logger.debug('Stream object built: {}'.format(stream)) self.__stream = stream
def test_url_with_streams(self): """Test EventStreams with url from default site.""" streams = 'recentchange' e = EventStreams(streams=streams) self.assertEqual(e._url, 'https://stream.wikimedia.org/v2/stream/' + streams) self.assertEqual(e._url, e.url) self.assertEqual(e._url, e.sse_kwargs.get('url')) self.assertIsNone(e._total) self.assertEqual(e._streams, streams)
def test_url_from_site(self, key): """Test EventStreams with url from site.""" site = self.get_site(key) stream = 'recentchanges' e = EventStreams(site=site, stream=stream) self.assertEqual(e._url, 'https://stream.wikimedia.org/v2/stream/' + stream) self.assertEqual(e._url, e.url) self.assertEqual(e._url, e.sse_kwargs.get('url')) self.assertIsNone(e._total) self.assertEqual(e._stream, stream)
def test_multiple_streams(self): """Test EventStreams with multiple streams.""" streams = ('page-create', 'page-move', 'page-delete') e = EventStreams(streams=streams) combined_streams = ','.join(streams) self.assertEqual( e._url, 'https://stream.wikimedia.org/v2/stream/' + combined_streams) self.assertEqual(e._url, e.url) self.assertEqual(e._url, e.sse_kwargs.get('url')) self.assertEqual(e._streams, combined_streams)
def test_url_with_stream(self): """Test EventStreams with url from default site.""" site = self.get_site() fam = site.family if not isinstance(fam, WikimediaFamily): self.skipTest( "Family '{0}' of site '{1}' is not a WikimediaFamily.".format( fam, site)) stream = 'recentchanges' e = EventStreams(stream=stream) self.assertEqual(e._url, 'https://stream.wikimedia.org/v2/stream/' + stream) self.assertEqual(e._url, e.url) self.assertEqual(e._url, e.sse_kwargs.get('url')) self.assertIsNone(e._total) self.assertEqual(e._stream, stream)
def setUp(self): """Set up unit test.""" super(TestEventStreamsSettingTests, self).setUp() with mock.patch('pywikibot.comms.eventstreams.EventSource'): self.es = EventStreams(url='dummy url')
def test_url_missing_stream(self): """Test EventStreams with url from site with missing stream.""" with self.assertRaises(NotImplementedError): EventStreams()
from botbase import * from pywikibot.comms.eventstreams import EventStreams # pylint: disable=import-error def sign_diff(old_id, new_id): """ Creates diff based on revisions and determines based on diff whether something needs to be signed, and where to insert sign """ # in reality, all namespaces except for main are valid # need to check category to find wikipedia space pages that need signing valid_namespaces = (1, 3, 5, 7, 9, 11, 13, 15, 101, 109, 119, 711, 829, 2301, 2303) stream = EventStreams(streams="recentchange") stream.register_filter( type="edit", wiki="enwiki", bot=False, namespace=valid_namespaces ) for edit in stream: print(edit)
def get_stream(stream_date, stream_server): stream = EventStreams(streams=["recentchange", "revision-create"], since=stream_date) stream.register_filter(server_name=stream_server, type="edit") return stream
def main_function(): query_timestamp = query_timestamp = ( datetime.datetime.utcnow() - datetime.timedelta(minutes=1) ).strftime( '%Y%m%d%H%M%S') #change mintnues=1 to 5 for last 5 min wiki-changes now = int(datetime.datetime.utcnow().strftime('%Y%m%d%H%M%S')) wiki_data = [] domain_to_data = {} stream = EventStreams(streams=['revision-create'], since=query_timestamp) while True: response = next(iter(stream)) # checking is user is bot or not , if not bot then only considering the response if (response['performer']['user_is_bot'] != True): wiki_data.append(response) if response['meta']['domain'] in domain_to_data: domain_to_data[response['meta']['domain']].append(response) else: domain_to_data[response['meta']['domain']] = [response] res_time = response['rev_timestamp'].split('T') res_time = int(''.join(res_time[0].split('-')) + ''.join(res_time[1][:-1].split(':'))) # fetching data since last minute of main_function executed if res_time >= now: break # For displaying Domain Reports d_t = {} users_data = [] pages_count = 0 for domain in domain_to_data.keys(): pages = [] user_list = [] for reports in domain_to_data[domain]: pages.append(reports['page_title']) name = reports['performer']['user_text'] try: e_count = reports['performer']['user_edit_count'] except: # if user_edit_count doesn't exist , we counting as 0 e_count = 0 user_list.append([name, e_count]) users_data.append(user_list) d_t[domain] = len(set(pages)) d_t = sorted(d_t.items(), key=lambda x: x[1], reverse=True) print("Sample Reports") print("Total number of Wikipedia Domains Updated ", len(domain_to_data.keys()), '\n') for i in d_t: print(f'{i[0]}: {i[1]} pages updated ') # For displaying User report k = 0 print("\n\t\tUSER Report \n") for domain_user in users_data: print(f'\nUsers who updates to {list(domain_to_data.keys())[k]}\n') user_to_count = {} for user in domain_user: if user[0] not in user_to_count: user_to_count[user[0]] = user[1] else: user_to_count[user[0]] = user[1] for name, count in user_to_count.items(): print(f'{name} : {count}') k += 1