def test_mediawiki_format(self):
t1 = T.utcnow()
ts1 = t1.totimestampformat()
t2 = T.fromtimestampformat(ts1)
ts2 = t2.totimestampformat()
# MediaWiki timestamp format doesn't include microseconds
self.assertNotEqual(t1, t2)
t1 = t1.replace(microsecond=0)
self.assertEqual(t1, t2)
self.assertEqual(ts1, ts2)
def test_iso_format(self):
t1 = T.utcnow()
ts1 = t1.toISOformat()
t2 = T.fromISOformat(ts1)
ts2 = t2.toISOformat()
# MediaWiki ISO format doesn't include microseconds
self.assertNotEqual(t1, t2)
t1 = t1.replace(microsecond=0)
self.assertEqual(t1, t2)
self.assertEqual(ts1, ts2)
def test_iso_format(self):
"""Test conversion from and to ISO format."""
t1 = Timestamp.utcnow()
ts1 = t1.isoformat()
t2 = Timestamp.fromISOformat(ts1)
ts2 = t2.isoformat()
# MediaWiki ISO format doesn't include microseconds
self.assertNotEqual(t1, t2)
t1 = t1.replace(microsecond=0)
self.assertEqual(t1, t2)
self.assertEqual(ts1, ts2)
def test_iso_format_with_sep(self):
"""Test conversion from and to ISO format with separator."""
SEP = '*'
t1 = Timestamp.utcnow().replace(microsecond=0)
ts1 = t1.isoformat(sep=SEP)
t2 = Timestamp.fromISOformat(ts1, sep=SEP)
ts2 = t2.isoformat(sep=SEP)
self.assertEqual(t1, t2)
self.assertEqual(t1, t2)
self.assertEqual(ts1, ts2)
date, sep, time = ts1.partition(SEP)
time = time.rstrip('Z')
self.assertEqual(date, str(t1.date()))
self.assertEqual(time, str(t1.time()))
def test_sub_timedate(self):
"""Test subtracting two timestamps."""
t1 = Timestamp.utcnow()
t2 = t1 - datetime.timedelta(days=1)
td = t1 - t2
self.assertIsInstance(td, datetime.timedelta)
self.assertEqual(t2 + td, t1)
def _upcast_dict(map_):
"""Upcast dictionary values."""
with suppress(KeyError): # enable doctest
map_['timestamp'] = Timestamp.fromISOformat(map_['timestamp'])
map_.update(anon='anon' in map_)
map_.update(minor='minor' in map_)
map_.update(userhidden='userhidden' in map_)
map_.update(commenthidden='commenthidden' in map_)
map_.setdefault('comment', '')
map_.setdefault('user', '')
if 'slots' in map_: # mw 1.32+
mainslot = map_['slots'].get('main', {})
map_['text'] = mainslot.get('*')
map_['contentmodel'] = mainslot.get('contentmodel')
else:
map_['slots'] = None
map_['text'] = map_.get('*')
map_.setdefault('sha1')
if map_['sha1'] is None and map_['text'] is not None:
map_['sha1'] = hashlib.sha1(
map_['text'].encode('utf8')).hexdigest()
def test_sub_timedate(self):
"""Test subtracting two timestamps."""
t1 = Timestamp.utcnow()
t2 = t1 - datetime.timedelta(days=1)
td = t1 - t2
self.assertIsInstance(td, datetime.timedelta)
self.assertEqual(t2 + td, t1)
def test_iso_format(self):
"""Test conversion from and to ISO format."""
SEP = 'T'
t1 = Timestamp.utcnow()
ts1 = t1.isoformat()
t2 = Timestamp.fromISOformat(ts1)
ts2 = t2.isoformat()
# MediaWiki ISO format doesn't include microseconds
self.assertNotEqual(t1, t2)
t1 = t1.replace(microsecond=0)
self.assertEqual(t1, t2)
self.assertEqual(ts1, ts2)
date, sep, time = ts1.partition(SEP)
time = time.rstrip('Z')
self.assertEqual(date, str(t1.date()))
self.assertEqual(time, str(t1.time()))
def test_build_table_with_check(self):
"""Test buildt table with check option."""
bot = imagereview.CheckImageBot(check=True, total=0)
bot.cat = 'Nonexisting page for imagereview'
table = bot.build_table(save=False, unittest=True)
if not table:
self.skipTest('Table of files to review is empty')
key = list(table.keys())[0] # py3 comp
data = table[key]
item = data[0]
self.assertIsInstance(key, StringTypes)
self.assertIsInstance(data, list)
self.assertIsInstance(item, list)
self.assertEqual(len(item), 4)
linkedtitle, uploader, filepage, reason = item
user, time = uploader
self.assertIsInstance(linkedtitle, StringTypes)
self.assertIsInstance(uploader, list)
self.assertIsInstance(filepage, imagereview.DUP_Image)
self.assertIsInstance(reason, StringTypes)
self.assertIsInstance(user, StringTypes)
self.assertIsInstance(time, StringTypes)
self.assertEqual(reason, '')
self.assertEqual(filepage.title(asLink=True, textlink=True),
linkedtitle)
self.assertEqual(user, key)
self.assertIsInstance(Timestamp.fromISOformat(time), Timestamp)
def test_iso_format(self):
"""Test conversion from and to ISO format."""
SEP = 'T'
t1 = Timestamp.utcnow()
ts1 = t1.isoformat()
t2 = Timestamp.fromISOformat(ts1)
ts2 = t2.isoformat()
# MediaWiki ISO format doesn't include microseconds
self.assertNotEqual(t1, t2)
t1 = t1.replace(microsecond=0)
self.assertEqual(t1, t2)
self.assertEqual(ts1, ts2)
date, sep, time = ts1.partition(SEP)
time = time.rstrip('Z')
self.assertEqual(date, str(t1.date()))
self.assertEqual(time, str(t1.time()))
def test_add_timedelta(self):
t1 = T.utcnow()
t2 = t1 + datetime.timedelta(days=1)
if t1.month != t2.month:
self.assertEqual(1, t2.day)
else:
self.assertEqual(t1.day + 1, t2.day)
self.assertIsInstance(t2, T)
def test_sub_timedelta(self):
t1 = T.utcnow()
t2 = t1 - datetime.timedelta(days=1)
if t1.month != t2.month:
self.assertEqual(calendar.monthrange(t2.year, t2.month)[1], t2.day)
else:
self.assertEqual(t1.day - 1, t2.day)
self.assertIsInstance(t2, T)
def test_add_timedelta(self):
t1 = T.utcnow()
t2 = t1 + datetime.timedelta(days=1)
if t1.month != t2.month:
self.assertEqual(1, t2.day)
else:
self.assertEqual(t1.day + 1, t2.day)
self.assertIsInstance(t2, T)
def test_sub_timedelta(self):
t1 = T.utcnow()
t2 = t1 - datetime.timedelta(days=1)
if t1.month != t2.month:
self.assertEqual(calendar.monthrange(t2.year, t2.month)[1], t2.day)
else:
self.assertEqual(t1.day - 1, t2.day)
self.assertIsInstance(t2, T)
def test_add_timedelta(self):
"""Test addin a timedelta to a Timestamp."""
t1 = Timestamp.utcnow()
t2 = t1 + datetime.timedelta(days=1)
if t1.month != t2.month:
self.assertEqual(1, t2.day)
else:
self.assertEqual(t1.day + 1, t2.day)
self.assertIsInstance(t2, Timestamp)
def test_sub_timedelta(self):
"""Test substracting a timedelta from a Timestamp."""
t1 = Timestamp.utcnow()
t2 = t1 - datetime.timedelta(days=1)
if t1.month != t2.month:
self.assertEqual(calendar.monthrange(t2.year, t2.month)[1], t2.day)
else:
self.assertEqual(t1.day - 1, t2.day)
self.assertIsInstance(t2, Timestamp)
def test_instantiate_from_instance(self):
"""Test passing instance to factory methods works."""
t1 = T.utcnow()
self.assertIsNot(t1, T.fromISOformat(t1))
self.assertEqual(t1, T.fromISOformat(t1))
self.assertIsInstance(T.fromISOformat(t1), T)
self.assertIsNot(t1, T.fromtimestampformat(t1))
self.assertEqual(t1, T.fromtimestampformat(t1))
self.assertIsInstance(T.fromtimestampformat(t1), T)
def test_two_revisions_reverse_order(self):
revisions_wrong_order = [
Revision(
2,
Timestamp.fromtimestampformat('20200115'),
'test_user',
slots={
'main': {
'contentmodel':
'wikitext',
'contentformat':
'text/x-wiki',
'*':
'[[Изображение:test.gif|thumb]]\n\ntest test test\n\nnewline\n'
}
}),
Revision(
1,
Timestamp.fromtimestampformat('20200112'),
'test_user',
slots={
'main': {
'contentmodel':
'wikitext',
'contentformat':
'text/x-wiki',
'*':
'[[Изображение:test.gif|thumb]]\n\nold\n\ntest test test\n\n'
}
})
]
diffs = ApiWrapper._convert_revisions_to_diffs(revisions_wrong_order)
self.assertEqual(1, len(diffs))
self.assertEqual(
RevisionDiff(old_ts=datetime.strptime('2020-01-12T00:00:00Z',
'%Y-%m-%dT%H:%M:%SZ'),
new_ts=datetime.strptime('2020-01-15T00:00:00Z',
'%Y-%m-%dT%H:%M:%SZ'),
old_revid=1,
new_revid=2,
lines_diffs=[(None, 'old\n'), (None, 'newline\n')]),
diffs[0])
def test_instantiate_from_instance(self):
"""Test passing instance to factory methods works."""
t1 = Timestamp.utcnow()
self.assertIsNot(t1, Timestamp.fromISOformat(t1))
self.assertEqual(t1, Timestamp.fromISOformat(t1))
self.assertIsInstance(Timestamp.fromISOformat(t1), Timestamp)
self.assertIsNot(t1, Timestamp.fromtimestampformat(t1))
self.assertEqual(t1, Timestamp.fromtimestampformat(t1))
self.assertIsInstance(Timestamp.fromtimestampformat(t1), Timestamp)
def test_add_timedate(self):
"""Test unsupported additions raise NotImplemented."""
t1 = datetime.datetime.utcnow()
t2 = t1 + datetime.timedelta(days=1)
t3 = t1.__add__(t2)
self.assertIs(t3, NotImplemented)
# Now check that the pywikibot sub-class behaves the same way
t1 = Timestamp.utcnow()
t2 = t1 + datetime.timedelta(days=1)
t3 = t1.__add__(t2)
self.assertIs(t3, NotImplemented)
def test_add_timedate(self):
"""Test unsupported additions raise NotImplemented."""
t1 = datetime.datetime.utcnow()
t2 = t1 + datetime.timedelta(days=1)
t3 = t1.__add__(t2)
self.assertIs(t3, NotImplemented)
# Now check that the pywikibot sub-class behaves the same way
t1 = T.utcnow()
t2 = t1 + datetime.timedelta(days=1)
t3 = t1.__add__(t2)
self.assertIs(t3, NotImplemented)
def __init__(self, **kwargs):
"""Initializer.
@keyword site: a project site object. Used when no url is given
@type site: APISite
@keyword since: a timestamp for older events; there will likely be
between 7 and 31 days of history available but is not guaranteed.
It may be given as a pywikibot.Timestamp, an ISO 8601 string
or a mediawiki timestamp string.
@type since: pywikibot.Timestamp or str
@keyword streams: event stream types. Mandatory when no url is given.
Multiple streams may be given as a string with comma separated
stream types or an iterable of strings
Refer https://stream.wikimedia.org/?doc for available
wikimedia stream types.
@type streams: str or iterable
@keyword timeout: a timeout value indication how long to wait to send
data before giving up
@type timeout: int, float or a tuple of two values of int or float
@keyword url: an url retrieving events from. Will be set up to a
default url using _site.family settings, stream types and timestamp
@type url: str
@param kwargs: keyword arguments passed to SSEClient and requests lib
@raises ImportError: sseclient is not installed
@raises NotImplementedError: no stream types specified
"""
if isinstance(EventSource, Exception):
raise ImportError('sseclient is required for EventStreams;\n'
'install it with "pip install sseclient"\n')
self.filter = {'all': [], 'any': [], 'none': []}
self._total = None
self._site = kwargs.pop('site', Site())
self._streams = kwargs.pop('streams', None)
if self._streams and not isinstance(self._streams, StringTypes):
self._streams = ','.join(self._streams)
self._since = kwargs.pop('since', None)
if self._since:
# assume this is a mw timestamp, convert it to a Timestamp object
if isinstance(self._streams, StringTypes) \
and '-' not in self._since:
self._since = Timestamp.fromtimestampformat(self._since)
if isinstance(self._streams, Timestamp):
self._since = self._since.isoformat
self._url = kwargs.get('url') or self.url
kwargs.setdefault('url', self._url)
kwargs.setdefault('timeout', config.socket_timeout)
self.sse_kwargs = kwargs
def __init__(self, **kwargs):
"""Initializer.
@keyword site: a project site object. Used when no url is given
@type site: APISite
@keyword since: a timestamp for older events; there will likely be
between 7 and 31 days of history available but is not guaranteed.
It may be given as a pywikibot.Timestamp, an ISO 8601 string
or a mediawiki timestamp string.
@type since: pywikibot.Timestamp or str
@keyword streams: event stream types. Mandatory when no url is given.
Multiple streams may be given as a string with comma separated
stream types or an iterable of strings
Refer https://stream.wikimedia.org/?doc for available
wikimedia stream types.
@type streams: str or iterable
@keyword timeout: a timeout value indication how long to wait to send
data before giving up
@type timeout: int, float or a tuple of two values of int or float
@keyword url: an url retrieving events from. Will be set up to a
default url using _site.family settings, stream types and timestamp
@type url: str
@param kwargs: keyword arguments passed to SSEClient and requests lib
@raises ImportError: sseclient is not installed
@raises NotImplementedError: no stream types specified
"""
if isinstance(EventSource, Exception):
raise ImportError('sseclient is required for EventStreams;\n'
'install it with "pip install sseclient"\n')
self.filter = {'all': [], 'any': [], 'none': []}
self._total = None
self._site = kwargs.pop('site', Site())
self._streams = kwargs.pop('streams', None)
if self._streams and not isinstance(self._streams, StringTypes):
self._streams = ','.join(self._streams)
self._since = kwargs.pop('since', None)
if self._since:
# assume this is a mw timestamp, convert it to a Timestamp object
if isinstance(self._streams, StringTypes) \
and '-' not in self._since:
self._since = Timestamp.fromtimestampformat(self._since)
if isinstance(self._streams, Timestamp):
self._since = self._since.isoformat
self._url = kwargs.get('url') or self.url
kwargs.setdefault('url', self._url)
kwargs.setdefault('timeout', config.socket_timeout)
self.sse_kwargs = kwargs
def active_and_future_campaigns():
from pywikibot.data.api import Request
from pywikibot import Timestamp
parameters = {
'action': 'query',
'list': 'centralnoticeactivecampaigns',
'cnacincludefuture': ''
}
request = Request(_site, parameters=parameters)
# TODO Error handling
raw_query_data = request.submit()
raw_campaigns = (
raw_query_data['query']['centralnoticeactivecampaigns']['campaigns'])
# Convert start and end to datetime objects
for c in raw_campaigns:
c['start'] = Timestamp.fromtimestampformat(c['start'])
c['end'] = Timestamp.fromtimestampformat(c['end'])
return raw_campaigns
def run(self):
"""Run the bot."""
starttime = time()
rc_listener = site_rc_listener(self.site, timeout=60)
while True:
pywikibot.output(Timestamp.now().strftime(">> %H:%M:%S: "))
self.read_lists()
try:
self.markBlockedusers(self.loadBlockedUsers())
self.contactDefendants(bootmode=self.start)
except pywikibot.EditConflict:
pywikibot.output("Edit conflict found, try again.")
continue # try again and skip waittime
except pywikibot.PageNotSaved:
pywikibot.output("Page not saved, try again.")
continue # try again and skip waittime
# wait for new block entry
print()
now = time()
pywikibot.stopme()
for i, entry in enumerate(rc_listener):
if i % 25 == 0:
print('\r', ' ' * 50, '\rWaiting for events', end='')
if entry['type'] == 'log' and \
entry['log_type'] == 'block' and \
entry['log_action'] in ('block', 'reblock'):
pywikibot.output('\nFound a new blocking event '
'by user "%s" for user "%s"'
% (entry['user'], entry['title']))
break
if entry['type'] == 'edit' and \
not entry['bot'] and \
entry['title'] == self.vmPageName:
pywikibot.output('\nFound a new edit by user "%s"'
% entry['user'])
break
if not entry['bot']:
print('.', end='')
print('\n')
self.optOutListAge += time() - now
# read older entries again after ~4 minutes
if time() - starttime > 250:
starttime = time()
self.reset_timestamp()
self.start = False
self.total = 10
def test_sub_timedelta(self):
t1 = T.utcnow()
t2 = t1 - datetime.timedelta(days=1)
self.assertEqual(t1.day - 1, t2.day)
self.assertIsInstance(t2, T)
def test_sub_timedate(self):
t1 = T.utcnow()
t2 = t1 - datetime.timedelta(days=1)
td = t1 - t2
self.assertIsInstance(td, datetime.timedelta)
self.assertEqual(t2 + td, t1)
def test_sub_timedelta(self):
t1 = T.utcnow()
t2 = t1 - datetime.timedelta(days=1)
self.assertEqual(t1.day - 1, t2.day)
self.assertIsInstance(t2, T)
def getEditCounts(site: pywikibot.site.BaseSite, user: pywikibot.User,
since: pywikibot.Timestamp) -> EditCounts:
edits = 0
articleEdits = 0
flaggedEdits = 0
fvnEdits = 0
ownUserTalkPageEdits = 0
otherUserTalkPageEdits = 0
contribsRequest = pywikibot.data.api.Request(
site=site,
parameters={
"action": "query",
"format": "json",
"list": "usercontribs",
"uclimit": "500",
"ucend": since.totimestampformat(),
"ucuser": user.username,
},
)
data = contribsRequest.submit()
contribs = data["query"]["usercontribs"]
edits = len(contribs)
revs = ""
for contrib in contribs:
if contrib["ns"] == 0:
articleEdits += 1
if len(revs) != 0:
revs += "|"
revs += str(contrib["revid"])
if contrib["ns"] == pywikibot.site.Namespace.PROJECT and contrib[
"title"] == "Wikipedia:Fragen von Neulingen":
fvnEdits += 1
if contrib["ns"] == pywikibot.site.Namespace.USER_TALK:
if contrib["title"][contrib["title"].find(":") +
1:] == user.username:
ownUserTalkPageEdits += 1
else:
otherUserTalkPageEdits += 1
if len(revs) != 0:
revisionsRequest = pywikibot.data.api.Request(
site=site,
parameters={
"action": "query",
"format": "json",
"prop": "revisions|flagged",
"rvprop": "flagged|ids",
"revids": revs,
},
)
data = revisionsRequest.submit()
pages = data["query"]["pages"]
for page in pages:
for revision in pages[page]["revisions"]:
if "flagged" in revision:
flaggedEdits += 1
return EditCounts(
edits=edits,
articleEdits=articleEdits,
flaggedEdits=flaggedEdits,
fvnEdits=fvnEdits,
ownUserTalkPageEdits=ownUserTalkPageEdits,
otherUserTalkPageEdits=otherUserTalkPageEdits,
)
def test_clone(self):
t1 = T.utcnow()
t2 = t1.clone()
self.assertEqual(t1, t2)
self.assertIsInstance(t2, T)
def test_iso_format_property(self):
"""Test iso format properties."""
self.assertEqual(Timestamp.ISO8601Format, Timestamp._ISO8601Format())
self.assertEqual(re.sub(r'[\-:TZ]', '', Timestamp.ISO8601Format),
Timestamp.mediawikiTSFormat)
def reduce_seconds(value):
if not value:
return None
return Timestamp(value.year, value.month, value.day,
value.hour, value.minute)
def test_clone(self):
t1 = T.utcnow()
t2 = t1.clone()
self.assertEqual(t1, t2)
self.assertIsInstance(t2, T)
def test_sub_timedate(self):
t1 = T.utcnow()
t2 = t1 - datetime.timedelta(days=1)
td = t1 - t2
self.assertIsInstance(td, datetime.timedelta)
self.assertEqual(t2 + td, t1)
def test_clone(self):
"""Test cloning a Timestamp instance."""
t1 = Timestamp.utcnow()
t2 = t1.clone()
self.assertEqual(t1, t2)
self.assertIsInstance(t2, Timestamp)
def test_iso_format_property(self):
"""Test iso format properties."""
self.assertEqual(Timestamp.ISO8601Format, Timestamp._ISO8601Format())
self.assertEqual(re.sub(r'[\-:TZ]', '', Timestamp.ISO8601Format),
Timestamp.mediawikiTSFormat)
def convert_date(value):
if not value:
return None
return Timestamp(value.year, value.month, value.day, value.hour,
value.minute, value.second)
