def setUp(self):
super(DryCachedRequestTests, self).setUp()
self.parms = {"site": self.basesite, "action": "query", "meta": "userinfo"}
self.req = CachedRequest(expiry=1, **self.parms)
self.expreq = CachedRequest(expiry=0, **self.parms)
self.diffreq = CachedRequest(expiry=1, site=self.basesite, action="query", meta="siteinfo")
self.diffsite = CachedRequest(expiry=1, site=self.altsite, action="query", meta="userinfo")
def refresh(site, sysop=False):
"""Fetch the watchlist."""
if not site.logged_in(sysop=sysop):
site.forceLogin(sysop=sysop)
params = {
'action': 'query',
'list': 'watchlistraw',
'site': site,
'wrlimit': config.special_page_limit,
}
pywikibot.output(u'Retrieving watchlist for %s via API.' % str(site))
# pywikibot.put_throttle() # It actually is a get, but a heavy one.
watchlist = []
while True:
req = CachedRequest(config.API_config_expiry, **params)
data = req.submit()
if 'error' in data:
raise RuntimeError('ERROR: %s' % data)
watchlist.extend([w['title'] for w in data['watchlistraw']])
if 'query-continue' in data:
params.update(data['query-continue']['watchlistraw'])
else:
break
return watchlist
def refresh(site, sysop=False):
"""Fetch the watchlist."""
if not site.logged_in(sysop=sysop):
site.login(sysop=sysop)
params = {
'action': 'query',
'list': 'watchlistraw',
'wrlimit': config.special_page_limit,
}
pywikibot.output(u'Retrieving watchlist for %s via API.' % str(site))
# pywikibot.put_throttle() # It actually is a get, but a heavy one.
watchlist = []
while True:
req = CachedRequest(config.API_config_expiry, site=site, **params)
data = req.submit()
if 'error' in data:
raise RuntimeError('ERROR: %s' % data)
watchlist.extend([w['title'] for w in data['watchlistraw']])
if 'query-continue' in data:
params.update(data['query-continue']['watchlistraw'])
else:
break
return watchlist
def setUp(self):
self.parms = {'site': pywikibot.Site('en'),
'action': 'query',
'meta': 'userinfo'}
self.req = CachedRequest(expiry=1, **self.parms)
self.expreq = CachedRequest(expiry=0, **self.parms)
self.diffreq = CachedRequest(expiry=1, site=pywikibot.Site('en'), action='query', meta='siteinfo')
self.diffsite = CachedRequest(expiry=1, site=pywikibot.Site('de'), action='query', meta='userinfo')
def setUp(self):
super(DryCachedRequestTests, self).setUp()
self.parms = {'site': self.basesite,
'action': 'query',
'meta': 'userinfo'}
self.req = CachedRequest(expiry=1, **self.parms)
self.expreq = CachedRequest(expiry=0, **self.parms)
self.diffreq = CachedRequest(expiry=1, site=self.basesite, action='query', meta='siteinfo')
self.diffsite = CachedRequest(expiry=1, site=self.altsite, action='query', meta='userinfo')
def test_cachefile_path_different_users(self):
"""Test and compare file paths when different usernames are used."""
req = CachedRequest(expiry=1,
site=self.mocksite,
parameters={
'action': 'query',
'meta': 'siteinfo'
})
anonpath = req._cachefile_path()
self.mocksite._userinfo = {'name': 'MyUser'}
self.mocksite._loginstatus = LoginStatus.AS_USER
req = CachedRequest(expiry=1,
site=self.mocksite,
parameters={
'action': 'query',
'meta': 'siteinfo'
})
userpath = req._cachefile_path()
self.assertNotEqual(anonpath, userpath)
self.mocksite._userinfo = {'name': 'MyOtherUser'}
self.mocksite._loginstatus = LoginStatus.AS_USER
req = CachedRequest(expiry=1,
site=self.mocksite,
parameters={
'action': 'query',
'meta': 'siteinfo'
})
otherpath = req._cachefile_path()
self.assertNotEqual(anonpath, otherpath)
self.assertNotEqual(userpath, otherpath)
def test_cachefile_path_different_users(self):
"""Test and compare the file paths when different usernames are used."""
req = CachedRequest(expiry=1,
site=self.mocksite,
action='query',
meta='siteinfo')
anonpath = req._cachefile_path()
self.mocksite._userinfo = {'name': u'MyUser'}
self.mocksite._loginstatus = 0
req = CachedRequest(expiry=1,
site=self.mocksite,
action='query',
meta='siteinfo')
userpath = req._cachefile_path()
self.assertNotEqual(anonpath, userpath)
self.mocksite._userinfo = {'name': u'MySysop'}
self.mocksite._loginstatus = 1
req = CachedRequest(expiry=1,
site=self.mocksite,
action='query',
meta='siteinfo')
sysoppath = req._cachefile_path()
self.assertNotEqual(anonpath, sysoppath)
self.assertNotEqual(userpath, sysoppath)
def test_cachefile_path_different_users(self):
# Mock basesite object to test this.
class MockSite(pywikibot.site.APISite):
_loginstatus = pywikibot.site.LoginStatus.NOT_ATTEMPTED
_namespaces = {2: 'User'}
def __init__(self):
self._user = '******'
def user(self):
return self._user
def encoding(self):
return 'utf-8'
def encodings(self):
return []
def _getsiteinfo(self):
self._siteinfo = {'case': 'first-letter'}
return {}
def __repr__(self):
return "MockSite()"
def __getattr__(self, attr):
raise Exception("Attribute %r not defined" % attr)
site = MockSite()
req = CachedRequest(expiry=1,
site=site,
action='query',
meta='siteinfo')
anonpath = req._cachefile_path()
site._userinfo = {'name': u'user'}
site._loginstatus = 0
req = CachedRequest(expiry=1,
site=site,
action='query',
meta='siteinfo')
userpath = req._cachefile_path()
self.assertNotEqual(anonpath, userpath)
site._userinfo = {'name': u'sysop'}
site._loginstatus = 1
req = CachedRequest(expiry=1,
site=site,
action='query',
meta='siteinfo')
sysoppath = req._cachefile_path()
self.assertNotEqual(anonpath, sysoppath)
self.assertNotEqual(userpath, sysoppath)
def test_unicode(self):
"""Test caching with Unicode content."""
self.mocksite._userinfo = {'name': u'محمد الفلسطيني'}
self.mocksite._loginstatus = 0
req = CachedRequest(expiry=1,
site=self.mocksite,
action='query',
meta='siteinfo')
en_user_path = req._cachefile_path()
self.mocksite._namespaces = {2: [u'مستخدم']}
req = CachedRequest(expiry=1,
site=self.mocksite,
action='query',
meta='siteinfo')
expect = (u'MockSite()User(User:محمد الفلسطيني)' +
"[('action', 'query'), ('meta', 'siteinfo')]")
self.assertEqual(repr(req._uniquedescriptionstr()), repr(expect))
self.assertEqual(req._uniquedescriptionstr().encode('utf-8'),
expect.encode('utf-8'))
ar_user_path = req._cachefile_path()
self.assertEqual(en_user_path, ar_user_path)
def setUp(self):
self.parms = {'site': pywikibot.Site('en'),
'action': 'query',
'meta': 'userinfo'}
self.req = CachedRequest(expiry=1, **self.parms)
self.expreq = CachedRequest(expiry=0, **self.parms)
self.diffreq = CachedRequest(expiry=1, site=pywikibot.Site('en'), action='query', meta='siteinfo')
self.diffsite = CachedRequest(expiry=1, site=pywikibot.Site('de'), action='query', meta='userinfo')
def test_expiry_formats(self):
"""Test using a timedelta as expiry."""
self.assertEqual(
self.req.expiry,
CachedRequest(datetime.timedelta(days=1),
site=self.basesite,
parameters=self.parms).expiry)
def setUp(self):
super(DryCachedRequestTests, self).setUp()
self.parms = {'site': self.basesite,
'action': 'query',
'meta': 'userinfo'}
self.req = CachedRequest(expiry=1, **self.parms)
self.expreq = CachedRequest(expiry=0, **self.parms)
self.diffreq = CachedRequest(expiry=1, site=self.basesite, action='query', meta='siteinfo')
self.diffsite = CachedRequest(expiry=1, site=self.altsite, action='query', meta='userinfo')
def test_unicode(self):
"""Test caching with Unicode content."""
self.mocksite._userinfo = {'name': u'محمد الفلسطيني'}
self.mocksite._loginstatus = 0
req = CachedRequest(expiry=1, site=self.mocksite,
parameters={'action': 'query', 'meta': 'siteinfo'})
en_user_path = req._cachefile_path()
self.mocksite._namespaces = {2: [u'مستخدم']}
req = CachedRequest(expiry=1, site=self.mocksite,
parameters={'action': 'query', 'meta': 'siteinfo'})
expect = (u'MockSite()User(User:محمد الفلسطيني)' +
"[('action', 'query'), ('meta', 'siteinfo')]")
self.assertEqual(repr(req._uniquedescriptionstr()), repr(expect))
self.assertEqual(req._uniquedescriptionstr().encode('utf-8'),
expect.encode('utf-8'))
ar_user_path = req._cachefile_path()
self.assertEqual(en_user_path, ar_user_path)
def refresh_all():
"""Reload watchlists for all wikis where a watchlist is already present."""
cache_path = CachedRequest._get_cache_dir()
files = os.scandir(cache_path)
seen = set()
for filename in files:
entry = CacheEntry(cache_path, filename)
entry._load_cache()
entry.parse_key()
entry._rebuild()
if entry.site not in seen and 'watchlistraw' in entry._data:
refresh(entry.site)
seen.add(entry.site)
def refresh_all(sysop=False):
"""Reload watchlists for all wikis where a watchlist is already present."""
cache_path = CachedRequest._get_cache_dir()
files = os.listdir(cache_path)
seen = []
for filename in files:
entry = CacheEntry(cache_path, filename)
entry._load_cache()
entry.parse_key()
entry._rebuild()
if entry.site not in seen:
if entry._data['watchlistraw']:
refresh(entry.site)
seen.append(entry.site)
def refresh_all(sysop=False):
"""Reload watchlists for all wikis where a watchlist is already present."""
cache_path = CachedRequest._get_cache_dir()
files = os.listdir(cache_path)
seen = []
for filename in files:
entry = CacheEntry(cache_path, filename)
entry._load_cache()
entry.parse_key()
entry._rebuild()
if entry.site not in seen:
if entry._data.get('watchlistraw'):
refresh(entry.site, sysop)
seen.append(entry.site)
def setUp(self):
"""Initialize the fake requests."""
super(DryCachedRequestTests, self).setUp()
self.parms = {"action": "query", "meta": "userinfo"}
self.req = CachedRequest(expiry=1, site=self.basesite, parameters=self.parms)
self.expreq = CachedRequest(expiry=0, site=self.basesite, parameters=self.parms)
self.diffreq = CachedRequest(expiry=1, site=self.basesite, parameters={"action": "query", "meta": "siteinfo"})
self.diffsite = CachedRequest(expiry=1, site=self.altsite, parameters={"action": "query", "meta": "userinfo"})
# When using ** the paramters are still unicode
self.deprecated_explicit = CachedRequest(expiry=1, site=self.basesite, action="query", meta="userinfo")
self.deprecated_asterisks = CachedRequest(expiry=1, site=self.basesite, **self.parms)
def test_cachefile_path_different_users(self):
# Mock basesite object to test this.
class MockSite(pywikibot.site.APISite):
_loginstatus = pywikibot.site.LoginStatus.NOT_ATTEMPTED
_namespaces = {2: 'User'}
def __init__(self):
self._user = '******'
def user(self):
return self._user
def encoding(self):
return 'utf-8'
def encodings(self):
return []
def _getsiteinfo(self):
self._siteinfo = {'case': 'first-letter'}
return {}
def __repr__(self):
return "MockSite()"
def __getattr__(self, attr):
raise Exception("Attribute %r not defined" % attr)
site = MockSite()
req = CachedRequest(expiry=1, site=site, action='query', meta='siteinfo')
anonpath = req._cachefile_path()
site._userinfo = {'name': u'user'}
site._loginstatus = 0
req = CachedRequest(expiry=1, site=site, action='query', meta='siteinfo')
userpath = req._cachefile_path()
self.assertNotEqual(anonpath, userpath)
site._userinfo = {'name': u'sysop'}
site._loginstatus = 1
req = CachedRequest(expiry=1, site=site, action='query', meta='siteinfo')
sysoppath = req._cachefile_path()
self.assertNotEqual(anonpath, sysoppath)
self.assertNotEqual(userpath, sysoppath)
def test_cachefile_path_different_users(self):
req = CachedRequest(expiry=1, site=self.mocksite, action="query", meta="siteinfo")
anonpath = req._cachefile_path()
self.mocksite._userinfo = {"name": "MyUser"}
self.mocksite._loginstatus = 0
req = CachedRequest(expiry=1, site=self.mocksite, action="query", meta="siteinfo")
userpath = req._cachefile_path()
self.assertNotEqual(anonpath, userpath)
self.mocksite._userinfo = {"name": "MySysop"}
self.mocksite._loginstatus = 1
req = CachedRequest(expiry=1, site=self.mocksite, action="query", meta="siteinfo")
sysoppath = req._cachefile_path()
self.assertNotEqual(anonpath, sysoppath)
self.assertNotEqual(userpath, sysoppath)
def setUp(self):
"""Initialize the fake requests."""
super(DryCachedRequestTests, self).setUp()
self.parms = {'action': 'query',
'meta': 'userinfo'}
self.req = CachedRequest(expiry=1, site=self.basesite,
parameters=self.parms)
self.expreq = CachedRequest(expiry=0, site=self.basesite,
parameters=self.parms)
self.diffreq = CachedRequest(
expiry=1, site=self.basesite,
parameters={'action': 'query', 'meta': 'siteinfo'})
self.diffsite = CachedRequest(
expiry=1, site=self.altsite,
parameters={'action': 'query', 'meta': 'userinfo'})
# When using ** the paramters are still unicode
self.deprecated_explicit = CachedRequest(
expiry=1, site=self.basesite, action='query', meta='userinfo')
self.deprecated_asterisks = CachedRequest(
expiry=1, site=self.basesite, **self.parms)
def test_unicode(self):
self.mocksite._userinfo = {"name": "محمد الفلسطيني"}
self.mocksite._loginstatus = 0
req = CachedRequest(expiry=1, site=self.mocksite, action="query", meta="siteinfo")
en_user_path = req._cachefile_path()
self.mocksite._namespaces = {2: ["مستخدم"]}
req = CachedRequest(expiry=1, site=self.mocksite, action="query", meta="siteinfo")
expect = "MockSite()User(User:محمد الفلسطيني)" + "[('action', 'query'), ('meta', 'siteinfo')]"
self.assertEqual(repr(req._uniquedescriptionstr()), repr(expect))
self.assertEqual(req._uniquedescriptionstr().encode("utf-8"), expect.encode("utf-8"))
ar_user_path = req._cachefile_path()
self.assertEqual(en_user_path, ar_user_path)
def test_cachefile_path_different_users(self):
req = CachedRequest(expiry=1, site=self.mocksite,
action='query', meta='siteinfo')
anonpath = req._cachefile_path()
self.mocksite._userinfo = {'name': u'MyUser'}
self.mocksite._loginstatus = 0
req = CachedRequest(expiry=1, site=self.mocksite,
action='query', meta='siteinfo')
userpath = req._cachefile_path()
self.assertNotEqual(anonpath, userpath)
self.mocksite._userinfo = {'name': u'MySysop'}
self.mocksite._loginstatus = 1
req = CachedRequest(expiry=1, site=self.mocksite,
action='query', meta='siteinfo')
sysoppath = req._cachefile_path()
self.assertNotEqual(anonpath, sysoppath)
self.assertNotEqual(userpath, sysoppath)
def test_cachefile_path_different_users(self):
"""Test and compare the file paths when different usernames are used."""
req = CachedRequest(expiry=1, site=self.mocksite,
parameters={'action': 'query', 'meta': 'siteinfo'})
anonpath = req._cachefile_path()
self.mocksite._userinfo = {'name': u'MyUser'}
self.mocksite._loginstatus = 0
req = CachedRequest(expiry=1, site=self.mocksite,
parameters={'action': 'query', 'meta': 'siteinfo'})
userpath = req._cachefile_path()
self.assertNotEqual(anonpath, userpath)
self.mocksite._userinfo = {'name': u'MySysop'}
self.mocksite._loginstatus = 1
req = CachedRequest(expiry=1, site=self.mocksite,
parameters={'action': 'query', 'meta': 'siteinfo'})
sysoppath = req._cachefile_path()
self.assertNotEqual(anonpath, sysoppath)
self.assertNotEqual(userpath, sysoppath)
def test_unicode(self):
self.mocksite._userinfo = {'name': u'محمد الفلسطيني'}
self.mocksite._loginstatus = 0
req = CachedRequest(expiry=1, site=self.mocksite,
action='query', meta='siteinfo')
en_user_path = req._cachefile_path()
self.mocksite._namespaces = {2: [u'مستخدم']}
req = CachedRequest(expiry=1, site=self.mocksite,
action='query', meta='siteinfo')
expect = u'MockSite()User(User:محمد الفلسطيني)' + \
"[('action', 'query'), ('meta', 'siteinfo')]"
self.assertEqual(repr(req._uniquedescriptionstr()), repr(expect))
self.assertEqual(req._uniquedescriptionstr().encode('utf-8'),
expect.encode('utf-8'))
ar_user_path = req._cachefile_path()
self.assertEqual(en_user_path, ar_user_path)
class DryAPITests(unittest.TestCase):
def setUp(self):
self.basesite = pywikibot.Site('en', 'wikipedia')
self.altsite = pywikibot.Site('de', 'wikipedia')
self.parms = {'site': self.basesite,
'action': 'query',
'meta': 'userinfo'}
self.req = CachedRequest(expiry=1, **self.parms)
self.expreq = CachedRequest(expiry=0, **self.parms)
self.diffreq = CachedRequest(expiry=1, site=self.basesite, action='query', meta='siteinfo')
self.diffsite = CachedRequest(expiry=1, site=self.altsite, action='query', meta='userinfo')
def test_expiry_formats(self):
self.assertEqual(self.req.expiry, CachedRequest(datetime.timedelta(days=1), **self.parms).expiry)
def test_get_cache_dir(self):
retval = self.req._get_cache_dir()
self.assertIn('apicache', retval)
def test_create_file_name(self):
self.assertEqual(self.req._create_file_name(), self.req._create_file_name())
self.assertEqual(self.req._create_file_name(), self.expreq._create_file_name())
self.assertNotEqual(self.req._create_file_name(), self.diffreq._create_file_name())
def test_cachefile_path(self):
self.assertEqual(self.req._cachefile_path(), self.req._cachefile_path())
self.assertEqual(self.req._cachefile_path(), self.expreq._cachefile_path())
self.assertNotEqual(self.req._cachefile_path(), self.diffreq._cachefile_path())
self.assertNotEqual(self.req._cachefile_path(), self.diffsite._cachefile_path())
def test_cachefile_path_different_users(self):
# Mock basesite object to test this.
class MockSite(pywikibot.site.APISite):
_loginstatus = pywikibot.site.LoginStatus.NOT_ATTEMPTED
_namespaces = {2: 'User'}
def __init__(self):
self._user = '******'
def user(self):
return self._user
def encoding(self):
return 'utf-8'
def encodings(self):
return []
def _getsiteinfo(self):
self._siteinfo = {'case': 'first-letter'}
return {}
def __repr__(self):
return "MockSite()"
def __getattr__(self, attr):
raise Exception("Attribute %r not defined" % attr)
site = MockSite()
req = CachedRequest(expiry=1, site=site, action='query', meta='siteinfo')
anonpath = req._cachefile_path()
site._userinfo = {'name': u'user'}
site._loginstatus = 0
req = CachedRequest(expiry=1, site=site, action='query', meta='siteinfo')
userpath = req._cachefile_path()
self.assertNotEqual(anonpath, userpath)
site._userinfo = {'name': u'sysop'}
site._loginstatus = 1
req = CachedRequest(expiry=1, site=site, action='query', meta='siteinfo')
sysoppath = req._cachefile_path()
self.assertNotEqual(anonpath, sysoppath)
self.assertNotEqual(userpath, sysoppath)
def test_expired(self):
self.assertFalse(self.req._expired(datetime.datetime.now()))
self.assertTrue(self.req._expired(datetime.datetime.now() - datetime.timedelta(days=2)))
def test_query_constructor(self):
qGen1 = QueryGenerator(action="query", meta="siteinfo")
qGen2 = QueryGenerator(meta="siteinfo")
self.assertEqual(str(qGen1.request), str(qGen2.request))
class DryAPITests(unittest.TestCase):
def setUp(self):
self.parms = {'site': pywikibot.Site('en'),
'action': 'query',
'meta': 'userinfo'}
self.req = CachedRequest(expiry=1, **self.parms)
self.expreq = CachedRequest(expiry=0, **self.parms)
self.diffreq = CachedRequest(expiry=1, site=pywikibot.Site('en'), action='query', meta='siteinfo')
self.diffsite = CachedRequest(expiry=1, site=pywikibot.Site('de'), action='query', meta='userinfo')
def test_expiry_formats(self):
self.assertEqual(self.req.expiry, CachedRequest(datetime.timedelta(days=1), **self.parms).expiry)
def test_get_cache_dir(self):
retval = self.req._get_cache_dir()
self.assertIn('apicache', retval)
def test_create_file_name(self):
self.assertEqual(self.req._create_file_name(), self.req._create_file_name())
self.assertEqual(self.req._create_file_name(), self.expreq._create_file_name())
self.assertNotEqual(self.req._create_file_name(), self.diffreq._create_file_name())
def test_cachefile_path(self):
self.assertEqual(self.req._cachefile_path(), self.req._cachefile_path())
self.assertEqual(self.req._cachefile_path(), self.expreq._cachefile_path())
self.assertNotEqual(self.req._cachefile_path(), self.diffreq._cachefile_path())
self.assertNotEqual(self.req._cachefile_path(), self.diffsite._cachefile_path())
def test_expired(self):
self.assertFalse(self.req._expired(datetime.datetime.now()))
self.assertTrue(self.req._expired(datetime.datetime.now() - datetime.timedelta(days=2)))
class DryCachedRequestTests(TestCase):
"""Test CachedRequest using real site objects."""
sites = {
'basesite': {
'family': 'wikipedia',
'code': 'en',
},
'altsite': {
'family': 'wikipedia',
'code': 'de',
},
}
dry = True
def setUp(self):
super(DryCachedRequestTests, self).setUp()
self.basesite = self.get_site('basesite')
self.altsite = self.get_site('altsite')
self.parms = {'site': self.basesite,
'action': 'query',
'meta': 'userinfo'}
self.req = CachedRequest(expiry=1, **self.parms)
self.expreq = CachedRequest(expiry=0, **self.parms)
self.diffreq = CachedRequest(expiry=1, site=self.basesite, action='query', meta='siteinfo')
self.diffsite = CachedRequest(expiry=1, site=self.altsite, action='query', meta='userinfo')
def test_expiry_formats(self):
self.assertEqual(self.req.expiry, CachedRequest(datetime.timedelta(days=1), **self.parms).expiry)
def test_expired(self):
self.assertFalse(self.req._expired(datetime.datetime.now()))
self.assertTrue(self.req._expired(datetime.datetime.now() - datetime.timedelta(days=2)))
def test_get_cache_dir(self):
retval = self.req._get_cache_dir()
self.assertIn('apicache', retval)
def test_create_file_name(self):
self.assertEqual(self.req._create_file_name(), self.req._create_file_name())
self.assertEqual(self.req._create_file_name(), self.expreq._create_file_name())
self.assertNotEqual(self.req._create_file_name(), self.diffreq._create_file_name())
def test_cachefile_path(self):
self.assertEqual(self.req._cachefile_path(), self.req._cachefile_path())
self.assertEqual(self.req._cachefile_path(), self.expreq._cachefile_path())
self.assertNotEqual(self.req._cachefile_path(), self.diffreq._cachefile_path())
self.assertNotEqual(self.req._cachefile_path(), self.diffsite._cachefile_path())
def test_expiry_formats(self):
self.assertEqual(
self.req.expiry,
CachedRequest(datetime.timedelta(days=1), **self.parms).expiry)
tests = loader.loadTestsFromNames(test_list)
suite = unittest.TestSuite()
suite.addTests(tests)
return suite
def load_tests(loader=unittest.loader.defaultTestLoader,
tests=None,
pattern=None):
"""Load the default modules."""
return collector(loader)
CachedRequest._get_cache_dir = staticmethod(
lambda *args: CachedRequest._make_dir(_cache_dir))
# Travis-CI builds are set to retry twice, which aims to reduce the number
# of 'red' builds caused by intermittant server problems, while also avoiding
# the builds taking a long time due to retries.
# The following allows builds to retry twice, but higher default values are
# overridden here to restrict retries to only 1, so developer builds fail more
# frequently in code paths resulting from mishandled server problems.
if config.max_retries > 2:
print('max_retries reduced from %d to 1 for tests' % config.max_retries)
config.max_retries = 1
cache_misses = 0
cache_hits = 0
def setUp(self):
"""Initialize the fake requests."""
super(DryCachedRequestTests, self).setUp()
self.parms = {'action': 'query', 'meta': 'userinfo'}
self.req = CachedRequest(expiry=1,
site=self.basesite,
parameters=self.parms)
self.expreq = CachedRequest(expiry=0,
site=self.basesite,
parameters=self.parms)
self.diffreq = CachedRequest(expiry=1,
site=self.basesite,
parameters={
'action': 'query',
'meta': 'siteinfo'
})
self.diffsite = CachedRequest(expiry=1,
site=self.altsite,
parameters={
'action': 'query',
'meta': 'userinfo'
})
# When using ** the paramters are still unicode
self.deprecated_explicit = CachedRequest(expiry=1,
site=self.basesite,
action='query',
meta='userinfo')
self.deprecated_asterisks = CachedRequest(expiry=1,
site=self.basesite,
**self.parms)
class DryCachedRequestTests(SiteAttributeTestCase):
"""Test CachedRequest using real site objects."""
sites = {
'basesite': {
'family': 'wikipedia',
'code': 'en',
},
'altsite': {
'family': 'wikipedia',
'code': 'de',
},
}
dry = True
def setUp(self):
"""Initialize the fake requests."""
super(DryCachedRequestTests, self).setUp()
self.parms = {'action': 'query', 'meta': 'userinfo'}
self.req = CachedRequest(expiry=1,
site=self.basesite,
parameters=self.parms)
self.expreq = CachedRequest(expiry=0,
site=self.basesite,
parameters=self.parms)
self.diffreq = CachedRequest(expiry=1,
site=self.basesite,
parameters={
'action': 'query',
'meta': 'siteinfo'
})
self.diffsite = CachedRequest(expiry=1,
site=self.altsite,
parameters={
'action': 'query',
'meta': 'userinfo'
})
# When using ** the paramters are still unicode
self.deprecated_explicit = CachedRequest(expiry=1,
site=self.basesite,
action='query',
meta='userinfo')
self.deprecated_asterisks = CachedRequest(expiry=1,
site=self.basesite,
**self.parms)
def test_expiry_formats(self):
"""Test using a timedelta as expiry."""
self.assertEqual(
self.req.expiry,
CachedRequest(datetime.timedelta(days=1),
site=self.basesite,
parameters=self.parms).expiry)
def test_expired(self):
"""Test if the request is expired."""
self.assertFalse(self.req._expired(datetime.datetime.now()))
self.assertTrue(
self.req._expired(datetime.datetime.now() -
datetime.timedelta(days=2)))
def test_parameter_types(self):
"""Test _uniquedescriptionstr is identical using different ways."""
# This test is done as create_file_name and cachefile_path only use
# the hashed name which is not very helpful
self.assertEqual(self.req._uniquedescriptionstr(),
self.req._uniquedescriptionstr())
self.assertEqual(self.req._uniquedescriptionstr(),
self.expreq._uniquedescriptionstr())
self.assertEqual(self.req._uniquedescriptionstr(),
self.deprecated_explicit._uniquedescriptionstr())
self.assertEqual(self.req._uniquedescriptionstr(),
self.deprecated_asterisks._uniquedescriptionstr())
self.assertNotEqual(self.req._uniquedescriptionstr(),
self.diffreq._uniquedescriptionstr())
self.assertNotEqual(self.req._uniquedescriptionstr(),
self.diffsite._uniquedescriptionstr())
def test_get_cache_dir(self):
"""Test that 'apicache' is in the cache dir."""
retval = self.req._get_cache_dir()
self.assertIn('apicache', retval)
def test_create_file_name(self):
"""Test the file names for the cache."""
self.assertEqual(self.req._create_file_name(),
self.req._create_file_name())
self.assertEqual(self.req._create_file_name(),
self.expreq._create_file_name())
self.assertEqual(self.req._create_file_name(),
self.deprecated_explicit._create_file_name())
self.assertEqual(self.req._create_file_name(),
self.deprecated_asterisks._create_file_name())
self.assertNotEqual(self.req._create_file_name(),
self.diffreq._create_file_name())
def test_cachefile_path(self):
"""Test the file paths for the cache."""
self.assertEqual(self.req._cachefile_path(),
self.req._cachefile_path())
self.assertEqual(self.req._cachefile_path(),
self.expreq._cachefile_path())
self.assertEqual(self.req._cachefile_path(),
self.deprecated_explicit._cachefile_path())
self.assertEqual(self.req._cachefile_path(),
self.deprecated_asterisks._cachefile_path())
self.assertNotEqual(self.req._cachefile_path(),
self.diffreq._cachefile_path())
self.assertNotEqual(self.req._cachefile_path(),
self.diffsite._cachefile_path())
class DryCachedRequestTests(SiteAttributeTestCase):
"""Test CachedRequest using real site objects."""
sites = {"basesite": {"family": "wikipedia", "code": "en"}, "altsite": {"family": "wikipedia", "code": "de"}}
dry = True
def setUp(self):
super(DryCachedRequestTests, self).setUp()
self.parms = {"site": self.basesite, "action": "query", "meta": "userinfo"}
self.req = CachedRequest(expiry=1, **self.parms)
self.expreq = CachedRequest(expiry=0, **self.parms)
self.diffreq = CachedRequest(expiry=1, site=self.basesite, action="query", meta="siteinfo")
self.diffsite = CachedRequest(expiry=1, site=self.altsite, action="query", meta="userinfo")
def test_expiry_formats(self):
self.assertEqual(self.req.expiry, CachedRequest(datetime.timedelta(days=1), **self.parms).expiry)
def test_expired(self):
self.assertFalse(self.req._expired(datetime.datetime.now()))
self.assertTrue(self.req._expired(datetime.datetime.now() - datetime.timedelta(days=2)))
def test_get_cache_dir(self):
retval = self.req._get_cache_dir()
self.assertIn("apicache", retval)
def test_create_file_name(self):
self.assertEqual(self.req._create_file_name(), self.req._create_file_name())
self.assertEqual(self.req._create_file_name(), self.expreq._create_file_name())
self.assertNotEqual(self.req._create_file_name(), self.diffreq._create_file_name())
def test_cachefile_path(self):
self.assertEqual(self.req._cachefile_path(), self.req._cachefile_path())
self.assertEqual(self.req._cachefile_path(), self.expreq._cachefile_path())
self.assertNotEqual(self.req._cachefile_path(), self.diffreq._cachefile_path())
self.assertNotEqual(self.req._cachefile_path(), self.diffsite._cachefile_path())
class DryAPITests(unittest.TestCase):
def setUp(self):
self.basesite = pywikibot.Site('en', 'wikipedia')
self.altsite = pywikibot.Site('de', 'wikipedia')
self.parms = {'site': self.basesite,
'action': 'query',
'meta': 'userinfo'}
self.req = CachedRequest(expiry=1, **self.parms)
self.expreq = CachedRequest(expiry=0, **self.parms)
self.diffreq = CachedRequest(expiry=1, site=self.basesite, action='query', meta='siteinfo')
self.diffsite = CachedRequest(expiry=1, site=self.altsite, action='query', meta='userinfo')
def test_expiry_formats(self):
self.assertEqual(self.req.expiry, CachedRequest(datetime.timedelta(days=1), **self.parms).expiry)
def test_get_cache_dir(self):
retval = self.req._get_cache_dir()
self.assertIn('apicache', retval)
def test_create_file_name(self):
self.assertEqual(self.req._create_file_name(), self.req._create_file_name())
self.assertEqual(self.req._create_file_name(), self.expreq._create_file_name())
self.assertNotEqual(self.req._create_file_name(), self.diffreq._create_file_name())
def test_cachefile_path(self):
self.assertEqual(self.req._cachefile_path(), self.req._cachefile_path())
self.assertEqual(self.req._cachefile_path(), self.expreq._cachefile_path())
self.assertNotEqual(self.req._cachefile_path(), self.diffreq._cachefile_path())
self.assertNotEqual(self.req._cachefile_path(), self.diffsite._cachefile_path())
def test_expired(self):
self.assertFalse(self.req._expired(datetime.datetime.now()))
self.assertTrue(self.req._expired(datetime.datetime.now() - datetime.timedelta(days=2)))
def test_query_constructor(self):
qGen1 = QueryGenerator(action="query", meta="siteinfo")
qGen2 = QueryGenerator(meta="siteinfo")
self.assertEqual(str(qGen1.request), str(qGen2.request))
class DryCachedRequestTests(SiteAttributeTestCase):
"""Test CachedRequest using real site objects."""
sites = {
'basesite': {
'family': 'wikipedia',
'code': 'en',
},
'altsite': {
'family': 'wikipedia',
'code': 'de',
},
}
dry = True
def setUp(self):
"""Initialize the fake requests."""
super(DryCachedRequestTests, self).setUp()
self.parms = {'action': 'query',
'meta': 'userinfo'}
self.req = CachedRequest(expiry=1, site=self.basesite,
parameters=self.parms)
self.expreq = CachedRequest(expiry=0, site=self.basesite,
parameters=self.parms)
self.diffreq = CachedRequest(
expiry=1, site=self.basesite,
parameters={'action': 'query', 'meta': 'siteinfo'})
self.diffsite = CachedRequest(
expiry=1, site=self.altsite,
parameters={'action': 'query', 'meta': 'userinfo'})
# When using ** the paramters are still unicode
self.deprecated_explicit = CachedRequest(
expiry=1, site=self.basesite, action='query', meta='userinfo')
self.deprecated_asterisks = CachedRequest(
expiry=1, site=self.basesite, **self.parms)
def test_expiry_formats(self):
"""Test using a timedelta as expiry."""
self.assertEqual(self.req.expiry,
CachedRequest(datetime.timedelta(days=1), site=self.basesite,
parameters=self.parms).expiry)
def test_expired(self):
"""Test if the request is expired."""
self.assertFalse(self.req._expired(datetime.datetime.now()))
self.assertTrue(self.req._expired(datetime.datetime.now() - datetime.timedelta(days=2)))
def test_parameter_types(self):
"""Test _uniquedescriptionstr is identical using different ways."""
# This test is done as create_file_name and cachefile_path only use
# the hashed name which is not very helpful
self.assertEqual(self.req._uniquedescriptionstr(),
self.req._uniquedescriptionstr())
self.assertEqual(self.req._uniquedescriptionstr(),
self.expreq._uniquedescriptionstr())
self.assertEqual(self.req._uniquedescriptionstr(),
self.deprecated_explicit._uniquedescriptionstr())
self.assertEqual(self.req._uniquedescriptionstr(),
self.deprecated_asterisks._uniquedescriptionstr())
self.assertNotEqual(self.req._uniquedescriptionstr(),
self.diffreq._uniquedescriptionstr())
self.assertNotEqual(self.req._uniquedescriptionstr(),
self.diffsite._uniquedescriptionstr())
def test_get_cache_dir(self):
"""Test that 'apicache' is in the cache dir."""
retval = self.req._get_cache_dir()
self.assertIn('apicache', retval)
def test_create_file_name(self):
"""Test the file names for the cache."""
self.assertEqual(self.req._create_file_name(), self.req._create_file_name())
self.assertEqual(self.req._create_file_name(), self.expreq._create_file_name())
self.assertEqual(self.req._create_file_name(),
self.deprecated_explicit._create_file_name())
self.assertEqual(self.req._create_file_name(),
self.deprecated_asterisks._create_file_name())
self.assertNotEqual(self.req._create_file_name(), self.diffreq._create_file_name())
def test_cachefile_path(self):
"""Test the file paths for the cache."""
self.assertEqual(self.req._cachefile_path(), self.req._cachefile_path())
self.assertEqual(self.req._cachefile_path(), self.expreq._cachefile_path())
self.assertEqual(self.req._cachefile_path(),
self.deprecated_explicit._cachefile_path())
self.assertEqual(self.req._cachefile_path(),
self.deprecated_asterisks._cachefile_path())
self.assertNotEqual(self.req._cachefile_path(), self.diffreq._cachefile_path())
self.assertNotEqual(self.req._cachefile_path(), self.diffsite._cachefile_path())
class DryAPITests(unittest.TestCase):
def setUp(self):
self.parms = {'site': pywikibot.Site('en'),
'action': 'query',
'meta': 'userinfo'}
self.req = CachedRequest(expiry=1, **self.parms)
self.expreq = CachedRequest(expiry=0, **self.parms)
self.diffreq = CachedRequest(expiry=1, site=pywikibot.Site('en'), action='query', meta='siteinfo')
self.diffsite = CachedRequest(expiry=1, site=pywikibot.Site('de'), action='query', meta='userinfo')
def test_expiry_formats(self):
self.assertEqual(self.req.expiry, CachedRequest(datetime.timedelta(days=1), **self.parms).expiry)
def test_get_cache_dir(self):
retval = self.req._get_cache_dir()
self.assertIn('apicache', retval)
def test_create_file_name(self):
self.assertEqual(self.req._create_file_name(), self.req._create_file_name())
self.assertEqual(self.req._create_file_name(), self.expreq._create_file_name())
self.assertNotEqual(self.req._create_file_name(), self.diffreq._create_file_name())
def test_cachefile_path(self):
self.assertEqual(self.req._cachefile_path(), self.req._cachefile_path())
self.assertEqual(self.req._cachefile_path(), self.expreq._cachefile_path())
self.assertNotEqual(self.req._cachefile_path(), self.diffreq._cachefile_path())
self.assertNotEqual(self.req._cachefile_path(), self.diffsite._cachefile_path())
def test_expired(self):
self.assertFalse(self.req._expired(datetime.datetime.now()))
self.assertTrue(self.req._expired(datetime.datetime.now() - datetime.timedelta(days=2)))
class DryCachedRequestTests(TestCase):
sites = {
'basesite': {
'family': 'wikipedia',
'code': 'en',
},
'altsite': {
'family': 'wikipedia',
'code': 'de',
},
}
def setUp(self):
super(DryCachedRequestTests, self).setUp()
self.basesite = self.get_site('basesite')
self.altsite = self.get_site('altsite')
self.parms = {
'site': self.basesite,
'action': 'query',
'meta': 'userinfo'
}
self.req = CachedRequest(expiry=1, **self.parms)
self.expreq = CachedRequest(expiry=0, **self.parms)
self.diffreq = CachedRequest(expiry=1,
site=self.basesite,
action='query',
meta='siteinfo')
self.diffsite = CachedRequest(expiry=1,
site=self.altsite,
action='query',
meta='userinfo')
def test_expiry_formats(self):
self.assertEqual(
self.req.expiry,
CachedRequest(datetime.timedelta(days=1), **self.parms).expiry)
def test_expired(self):
self.assertFalse(self.req._expired(datetime.datetime.now()))
self.assertTrue(
self.req._expired(datetime.datetime.now() -
datetime.timedelta(days=2)))
def test_get_cache_dir(self):
retval = self.req._get_cache_dir()
self.assertIn('apicache', retval)
def test_create_file_name(self):
self.assertEqual(self.req._create_file_name(),
self.req._create_file_name())
self.assertEqual(self.req._create_file_name(),
self.expreq._create_file_name())
self.assertNotEqual(self.req._create_file_name(),
self.diffreq._create_file_name())
def test_cachefile_path(self):
self.assertEqual(self.req._cachefile_path(),
self.req._cachefile_path())
self.assertEqual(self.req._cachefile_path(),
self.expreq._cachefile_path())
self.assertNotEqual(self.req._cachefile_path(),
self.diffreq._cachefile_path())
self.assertNotEqual(self.req._cachefile_path(),
self.diffsite._cachefile_path())
class DryAPITests(unittest.TestCase):
def setUp(self):
self.basesite = pywikibot.Site('en', 'wikipedia')
self.altsite = pywikibot.Site('de', 'wikipedia')
self.parms = {
'site': self.basesite,
'action': 'query',
'meta': 'userinfo'
}
self.req = CachedRequest(expiry=1, **self.parms)
self.expreq = CachedRequest(expiry=0, **self.parms)
self.diffreq = CachedRequest(expiry=1,
site=self.basesite,
action='query',
meta='siteinfo')
self.diffsite = CachedRequest(expiry=1,
site=self.altsite,
action='query',
meta='userinfo')
def test_expiry_formats(self):
self.assertEqual(
self.req.expiry,
CachedRequest(datetime.timedelta(days=1), **self.parms).expiry)
def test_get_cache_dir(self):
retval = self.req._get_cache_dir()
self.assertIn('apicache', retval)
def test_create_file_name(self):
self.assertEqual(self.req._create_file_name(),
self.req._create_file_name())
self.assertEqual(self.req._create_file_name(),
self.expreq._create_file_name())
self.assertNotEqual(self.req._create_file_name(),
self.diffreq._create_file_name())
def test_cachefile_path(self):
self.assertEqual(self.req._cachefile_path(),
self.req._cachefile_path())
self.assertEqual(self.req._cachefile_path(),
self.expreq._cachefile_path())
self.assertNotEqual(self.req._cachefile_path(),
self.diffreq._cachefile_path())
self.assertNotEqual(self.req._cachefile_path(),
self.diffsite._cachefile_path())
def test_cachefile_path_different_users(self):
# Mock basesite object to test this.
class MockSite(pywikibot.site.APISite):
_loginstatus = pywikibot.site.LoginStatus.NOT_ATTEMPTED
_namespaces = {2: 'User'}
def __init__(self):
self._user = '******'
def user(self):
return self._user
def encoding(self):
return 'utf-8'
def encodings(self):
return []
def _getsiteinfo(self):
self._siteinfo = {'case': 'first-letter'}
return {}
def __repr__(self):
return "MockSite()"
def __getattr__(self, attr):
raise Exception("Attribute %r not defined" % attr)
site = MockSite()
req = CachedRequest(expiry=1,
site=site,
action='query',
meta='siteinfo')
anonpath = req._cachefile_path()
site._userinfo = {'name': u'user'}
site._loginstatus = 0
req = CachedRequest(expiry=1,
site=site,
action='query',
meta='siteinfo')
userpath = req._cachefile_path()
self.assertNotEqual(anonpath, userpath)
site._userinfo = {'name': u'sysop'}
site._loginstatus = 1
req = CachedRequest(expiry=1,
site=site,
action='query',
meta='siteinfo')
sysoppath = req._cachefile_path()
self.assertNotEqual(anonpath, sysoppath)
self.assertNotEqual(userpath, sysoppath)
def test_expired(self):
self.assertFalse(self.req._expired(datetime.datetime.now()))
self.assertTrue(
self.req._expired(datetime.datetime.now() -
datetime.timedelta(days=2)))
def test_query_constructor(self):
qGen1 = QueryGenerator(action="query", meta="siteinfo")
qGen2 = QueryGenerator(meta="siteinfo")
self.assertEqual(str(qGen1.request), str(qGen2.request))
def setUp(self):
"""Initialize the fake requests."""
super().setUp()
self.parms = {'action': 'query', 'meta': 'userinfo'}
self.req = CachedRequest(expiry=1,
site=self.basesite,
parameters=self.parms)
self.expreq = CachedRequest(expiry=0,
site=self.basesite,
parameters=self.parms)
self.diffreq = CachedRequest(expiry=1,
site=self.basesite,
parameters={
'action': 'query',
'meta': 'siteinfo'
})
self.diffsite = CachedRequest(expiry=1,
site=self.altsite,
parameters={
'action': 'query',
'meta': 'userinfo'
})
# When using ** the parameters are still unicode
with suppress_warnings('Instead of using kwargs ', FutureWarning):
self.deprecated_explicit = CachedRequest(expiry=1,
site=self.basesite,
action='query',
meta='userinfo')
self.deprecated_asterisks = CachedRequest(expiry=1,
site=self.basesite,
**self.parms)
test_list.append(module_class_name)
tests = loader.loadTestsFromNames(test_list)
suite = unittest.TestSuite()
suite.addTests(tests)
return suite
def load_tests(loader=unittest.loader.defaultTestLoader,
tests=None, pattern=None):
"""Load the default modules."""
return collector(loader)
CachedRequest._get_cache_dir = staticmethod(
lambda *args: CachedRequest._make_dir(_cache_dir))
# Travis-CI builds are set to retry twice, which aims to reduce the number
# of 'red' builds caused by intermittant server problems, while also avoiding
# the builds taking a long time due to retries.
# The following allows builds to retry twice, but higher default values are
# overridden here to restrict retries to only 1, so developer builds fail more
# frequently in code paths resulting from mishandled server problems.
if config.max_retries > 2:
print('max_retries reduced from %d to 1 for tests' % config.max_retries)
config.max_retries = 1
cache_misses = 0
cache_hits = 0