def _get(self, action, show, proxy, timeout):
"""
make HTTP request and cache response
"""
silent = self.flags['silent']
if action in self.cache:
if action != 'imageinfo':
utils.stderr("+ %s results in cache" % action, silent)
return
else:
self.cache[action] = {}
if self.flags.get('skip') and action in self.flags['skip']:
if not self.flags['silent']:
utils.stderr("+ skipping %s" % action)
return
# make the request
qobj = WPToolsQuery(lang=self.params['lang'],
variant=self.params.get('variant'),
wiki=self.params.get('wiki'))
qstr = self._query(action, qobj)
self.cache[action]['query'] = qstr
req = self._request(proxy, timeout)
response = req.get(qstr, qobj.status)
self.cache[action]['response'] = response
self.cache[action]['info'] = req.info
self._set_data(action)
if show:
self.show()
def get(args):
"""
invoke wptools and assemble selected output
"""
html = args.H
lang = args.l
nowrap = args.n
query = args.q
silent = args.s
title = args.t
verbose = args.v
wiki = args.w
start = time.time()
if query:
qobj = WPToolsQuery(lang=lang, wiki=wiki)
if title:
return qobj.query(title)
return qobj.random()
page = wptools.page(title,
lang=lang,
silent=silent,
verbose=verbose,
wiki=wiki)
try:
page.get_query()
except (StandardError, ValueError, LookupError):
return "NOT_FOUND"
if not page.data.get('extext'):
out = page.cache['query']['query']
out = _page_text(page, nowrap)
if html:
out = _page_html(page)
print("%5.3f seconds" % (time.time() - start), file=sys.stderr)
try:
return out.encode('utf-8')
except KeyError:
return out
def get(args):
"""
invoke wptools and assemble selected output
"""
html = args.H
lang = args.l
nowrap = args.n
query = args.q
silent = args.s
title = args.t
verbose = args.v
wiki = args.w
if query:
qobj = WPToolsQuery(lang=lang, wiki=wiki)
if title:
return qobj.query(title)
return qobj.random()
page = wptools.page(title, lang=lang, silent=silent,
verbose=verbose, wiki=wiki)
try:
page.get_query()
except (StandardError, ValueError, LookupError):
return "NOT_FOUND"
if not page.data.get('extext'):
out = page.cache['query']['query']
out = _page_text(page, nowrap)
if html:
out = _page_html(page)
try:
return out.encode('utf-8')
except KeyError:
return out
def _get(self, action, show, proxy, timeout):
"""
make HTTP request and cache response
"""
silent = self.flags['silent']
if action in self.cache:
if action != 'imageinfo' and action != 'labels':
utils.stderr("+ %s results in cache" % action, silent)
return
else:
self.cache[action] = {}
if self.flags.get('skip') and action in self.flags['skip']:
if not self.flags['silent']:
utils.stderr("+ skipping %s" % action)
return
if 'requests' not in self.data:
self.data['requests'] = []
if len(self.data['requests']) >= self.REQUEST_LIMIT:
raise StopIteration("Hit REQUEST_LIMIT = %d" % self.REQUEST_LIMIT)
if self.data['requests'] and self.REQUEST_DELAY:
utils.stderr("REQUEST_DELAY = %d seconds" % self.REQUEST_DELAY)
sleep(self.REQUEST_DELAY)
# make the request
qobj = WPToolsQuery(lang=self.params['lang'],
variant=self.params.get('variant'),
wiki=self.params.get('wiki'),
endpoint=self.params.get('endpoint'))
qstr = self._query(action, qobj)
req = self._request(proxy, timeout)
response = req.get(qstr, qobj.status)
self.cache[action]['query'] = qstr
self.cache[action]['response'] = response
self.cache[action]['info'] = req.info
self.data['requests'].append(action)
self._set_data(action)
if show and not self.flags.get('silent'):
self.show()