def __init__(self, config: SectionProxy, descend: bool = False) -> None:
RedFetcher.__init__(self, config)
self.descend = descend # type: bool
self.check_done = False # type: bool
self.partial_support = None # type: bool
self.inm_support = None # type: bool
self.ims_support = None # type: bool
self.gzip_support = None # type: bool
self.gzip_savings = 0 # type: int
self._task_map = set(
[None]) # type: Set[RedFetcher] # None is the original request
self.subreqs = {
ac.check_name: ac(config, self)
for ac in active_checks
} # type: ignore
self.response.once("content_available", self.run_active_checks)
def _finish_check() -> None:
self.finish_check(None)
self.on("fetch_done", _finish_check)
self.links = {} # type: Dict[str, Set[str]]
self.link_count = 0 # type: int
self.linked = [
] # type: List[Tuple[HttpResource, str]] # linked HttpResources
self._link_parser = link_parse.HTMLLinkParser(self.response,
[self.process_link])
self.response.on("chunk", self._link_parser.feed)
def check(self) -> None:
modified_headers = self.modify_request_headers(
list(self.base.request.headers))
RedFetcher.set_request(self, self.base.request.uri,
self.base.request.method, modified_headers,
self.base.request.payload)
RedFetcher.check(self)
def __init__(self, config: SectionProxy,
base_resource: "HttpResource") -> None:
self.config = config
self.base = base_resource # type: HttpResource
RedFetcher.__init__(self, config)
self.check_done = False
self.on("fetch_done", self._check_done)
def __init__(self, base_resource, name):
self.base = base_resource
req_hdrs = self.modify_req_hdrs()
RedFetcher.__init__(self,
self.base.request.uri,
self.base.request.method,
req_hdrs,
self.base.request.payload,
self.base.status_cb,
[],
name
)
self.base.subreqs[name] = self
def __init__(self, uri, method="GET", req_hdrs=None, req_body=None,
status_cb=None, body_procs=None, descend=False):
orig_req_hdrs = req_hdrs or []
new_req_hdrs = orig_req_hdrs + [(u'Accept-Encoding', u'gzip')]
RedFetcher.__init__(self, uri, method, new_req_hdrs, req_body,
status_cb, body_procs, name=method)
self.descend = descend
self.response.set_link_procs([self.process_link])
self.subreqs = {} # sub-requests' RedState objects
self.links = {} # {type: set(link...)}
self.link_count = 0
self.linked = [] # list of linked HttpResources (if descend=True)
self.orig_req_hdrs = orig_req_hdrs
self.partial_support = None
self.inm_support = None
self.ims_support = None
self.gzip_support = None
self.gzip_savings = 0
def __init__(self, config: SectionProxy, descend: bool = False) -> None:
RedFetcher.__init__(self, config)
self.descend = descend # type: bool
self.check_done = False # type: bool
self.partial_support = None # type: bool
self.inm_support = None # type: bool
self.ims_support = None # type: bool
self.gzip_support = None # type: bool
self.gzip_savings = 0 # type: int
self._task_map = set([None]) # type: Set[RedFetcher] # None is the original request
self.subreqs = {ac.check_name:ac(config, self) for ac in active_checks} # type: ignore
self.response.once("content_available", self.run_active_checks)
def _finish_check() -> None:
self.finish_check(None)
self.on("fetch_done", _finish_check)
self.links = {} # type: Dict[str, Set[str]]
self.link_count = 0 # type: int
self.linked = [] # type: List[Tuple[HttpResource, str]] # linked HttpResources
self._link_parser = link_parse.HTMLLinkParser(self.response, [self.process_link])
self.response.on("chunk", self._link_parser.feed)
self.last_err_pos, offset = self.getpos()
if self.err:
self.err(message)
class BadErrorIReallyMeanIt(Exception):
"""See http://bugs.python.org/issue8885 for why this is necessary."""
pass
if __name__ == "__main__":
import sys
import thor
from redbot.resource.fetch import RedFetcher # pylint: disable=ungrouped-imports
T = RedFetcher()
T.set_request(sys.argv[1], req_hdrs=[('Accept-Encoding', "gzip")])
def show_link(base: str, link: str, tag: str, title: str) -> None:
print("* [%s] %s -- %s" % (tag, base, link))
P = HTMLLinkParser(T.response, [show_link], sys.stderr.write)
@thor.events.on(T)
def fetch_done() -> None:
print('done')
thor.stop()
@thor.events.on(T)
def status(msg: str) -> None:
print(msg)
raise BadErrorIReallyMeanIt()
else:
self.last_err_pos, offset = self.getpos()
if self.err:
self.err(message)
class BadErrorIReallyMeanIt(Exception):
"""See http://bugs.python.org/issue8885 for why this is necessary."""
pass
if __name__ == "__main__":
import sys
import thor
from redbot.resource.fetch import RedFetcher # pylint: disable=ungrouped-imports
T = RedFetcher()
T.set_request(sys.argv[1], req_hdrs=[('Accept-Encoding', "gzip")])
def show_link(base: str, link: str, tag: str, title: str) -> None:
print("* [%s] %s -- %s" % (tag, base, link))
P = HTMLLinkParser(T.response, [show_link], sys.stderr.write)
@thor.events.on(T)
def fetch_done() -> None:
print('done')
thor.stop()
@thor.events.on(T)
def status(msg: str) -> None:
print(msg)
@thor.events.on(T.response)
def chunk(decoded_chunk: bytes) -> None:
P.feed(decoded_chunk.decode(P.message.character_encoding, 'ignore'))
T.check()
def check(self) -> None:
modified_headers = self.modify_request_headers(list(self.base.request.headers))
RedFetcher.set_request(
self, self.base.request.uri, self.base.request.method, modified_headers, self.base.request.payload
)
RedFetcher.check(self)
def __init__(self, base_resource: "HttpResource") -> None:
self.base = base_resource # type: HttpResource
RedFetcher.__init__(self)
self.check_done = False
self.on("fetch_done", self._check_done)
def __init__(self, config: SectionProxy, base_resource: 'HttpResource') -> None:
self.config = config
self.base = base_resource # type: HttpResource
RedFetcher.__init__(self, config)
self.check_done = False
self.on('fetch_done', self._check_done)
else:
self.last_err_pos, offset = self.getpos()
if self.err:
self.err(message)
class BadErrorIReallyMeanIt(Exception):
"""See http://bugs.python.org/issue8885 for why this is necessary."""
pass
if __name__ == "__main__":
import sys
from configparser import ConfigParser
import thor
from redbot.resource.fetch import RedFetcher # pylint: disable=ungrouped-imports
T = RedFetcher(ConfigParser()['DEFAULT'])
T.set_request(sys.argv[1], req_hdrs=[('Accept-Encoding', "gzip")])
def show_link(base: str, link: str, tag: str, title: str) -> None:
print("* [%s] %s -- %s" % (tag, base, link))
P = HTMLLinkParser(T.response, [show_link], sys.stderr.write)
@thor.events.on(T)
def fetch_done() -> None:
print('done')
thor.stop()
@thor.events.on(T)
def status(msg: str) -> None:
print(msg)
@thor.events.on(T.response)
def chunk(decoded_chunk: bytes) -> None:
P.feed(decoded_chunk.decode(P.message.character_encoding, 'ignore'))
T.check()
def __init__(self, base_resource: 'HttpResource') -> None:
self.base = base_resource # type: HttpResource
RedFetcher.__init__(self)
self.check_done = False
self.on('fetch_done', self._check_done)
self.err(message)
class BadErrorIReallyMeanIt(Exception):
"""See http://bugs.python.org/issue8885 for why this is necessary."""
pass
if __name__ == "__main__":
import sys
from configparser import ConfigParser
import thor
from redbot.resource.fetch import RedFetcher # pylint: disable=ungrouped-imports
T = RedFetcher(ConfigParser()["DEFAULT"])
T.set_request(sys.argv[1], req_hdrs=[("Accept-Encoding", "gzip")])
def show_link(base: str, link: str, tag: str, title: str) -> None:
print("* [%s] %s -- %s" % (tag, base, link))
P = HTMLLinkParser(T.response, [show_link], sys.stderr.write)
@thor.events.on(T)
def fetch_done() -> None:
print("done")
thor.stop()
@thor.events.on(T)
def status(msg: str) -> None:
print(msg)
