def __init__(self, config: SectionProxy, descend: bool = False) -> None: RedFetcher.__init__(self, config) self.descend = descend # type: bool self.check_done = False # type: bool self.partial_support = None # type: bool self.inm_support = None # type: bool self.ims_support = None # type: bool self.gzip_support = None # type: bool self.gzip_savings = 0 # type: int self._task_map = set( [None]) # type: Set[RedFetcher] # None is the original request self.subreqs = { ac.check_name: ac(config, self) for ac in active_checks } # type: ignore self.response.once("content_available", self.run_active_checks) def _finish_check() -> None: self.finish_check(None) self.on("fetch_done", _finish_check) self.links = {} # type: Dict[str, Set[str]] self.link_count = 0 # type: int self.linked = [ ] # type: List[Tuple[HttpResource, str]] # linked HttpResources self._link_parser = link_parse.HTMLLinkParser(self.response, [self.process_link]) self.response.on("chunk", self._link_parser.feed)
def check(self) -> None: modified_headers = self.modify_request_headers( list(self.base.request.headers)) RedFetcher.set_request(self, self.base.request.uri, self.base.request.method, modified_headers, self.base.request.payload) RedFetcher.check(self)
def __init__(self, config: SectionProxy, base_resource: "HttpResource") -> None: self.config = config self.base = base_resource # type: HttpResource RedFetcher.__init__(self, config) self.check_done = False self.on("fetch_done", self._check_done)
def __init__(self, base_resource, name): self.base = base_resource req_hdrs = self.modify_req_hdrs() RedFetcher.__init__(self, self.base.request.uri, self.base.request.method, req_hdrs, self.base.request.payload, self.base.status_cb, [], name ) self.base.subreqs[name] = self
def __init__(self, uri, method="GET", req_hdrs=None, req_body=None, status_cb=None, body_procs=None, descend=False): orig_req_hdrs = req_hdrs or [] new_req_hdrs = orig_req_hdrs + [(u'Accept-Encoding', u'gzip')] RedFetcher.__init__(self, uri, method, new_req_hdrs, req_body, status_cb, body_procs, name=method) self.descend = descend self.response.set_link_procs([self.process_link]) self.subreqs = {} # sub-requests' RedState objects self.links = {} # {type: set(link...)} self.link_count = 0 self.linked = [] # list of linked HttpResources (if descend=True) self.orig_req_hdrs = orig_req_hdrs self.partial_support = None self.inm_support = None self.ims_support = None self.gzip_support = None self.gzip_savings = 0
def __init__(self, config: SectionProxy, descend: bool = False) -> None: RedFetcher.__init__(self, config) self.descend = descend # type: bool self.check_done = False # type: bool self.partial_support = None # type: bool self.inm_support = None # type: bool self.ims_support = None # type: bool self.gzip_support = None # type: bool self.gzip_savings = 0 # type: int self._task_map = set([None]) # type: Set[RedFetcher] # None is the original request self.subreqs = {ac.check_name:ac(config, self) for ac in active_checks} # type: ignore self.response.once("content_available", self.run_active_checks) def _finish_check() -> None: self.finish_check(None) self.on("fetch_done", _finish_check) self.links = {} # type: Dict[str, Set[str]] self.link_count = 0 # type: int self.linked = [] # type: List[Tuple[HttpResource, str]] # linked HttpResources self._link_parser = link_parse.HTMLLinkParser(self.response, [self.process_link]) self.response.on("chunk", self._link_parser.feed)
self.last_err_pos, offset = self.getpos() if self.err: self.err(message) class BadErrorIReallyMeanIt(Exception): """See http://bugs.python.org/issue8885 for why this is necessary.""" pass if __name__ == "__main__": import sys import thor from redbot.resource.fetch import RedFetcher # pylint: disable=ungrouped-imports T = RedFetcher() T.set_request(sys.argv[1], req_hdrs=[('Accept-Encoding', "gzip")]) def show_link(base: str, link: str, tag: str, title: str) -> None: print("* [%s] %s -- %s" % (tag, base, link)) P = HTMLLinkParser(T.response, [show_link], sys.stderr.write) @thor.events.on(T) def fetch_done() -> None: print('done') thor.stop() @thor.events.on(T) def status(msg: str) -> None: print(msg)
raise BadErrorIReallyMeanIt() else: self.last_err_pos, offset = self.getpos() if self.err: self.err(message) class BadErrorIReallyMeanIt(Exception): """See http://bugs.python.org/issue8885 for why this is necessary.""" pass if __name__ == "__main__": import sys import thor from redbot.resource.fetch import RedFetcher # pylint: disable=ungrouped-imports T = RedFetcher() T.set_request(sys.argv[1], req_hdrs=[('Accept-Encoding', "gzip")]) def show_link(base: str, link: str, tag: str, title: str) -> None: print("* [%s] %s -- %s" % (tag, base, link)) P = HTMLLinkParser(T.response, [show_link], sys.stderr.write) @thor.events.on(T) def fetch_done() -> None: print('done') thor.stop() @thor.events.on(T) def status(msg: str) -> None: print(msg) @thor.events.on(T.response) def chunk(decoded_chunk: bytes) -> None: P.feed(decoded_chunk.decode(P.message.character_encoding, 'ignore')) T.check()
def check(self) -> None: modified_headers = self.modify_request_headers(list(self.base.request.headers)) RedFetcher.set_request( self, self.base.request.uri, self.base.request.method, modified_headers, self.base.request.payload ) RedFetcher.check(self)
def __init__(self, base_resource: "HttpResource") -> None: self.base = base_resource # type: HttpResource RedFetcher.__init__(self) self.check_done = False self.on("fetch_done", self._check_done)
def __init__(self, config: SectionProxy, base_resource: 'HttpResource') -> None: self.config = config self.base = base_resource # type: HttpResource RedFetcher.__init__(self, config) self.check_done = False self.on('fetch_done', self._check_done)
else: self.last_err_pos, offset = self.getpos() if self.err: self.err(message) class BadErrorIReallyMeanIt(Exception): """See http://bugs.python.org/issue8885 for why this is necessary.""" pass if __name__ == "__main__": import sys from configparser import ConfigParser import thor from redbot.resource.fetch import RedFetcher # pylint: disable=ungrouped-imports T = RedFetcher(ConfigParser()['DEFAULT']) T.set_request(sys.argv[1], req_hdrs=[('Accept-Encoding', "gzip")]) def show_link(base: str, link: str, tag: str, title: str) -> None: print("* [%s] %s -- %s" % (tag, base, link)) P = HTMLLinkParser(T.response, [show_link], sys.stderr.write) @thor.events.on(T) def fetch_done() -> None: print('done') thor.stop() @thor.events.on(T) def status(msg: str) -> None: print(msg) @thor.events.on(T.response) def chunk(decoded_chunk: bytes) -> None: P.feed(decoded_chunk.decode(P.message.character_encoding, 'ignore')) T.check()
def __init__(self, base_resource: 'HttpResource') -> None: self.base = base_resource # type: HttpResource RedFetcher.__init__(self) self.check_done = False self.on('fetch_done', self._check_done)
self.err(message) class BadErrorIReallyMeanIt(Exception): """See http://bugs.python.org/issue8885 for why this is necessary.""" pass if __name__ == "__main__": import sys from configparser import ConfigParser import thor from redbot.resource.fetch import RedFetcher # pylint: disable=ungrouped-imports T = RedFetcher(ConfigParser()["DEFAULT"]) T.set_request(sys.argv[1], req_hdrs=[("Accept-Encoding", "gzip")]) def show_link(base: str, link: str, tag: str, title: str) -> None: print("* [%s] %s -- %s" % (tag, base, link)) P = HTMLLinkParser(T.response, [show_link], sys.stderr.write) @thor.events.on(T) def fetch_done() -> None: print("done") thor.stop() @thor.events.on(T) def status(msg: str) -> None: print(msg)