def _mirror_try(self, func, url, kw): gr = GrabRequest() gr.func = func gr.url = url gr.kw = dict(kw) self._load_gr(gr) gr.errors = [] for k in self.options: try: del kw[k] except KeyError: pass tries = 0 while 1: tries += 1 mirrorchoice = self._get_mirror(gr) fullurl = self._join_url(mirrorchoice['mirror'], gr.url) grabber = mirrorchoice.get('grabber') or self.grabber # apply mirrorchoice kwargs on top of grabber.opts opts = grabber.opts.derive(**mirrorchoice.get('kwargs', {})) func_ref = getattr(grabber, func) if DEBUG: DEBUG.info('MIRROR: trying %s -> %s', url, fullurl) try: return func_ref( *(fullurl,), opts=opts, **kw ) except URLGrabError, e: if DEBUG: DEBUG.info('MIRROR: failed') gr.errors.append((fullurl, exception2msg(e))) obj = CallbackObject() obj.exception = e obj.mirror = mirrorchoice['mirror'] obj.relative_url = gr.url obj.url = fullurl obj.tries = tries self._failure(gr, obj)
def _mirror_try(self, func, url, kw): gr = GrabRequest() gr.func = func gr.url = url gr.kw = dict(kw) self._load_gr(gr) for k in self.options: try: del kw[k] except KeyError: pass while 1: mirrorchoice = self._get_mirror(gr) fullurl = self._join_url(mirrorchoice['mirror'], gr.url) kwargs = dict(mirrorchoice.get('kwargs', {})) kwargs.update(kw) grabber = mirrorchoice.get('grabber') or self.grabber func_ref = getattr(grabber, func) if DEBUG: DEBUG.info('MIRROR: trying %s -> %s', url, fullurl) try: return func_ref( *(fullurl,), **kwargs ) except URLGrabError, e: if DEBUG: DEBUG.info('MIRROR: failed') obj = CallbackObject() obj.exception = e obj.mirror = mirrorchoice['mirror'] obj.relative_url = gr.url obj.url = fullurl self._failure(gr, obj)