def _mirror_try(self, func, url, kw): gr = GrabRequest() gr.func = func gr.url = url gr.kw = dict(kw) self._load_gr(gr) for k in self.options: try: del kw[k] except KeyError: pass while 1: mirrorchoice = self._get_mirror(gr) fullurl = self._join_url(mirrorchoice['mirror'], gr.url) kwargs = dict(mirrorchoice.get('kwargs', {})) kwargs.update(kw) grabber = mirrorchoice.get('grabber') or self.grabber func_ref = getattr(grabber, func) if DEBUG: DEBUG.info('MIRROR: trying %s -> %s', url, fullurl) try: return func_ref( *(fullurl,), **kwargs ) except URLGrabError, e: if DEBUG: DEBUG.info('MIRROR: failed') obj = CallbackObject() obj.exception = e obj.mirror = mirrorchoice['mirror'] obj.relative_url = gr.url obj.url = fullurl self._failure(gr, obj)
def _mirror_try(self, func, url, kw): gr = GrabRequest() gr.func = func gr.url = url gr.kw = dict(kw) self._load_gr(gr) gr.errors = [] for k in self.options: try: del kw[k] except KeyError: pass tries = 0 while 1: tries += 1 mirrorchoice = self._get_mirror(gr) fullurl = self._join_url(mirrorchoice['mirror'], gr.url) grabber = mirrorchoice.get('grabber') or self.grabber # apply mirrorchoice kwargs on top of grabber.opts opts = grabber.opts.derive(**mirrorchoice.get('kwargs', {})) func_ref = getattr(grabber, func) if DEBUG: DEBUG.info('MIRROR: trying %s -> %s', url, fullurl) try: return func_ref( *(fullurl,), opts=opts, **kw ) except URLGrabError, e: if DEBUG: DEBUG.info('MIRROR: failed') gr.errors.append((fullurl, exception2msg(e))) obj = CallbackObject() obj.exception = e obj.mirror = mirrorchoice['mirror'] obj.relative_url = gr.url obj.url = fullurl obj.tries = tries self._failure(gr, obj)
def urlgrab(self, url, filename=None, **kwargs): kw = dict(kwargs) kw['filename'] = filename if kw.get('async'): # enable mirror failovers in async path kw['mirror_group'] = self, [], {}, set() kw['relative_url'] = url else: kw.pop('failfunc', None) func = 'urlgrab' try: return self._mirror_try(func, url, kw) except URLGrabError, e: obj = CallbackObject(url=url, filename=filename, exception=e, **kwargs) return _run_callback(kwargs.get('failfunc', _do_raise), obj)