示例#1
0
    def _mirror_try(self, func, url, kw):
        gr = GrabRequest()
        gr.func = func
        gr.url  = url
        gr.kw   = dict(kw)
        self._load_gr(gr)
        gr.errors = []

        for k in self.options:
            try: del kw[k]
            except KeyError: pass

        tries = 0
        while 1:
            tries += 1
            mirrorchoice = self._get_mirror(gr)
            fullurl = self._join_url(mirrorchoice['mirror'], gr.url)
            grabber = mirrorchoice.get('grabber') or self.grabber
            # apply mirrorchoice kwargs on top of grabber.opts
            opts = grabber.opts.derive(**mirrorchoice.get('kwargs', {}))
            func_ref = getattr(grabber, func)
            if DEBUG: DEBUG.info('MIRROR: trying %s -> %s', url, fullurl)
            try:
                return func_ref( *(fullurl,), opts=opts, **kw )
            except URLGrabError, e:
                if DEBUG: DEBUG.info('MIRROR: failed')
                gr.errors.append((fullurl, exception2msg(e)))
                obj = CallbackObject()
                obj.exception = e
                obj.mirror = mirrorchoice['mirror']
                obj.relative_url = gr.url
                obj.url = fullurl
                obj.tries = tries
                self._failure(gr, obj)
示例#2
0
    def _mirror_try(self, func, url, kw):
        gr = GrabRequest()
        gr.func = func
        gr.url  = url
        gr.kw   = dict(kw)
        self._load_gr(gr)
        gr.errors = []

        for k in self.options:
            try: del kw[k]
            except KeyError: pass

        tries = 0
        while 1:
            tries += 1
            mirrorchoice = self._get_mirror(gr)
            fullurl = self._join_url(mirrorchoice['mirror'], gr.url)
            grabber = mirrorchoice.get('grabber') or self.grabber
            # apply mirrorchoice kwargs on top of grabber.opts
            opts = grabber.opts.derive(**mirrorchoice.get('kwargs', {}))
            func_ref = getattr(grabber, func)
            if DEBUG: DEBUG.info('MIRROR: trying %s -> %s', url, fullurl)
            try:
                return func_ref( *(fullurl,), opts=opts, **kw )
            except URLGrabError, e:
                if DEBUG: DEBUG.info('MIRROR: failed')
                gr.errors.append((fullurl, exception2msg(e)))
                obj = CallbackObject()
                obj.exception = e
                obj.mirror = mirrorchoice['mirror']
                obj.relative_url = gr.url
                obj.url = fullurl
                obj.tries = tries
                self._failure(gr, obj)
示例#3
0
文件: mirror.py 项目: 01org/mic
    def _mirror_try(self, func, url, kw):
        gr = GrabRequest()
        gr.func = func
        gr.url  = url
        gr.kw   = dict(kw)
        self._load_gr(gr)

        for k in self.options:
            try: del kw[k]
            except KeyError: pass

        while 1:
            mirrorchoice = self._get_mirror(gr)
            fullurl = self._join_url(mirrorchoice['mirror'], gr.url)
            kwargs = dict(mirrorchoice.get('kwargs', {}))
            kwargs.update(kw)
            grabber = mirrorchoice.get('grabber') or self.grabber
            func_ref = getattr(grabber, func)
            if DEBUG: DEBUG.info('MIRROR: trying %s -> %s', url, fullurl)
            try:
                return func_ref( *(fullurl,), **kwargs )
            except URLGrabError, e:
                if DEBUG: DEBUG.info('MIRROR: failed')
                obj = CallbackObject()
                obj.exception = e
                obj.mirror = mirrorchoice['mirror']
                obj.relative_url = gr.url
                obj.url = fullurl
                self._failure(gr, obj)
示例#4
0
    def _mirror_try(self, func, url, kw):
        gr = GrabRequest()
        gr.func = func
        gr.url  = url
        gr.kw   = dict(kw)
        self._load_gr(gr)

        for k in self.options:
            try: del kw[k]
            except KeyError: pass

        while 1:
            mirrorchoice = self._get_mirror(gr)
            fullurl = self._join_url(mirrorchoice['mirror'], gr.url)
            kwargs = dict(mirrorchoice.get('kwargs', {}))
            kwargs.update(kw)
            grabber = mirrorchoice.get('grabber') or self.grabber
            func_ref = getattr(grabber, func)
            if DEBUG: DEBUG.info('MIRROR: trying %s -> %s', url, fullurl)
            try:
                return func_ref( *(fullurl,), **kwargs )
            except URLGrabError, e:
                if DEBUG: DEBUG.info('MIRROR: failed')
                obj = CallbackObject()
                obj.exception = e
                obj.mirror = mirrorchoice['mirror']
                obj.relative_url = gr.url
                obj.url = fullurl
                self._failure(gr, obj)
示例#5
0
    def increment_mirror(self, gr, action={}):
        """Tell the mirror object increment the mirror index

        This increments the mirror index, which amounts to telling the
        mirror object to use a different mirror (for this and future
        downloads).

        This is a SEMI-public method.  It will be called internally,
        and you may never need to call it.  However, it is provided
        (and is made public) so that the calling program can increment
        the mirror choice for methods like urlopen.  For example, with
        urlopen, there's no good way for the mirror group to know that
        an error occurs mid-download (it's already returned and given
        you the file object).
        
        remove  ---  can have several values
           0   do not remove the mirror from the list
           1   remove the mirror for this download only
           2   remove the mirror permanently

        beware of remove=0 as it can lead to infinite loops
        """
        badmirror = gr.mirrors[gr._next]

        self._lock.acquire()
        try:
            ind = self.mirrors.index(badmirror)
        except ValueError:
            pass
        else:
            if action.get('remove_master', 0):
                del self.mirrors[ind]
            elif self._next == ind and action.get('increment_master', 1):
                self._next += 1
            if self._next >= len(self.mirrors): self._next = 0
        self._lock.release()
        
        if action.get('remove', 1):
            del gr.mirrors[gr._next]
        elif action.get('increment', 1):
            gr._next += 1
        if gr._next >= len(gr.mirrors): gr._next = 0

        if DEBUG:
            grm = [m['mirror'] for m in gr.mirrors]
            DEBUG.info('GR   mirrors: [%s] %i', ' '.join(grm), gr._next)
            selfm = [m['mirror'] for m in self.mirrors]
            DEBUG.info('MAIN mirrors: [%s] %i', ' '.join(selfm), self._next)
示例#6
0
    def increment_mirror(self, gr, action={}):
        """Tell the mirror object increment the mirror index

        This increments the mirror index, which amounts to telling the
        mirror object to use a different mirror (for this and future
        downloads).

        This is a SEMI-public method.  It will be called internally,
        and you may never need to call it.  However, it is provided
        (and is made public) so that the calling program can increment
        the mirror choice for methods like urlopen.  For example, with
        urlopen, there's no good way for the mirror group to know that
        an error occurs mid-download (it's already returned and given
        you the file object).
        
        remove  ---  can have several values
           0   do not remove the mirror from the list
           1   remove the mirror for this download only
           2   remove the mirror permanently

        beware of remove=0 as it can lead to infinite loops
        """
        badmirror = gr.mirrors[gr._next]

        self._lock.acquire()
        try:
            ind = self.mirrors.index(badmirror)
        except ValueError:
            pass
        else:
            if action.get('remove_master', 0):
                del self.mirrors[ind]
            elif self._next == ind and action.get('increment_master', 1):
                self._next += 1
            if self._next >= len(self.mirrors): self._next = 0
        self._lock.release()
        
        if action.get('remove', 1):
            del gr.mirrors[gr._next]
        elif action.get('increment', 1):
            gr._next += 1
        if gr._next >= len(gr.mirrors): gr._next = 0

        if DEBUG:
            grm = [m['mirror'] for m in gr.mirrors]
            DEBUG.info('GR   mirrors: [%s] %i', ' '.join(grm), gr._next)
            selfm = [m['mirror'] for m in self.mirrors]
            DEBUG.info('MAIN mirrors: [%s] %i', ' '.join(selfm), self._next)