Esempio n. 1
0
 def fetch(self, length=0):
     """
     Start urllib2 data fetching. If length is 0 the complete data will
     be fetched and send to the data and completed signal. If data is
     greater 0, the fd will read 'length' bytes and send it to the data
     signal until everything is read or no callback is connected anymore.
     The function returns the 'completed' signal which is an InProgress
     object.
     """
     t = ThreadCallable(self._fetch_thread, length)
     t.wait_on_exit = False
     return t()
Esempio n. 2
0
 def fetch(self, length=0):
     """
     Start urllib2 data fetching. If length is 0 the complete data will
     be fetched and send to the data and completed signal. If data is
     greater 0, the fd will read 'length' bytes and send it to the data
     signal until everything is read or no callback is connected anymore.
     The function returns the 'completed' signal which is an InProgress
     object.
     """
     t = ThreadCallable(self._fetch_thread, length)
     t.wait_on_exit = False
     return t()
Esempio n. 3
0
def _fetch_HTTP(url, filename, tmpname):
    """
    Fetch HTTP URL.
    """
    def download(url, filename, tmpname, status):
        with auth_handler_lock:
            auth_handler.retried = 0
            src = urllib2.urlopen(url)
            length = int(src.info().get('Content-Length', 0))
        if not tmpname:
            tmpname = filename
        if not os.path.exists(os.path.dirname(tmpname)):
            os.makedirs(os.path.dirname(tmpname))
        dst = open(tmpname, 'w')
        status.set(0, length)
        while True:
            data = src.read(1024)
            if len(data) == 0:
                src.close()
                dst.close()
                if length and os.stat(tmpname)[stat.ST_SIZE] != length:
                    # something went wrong
                    os.unlink(tmpname)
                    raise IOError('download %s failed' % url)
                if tmpname != filename:
                    os.rename(tmpname, filename)
                return True
            status.update(len(data))
            dst.write(data)

    if url.find(' ') > 0:
        # stupid url encoding in url
        url = url[:8+url[8:].find('/')] + \
              urllib.quote(url[8+url[8:].find('/'):])
    # FIXME: use kaa.threaded()
    s = InProgressStatus()
    t = ThreadCallable(download, url, filename, tmpname, s)
    t.wait_on_exit = False
    async = t()
    async .progress = s
    return async
Esempio n. 4
0
def _fetch_HTTP(url, filename, tmpname):
    """
    Fetch HTTP URL.
    """
    def download(url, filename, tmpname, status):
        with auth_handler_lock:
            auth_handler.retried = 0
            src = urllib2.urlopen(url)
            length = int(src.info().get('Content-Length', 0))
        if not tmpname:
            tmpname = filename
        if not os.path.exists(os.path.dirname(tmpname)):
            os.makedirs(os.path.dirname(tmpname))
        dst = open(tmpname, 'w')
        status.set(0, length)
        while True:
            data = src.read(1024)
            if len(data) == 0:
                src.close()
                dst.close()
                if length and os.stat(tmpname)[stat.ST_SIZE] != length:
                    # something went wrong
                    os.unlink(tmpname)
                    raise IOError('download %s failed' % url)
                if tmpname != filename:
                    os.rename(tmpname, filename)
                return True
            status.update(len(data))
            dst.write(data)

    if url.find(' ') > 0:
        # stupid url encoding in url
        url = url[:8+url[8:].find('/')] + \
              urllib.quote(url[8+url[8:].find('/'):])
    # FIXME: use kaa.threaded()
    s = InProgressStatus()
    t = ThreadCallable(download, url, filename, tmpname, s)
    t.wait_on_exit = False
    async = t()
    async.progress = s
    return async