Esempio n. 1
0
    def progstr():
        if size_is_known:
            if size == 0:
                pct = 100.0  # Seems logical.
            else:
                pct = 100.0 * bytes_read / size

            print("\r\t%s (file progress %s/%s [%.0f%%])" %
                  (pfx, IECUnitConverter.bytes_to_unit(bytes_read), sizestr,
                   pct),
                  end='')
        else:
            print("\r%s (download %s/unknown)" %
                  (pfx, IECUnitConverter.bytes_to_unit(bytes_read)),
                  end='')
Esempio n. 2
0
    def progstr():
        if size_is_known:
            if size == 0:
                pct = 100.0     # Seems logical.
            else:
                pct = 100.0 * bytes_read / size

            print ("\r\t%s (file progress %s/%s [%.0f%%])" %
                   (pfx,
                    IECUnitConverter.bytes_to_unit(
                        bytes_read),
                    sizestr,
                    pct),
                   end='')
        else:
            print("\r%s (download %s/unknown)" %
                  (pfx,
                   IECUnitConverter.bytes_to_unit(bytes_read)),
                  end='')
Esempio n. 3
0
def fetch_contents(fpath,
                   opts,
                   root='',
                   no_trim=False,
                   for_filehash=None,
                   short_name=None,
                   file_count_number=None,
                   file_count_total=None,
                   remote_flag=True,
                   include_in_total=True):
    '''
    Wrap a fetch, which may be from a file or URL depending on the options.

    Returns None on a 404, re-raises the Exception otherwise.
    '''

    fullpath = fpath

    if not no_trim and opts.trim_path and root:
        fullpath = os.path.join(opts.source_url, fpath)

    log.debug("fetch_contents: %s", fullpath)

    fh = None
    if for_filehash is not None:
        fh = for_filehash

    # Try to get a friendly name.
    fname = fpath
    if short_name:
        fname = short_name
    elif fh is not None:
        fname = fh.fpath

    filecountstr = ''
    if file_count_number is not None:
        if file_count_total is not None:
            if file_count_total == 0:
                pct = 100.0
            else:
                pct = 100.0 * file_count_number / file_count_total
            filecountstr = ' [object %d/%d (%.0f%%)]' % (file_count_number,
                                                         file_count_total, pct)
        else:
            filecountstr = ' [file %d]' % file_count_number

    # This part of the progress meter doesn't change, so cache it.
    pfx = "%s" % (filecountstr)

    if not opts.quiet:
        if opts.progress:
            progress_spacer = "\n\t"
            pfx = "\r" + pfx
        else:
            progress_spacer = ''

        print('F: %s%s%s' % (fname, progress_spacer, pfx), end='')

    outfile = ''

    try:
        if remote_flag and include_in_total:
            opts.stats.content_fetches += 1
        else:
            opts.stats.metadata_fetches += 1

        url = urllib2.urlopen(fullpath)
    except urllib2.HTTPError as e:
        if e.code == 404:
            resp = BaseHTTPRequestHandler.responses
            log.warn("Failed to retrieve '%s': %s", fullpath, resp[404][0])
        return None
    except urllib2.URLError as e:
        log.warn("Failed to retrieve '%s': %s", fullpath, e)
        return None

    size = 0
    size_is_known = False
    block_size = int(opts.fetch_blocksize)

    if fh is not None:
        size = fh.size
        size_is_known = True

    elif 'content-length' in url.info()['content-length']:
        size = int(url.info()['content-length'])
        size_is_known = True

    if opts.progress and size_is_known:
        sizestr = IECUnitConverter.bytes_to_unit(size)

    bytes_read = 0
    more_to_read = True

    def progstr():
        if size_is_known:
            if size == 0:
                pct = 100.0  # Seems logical.
            else:
                pct = 100.0 * bytes_read / size

            print("\r\t%s (file progress %s/%s [%.0f%%])" %
                  (pfx, IECUnitConverter.bytes_to_unit(bytes_read), sizestr,
                   pct),
                  end='')
        else:
            print("\r%s (download %s/unknown)" %
                  (pfx, IECUnitConverter.bytes_to_unit(bytes_read)),
                  end='')

    if opts.progress:
        progstr()

    while more_to_read:
        # if log.isEnabledFor(logging.DEBUG):
        #     log.debug("Read: %d bytes (%d/%d)",
        #               block_size,bytes_read, size)
        try:
            new_bytes = url.read(block_size)
            nblen = len(new_bytes)
            if not new_bytes:
                more_to_read = False
            else:
                bytes_read += nblen
                if remote_flag:
                    if include_in_total:
                        opts.stats.bytes_transferred += nblen
                    else:
                        opts.stats.metadata_bytes_transferred += nblen

                outfile += new_bytes
                if opts.progress:
                    progstr()

        except urllib2.URLError as e:
            log.warn("'%s' fetch failed: %s", str(e))
            raise e

    if not opts.progress:
        # In progress mode, give the caller a chance to add information.
        print('')

    if size_is_known and bytes_read != size:
        # That's an error. No need for a cryptochecksum to tell that.
        log.warn("'%s': Fetched %d bytes, expected %d bytes", fname,
                 bytes_read, size)
        return None

    return outfile
Esempio n. 4
0
def fetch_contents(fpath, opts, root='', no_trim=False, for_filehash=None,
                   short_name=None, file_count_number=None,
                   file_count_total=None, remote_flag=True,
                   include_in_total=True):
    '''
    Wrap a fetch, which may be from a file or URL depending on the options.

    Returns None on a 404, re-raises the Exception otherwise.
    '''

    fullpath = fpath

    if not no_trim and opts.trim_path and root:
        fullpath = os.path.join(opts.source_url, fpath)

    log.debug("fetch_contents: %s", fullpath)

    fh = None
    if for_filehash is not None:
        fh = for_filehash

    # Try to get a friendly name.
    fname = fpath
    if short_name:
        fname = short_name
    elif fh is not None:
        fname = fh.fpath

    filecountstr = ''
    if file_count_number is not None:
        if file_count_total is not None:
            if file_count_total == 0:
                pct = 100.0
            else:
                pct = 100.0 * file_count_number / file_count_total
            filecountstr = ' [object %d/%d (%.0f%%)]' % (
                file_count_number,
                file_count_total,
                pct)
        else:
            filecountstr = ' [file %d]' % file_count_number

    # This part of the progress meter doesn't change, so cache it.
    pfx = "%s" % (filecountstr)

    if not opts.quiet:
        if opts.progress:
            progress_spacer = "\n\t"
            pfx = "\r" + pfx
        else:
            progress_spacer = ''

        print('F: %s%s%s' % (fname, progress_spacer, pfx), end='')

    outfile = ''

    try:
        if remote_flag and include_in_total:
            opts.stats.content_fetches += 1
        else:
            opts.stats.metadata_fetches += 1

        url = urllib2.urlopen(fullpath)
    except urllib2.HTTPError as e:
        if e.code == 404:
            resp = BaseHTTPRequestHandler.responses
            log.warn("Failed to retrieve '%s': %s", fullpath, resp[404][0])
        return None
    except urllib2.URLError as e:
        log.warn("Failed to retrieve '%s': %s", fullpath, e)
        return None

    size = 0
    size_is_known = False
    block_size = int(opts.fetch_blocksize)

    if fh is not None:
        size = fh.size
        size_is_known = True

    elif 'content-length' in url.info()['content-length']:
        size = int(url.info()['content-length'])
        size_is_known = True

    if opts.progress and size_is_known:
        sizestr = IECUnitConverter.bytes_to_unit(size)

    bytes_read = 0
    more_to_read = True

    def progstr():
        if size_is_known:
            if size == 0:
                pct = 100.0     # Seems logical.
            else:
                pct = 100.0 * bytes_read / size

            print ("\r\t%s (file progress %s/%s [%.0f%%])" %
                   (pfx,
                    IECUnitConverter.bytes_to_unit(
                        bytes_read),
                    sizestr,
                    pct),
                   end='')
        else:
            print("\r%s (download %s/unknown)" %
                  (pfx,
                   IECUnitConverter.bytes_to_unit(bytes_read)),
                  end='')

    if opts.progress:
        progstr()

    while more_to_read:
        # if log.isEnabledFor(logging.DEBUG):
        #     log.debug("Read: %d bytes (%d/%d)",
        #               block_size,bytes_read, size)
        try:
            new_bytes = url.read(block_size)
            nblen = len(new_bytes)
            if not new_bytes:
                more_to_read = False
            else:
                bytes_read += nblen
                if remote_flag:
                    if include_in_total:
                        opts.stats.bytes_transferred += nblen
                    else:
                        opts.stats.metadata_bytes_transferred += nblen

                outfile += new_bytes
                if opts.progress:
                    progstr()

        except urllib2.URLError as e:
            log.warn("'%s' fetch failed: %s", str(e))
            raise e

    if not opts.progress:
        # In progress mode, give the caller a chance to add information.
        print('')

    if size_is_known and bytes_read != size:
        # That's an error. No need for a cryptochecksum to tell that.
        log.warn("'%s': Fetched %d bytes, expected %d bytes",
                 fname, bytes_read, size)
        return None

    return outfile