Beispiel #1
0
def _getTarball(url, directory, sha256):
    logger.debug('registry: get: %s' % url)

    if not sha256:
        logger.warn('tarball %s has no hash to check' % url)

    try:
        access_common.unpackFromCache(sha256, directory)
    except KeyError as e:
        # figure out which registry we're fetching this tarball from (if any)
        # and add appropriate headers
        registry = Registry_Base_URL

        for source in _getSources():
            if ('type' in source and source['type'] == 'registry' and
                 'url' in source and url.startswith(source['url'])):
                registry = source['url']
                break

        request_headers = _headersForRegistry(registry)

        logger.debug('GET %s, %s', url, request_headers)
        response = requests.get(url, headers=request_headers, allow_redirects=True, stream=True)
        response.raise_for_status()

        access_common.unpackTarballStream(
                    stream = response,
            into_directory = directory,
                      hash = {'sha256':sha256},
                 cache_key = sha256,
               origin_info = {'url':url}
        )
Beispiel #2
0
def _getTarball(url, directory, sha256):
    logger.debug('registry: get: %s' % url)

    if not sha256:
        logger.warn('tarball %s has no hash to check' % url)

    try:
        access_common.unpackFromCache(sha256, directory)
    except KeyError as e:
        # figure out which registry we're fetching this tarball from (if any)
        # and add appropriate headers
        registry = Registry_Base_URL

        for source in _getSources():
            if ('type' in source and source['type'] == 'registry'
                    and 'url' in source and url.startswith(source['url'])):
                registry = source['url']
                break

        request_headers = _headersForRegistry(registry)

        logger.debug('GET %s, %s', url, request_headers)
        response = requests.get(url,
                                headers=request_headers,
                                allow_redirects=True,
                                stream=True)
        response.raise_for_status()

        access_common.unpackTarballStream(stream=response,
                                          into_directory=directory,
                                          hash={'sha256': sha256},
                                          cache_key=sha256,
                                          origin_info={'url': url})
Beispiel #3
0
def _getTarball(url, into_directory, cache_key, origin_info=None):
    '''unpack the specified tarball url into the specified directory'''

    try:
        access_common.unpackFromCache(cache_key, into_directory)
    except KeyError as e:
        tok = settings.getProperty('github', 'authtoken')
        headers = {}
        if tok is not None:
            headers['Authorization'] = 'token ' + str(tok)

        logger.debug('GET %s', url)
        response = requests.get(url,
                                allow_redirects=True,
                                stream=True,
                                headers=headers)
        response.raise_for_status()

        logger.debug('getting file: %s', url)
        logger.debug('headers: %s', response.headers)
        response.raise_for_status()

        # github doesn't exposes hashes of the archives being downloaded as far
        # as I can tell :(
        access_common.unpackTarballStream(stream=response,
                                          into_directory=into_directory,
                                          hash={},
                                          cache_key=cache_key,
                                          origin_info=origin_info)
Beispiel #4
0
def _getTarball(url, into_directory, cache_key, origin_info=None):
    '''unpack the specified tarball url into the specified directory'''

    try:
        access_common.unpackFromCache(cache_key, into_directory)
    except KeyError as e:
        tok = settings.getProperty('github', 'authtoken')
        headers = {}
        if tok is not None:
            headers['Authorization'] = 'token ' + str(tok)

        logger.debug('GET %s', url)
        response = requests.get(url, allow_redirects=True, stream=True, headers=headers)
        response.raise_for_status()

        logger.debug('getting file: %s', url)
        logger.debug('headers: %s', response.headers)
        response.raise_for_status()

        # github doesn't exposes hashes of the archives being downloaded as far
        # as I can tell :(
        access_common.unpackTarballStream(
                    stream = response,
            into_directory = into_directory,
                      hash = {},
                 cache_key = cache_key,
               origin_info = origin_info
        )
Beispiel #5
0
def main():

    url = 'https://blobs.yottabuild.org/targets/stk3700-0.0.0.tar.gz'
    print 'get:', url
    resource = Resource(url)
    response = resource.get()
    print 'response:', response
    print 'headers:', dict(response.headers.items())
    print 'body len:', len(response.body_string())


    url = 'https://blobs.yottabuild.org/targets/stk3700-0.0.0.tar.gz'
    headers = { }
    print 'get:', url
    resource = Resource(url, pool=connection_pool.getPool(), follow_redirect=True)
    response = resource.get(
        headers = headers
    )
    print 'response:', response
    print 'headers:', dict(response.headers.items())
    print 'body len:', len(response.body_string())

    url = 'https://blobs.yottabuild.org/targets/stk3700-0.0.0.tar.gz'
    headers = { }
    print 'get:', url
    resource = Resource(url, pool=connection_pool.getPool(), follow_redirect=True)
    response = resource.get(
        headers = headers
    )
    access_common.unpackTarballStream(response.body_stream(), '/tmp/yttest/blobs/')