Example #1
0
def _getTarball(url, into_directory, cache_key, origin_info=None):
    '''unpack the specified tarball url into the specified directory'''

    try:
        access_common.unpackFromCache(cache_key, into_directory)
    except KeyError as e:
        tok = settings.getProperty('github', 'authtoken')
        headers = {}
        if tok is not None:
            headers['Authorization'] = 'token ' + str(tok)

        logger.debug('GET %s', url)
        response = requests.get(url, allow_redirects=True, stream=True, headers=headers)
        response.raise_for_status()

        logger.debug('getting file: %s', url)
        logger.debug('headers: %s', response.headers)
        response.raise_for_status()

        # github doesn't exposes hashes of the archives being downloaded as far
        # as I can tell :(
        access_common.unpackTarballStream(
                    stream = response,
            into_directory = into_directory,
                      hash = {},
                 cache_key = cache_key,
               origin_info = origin_info
        )
def _getTarball(url, directory, sha256):
    logger.debug('registry: get: %s' % url)

    if not sha256:
        logger.warn('tarball %s has no hash to check' % url)

    try:
        access_common.unpackFromCache(sha256, directory)
    except KeyError as e:
        # figure out which registry we're fetching this tarball from (if any)
        # and add appropriate headers
        registry = Registry_Base_URL

        for source in _getSources():
            if ('type' in source and source['type'] == 'registry' and
                 'url' in source and url.startswith(source['url'])):
                registry = source['url']
                break

        request_headers = _headersForRegistry(registry)

        logger.debug('GET %s, %s', url, request_headers)
        response = requests.get(url, headers=request_headers, allow_redirects=True, stream=True)
        response.raise_for_status()

        access_common.unpackTarballStream(
                    stream = response,
            into_directory = directory,
                      hash = {'sha256':sha256},
                 cache_key = sha256
        )
Example #3
0
def _getTarball(url, into_directory, cache_key):
    '''unpack the specified tarball url into the specified directory'''

    try:
        access_common.unpackFromCache(cache_key, into_directory)
    except KeyError as e:
        tok = settings.getProperty('github', 'authtoken')
        headers = {}
        if tok is not None:
            headers['Authorization'] = 'token ' + str(tok)

        logger.debug('GET %s', url)
        response = requests.get(url,
                                allow_redirects=True,
                                stream=True,
                                headers=headers)
        response.raise_for_status()

        logger.debug('getting file: %s', url)
        logger.debug('headers: %s', response.headers)
        response.raise_for_status()

        # github doesn't exposes hashes of the archives being downloaded as far
        # as I can tell :(
        access_common.unpackTarballStream(stream=response,
                                          into_directory=into_directory,
                                          hash={},
                                          cache_key=cache_key)
Example #4
0
def _getTarball(url, directory, sha256):
    logger.debug('registry: get: %s' % url)

    if not sha256:
        logger.warn('tarball %s has no hash to check' % url)

    try:
        access_common.unpackFromCache(sha256, directory)
    except KeyError as e:
        # figure out which registry we're fetching this tarball from (if any)
        # and add appropriate headers
        registry = Registry_Base_URL

        for source in _getSources():
            if ('type' in source and source['type'] == 'registry'
                    and 'url' in source and url.startswith(source['url'])):
                registry = source['url']
                break

        request_headers = _headersForRegistry(registry)

        logger.debug('GET %s, %s', url, request_headers)
        response = requests.get(url,
                                headers=request_headers,
                                allow_redirects=True,
                                stream=True)
        response.raise_for_status()

        access_common.unpackTarballStream(stream=response,
                                          into_directory=directory,
                                          hash={'sha256': sha256},
                                          cache_key=sha256,
                                          origin_info={'url': url})
Example #5
0
def _getTarball(url, into_directory):
    '''unpack the specified tarball url into the specified directory'''
    resource = Resource(url, pool=connection_pool.getPool(), follow_redirect=True)
    response = resource.get(
        headers = {'Authorization': 'token ' + settings.getProperty('github', 'authtoken')}, 
    )
    logger.debug('getting file: %s', url)
    # TODO: there's an MD5 in the response headers, verify it
    access_common.unpackTarballStream(response.body_stream(), into_directory)
Example #6
0
def _getTarball(url, into_directory):
    '''unpack the specified tarball url into the specified directory'''
    headers = {'Authorization': 'token ' + str(settings.getProperty('github', 'authtoken'))}

    response = requests.get(url, allow_redirects=True, stream=True, headers=headers)

    logger.debug('getting file: %s', url)
    # TODO: there's an MD5 in the response headers, verify it

    access_common.unpackTarballStream(response, into_directory)
Example #7
0
def _getTarball(url, into_directory):
    '''unpack the specified tarball url into the specified directory'''
    tok = settings.getProperty('github', 'authtoken')
    headers = {}
    if tok is not None:
        headers['Authorization'] = 'token ' + str(tok)

    logger.debug('GET %s', url)
    response = requests.get(url, allow_redirects=True, stream=True, headers=headers)
    response.raise_for_status()

    logger.debug('getting file: %s', url)
    # TODO: there's an MD5 in the response headers, verify it

    access_common.unpackTarballStream(response, into_directory)
Example #8
0
def _getTarball(url, directory, sha256):
    logger.debug('registry: get: %s' % url)

    if not sha256:
        logger.warn('tarball %s has no hash to check' % url)

    # figure out which registry we're fetching this tarball from (if any) and
    # add appropriate headers
    registry = Registry_Base_URL

    for source in _getSources():
        if ('type' in source and source['type'] == 'registry'
                and 'url' in source and url.startswith(source['url'])):
            registry = source['url']
            break

    request_headers = _headersForRegistry(registry)

    logger.debug('GET %s, %s', url, request_headers)
    response = requests.get(url,
                            headers=request_headers,
                            allow_redirects=True,
                            stream=True)
    response.raise_for_status()

    return access_common.unpackTarballStream(response, directory,
                                             ('sha256', sha256))
Example #9
0
def _getTarball(url, into_directory):
    '''unpack the specified tarball url into the specified directory'''
    headers = {
        'Authorization':
        'token ' + settings.getProperty('github', 'authtoken')
    }

    response = requests.get(url,
                            allow_redirects=True,
                            stream=True,
                            headers=headers)

    logger.debug('getting file: %s', url)
    # TODO: there's an MD5 in the response headers, verify it

    access_common.unpackTarballStream(response, into_directory)
Example #10
0
def _getTarball(url, into_directory):
    '''unpack the specified tarball url into the specified directory'''
    tok = settings.getProperty('github', 'authtoken')
    headers = {}
    if tok is not None:
        headers['Authorization'] = 'token ' + str(tok)

    logger.debug('GET %s', url)
    response = requests.get(url,
                            allow_redirects=True,
                            stream=True,
                            headers=headers)
    response.raise_for_status()

    logger.debug('getting file: %s', url)
    # TODO: there's an MD5 in the response headers, verify it

    access_common.unpackTarballStream(response, into_directory)
Example #11
0
def _getTarball(url, into_directory, cache_key):
    """unpack the specified tarball url into the specified directory"""

    try:
        access_common.unpackFromCache(cache_key, into_directory)
    except KeyError as e:
        tok = settings.getProperty("github", "authtoken")
        headers = {}
        if tok is not None:
            headers["Authorization"] = "token " + str(tok)

        logger.debug("GET %s", url)
        response = requests.get(url, allow_redirects=True, stream=True, headers=headers)
        response.raise_for_status()

        logger.debug("getting file: %s", url)
        logger.debug("headers: %s", response.headers)
        response.raise_for_status()

        # github doesn't exposes hashes of the archives being downloaded as far
        # as I can tell :(
        access_common.unpackTarballStream(stream=response, into_directory=into_directory, hash={}, cache_key=cache_key)
Example #12
0
def _getTarball(url, directory, sha256):
    auth = _registryAuthFilter()
    logger.debug('registry: get: %s' % url)
    if not sha256:
        logger.warn('tarball %s has no hash to check' % url)
    resource = Resource(url, pool=connection_pool.getPool(), filters=[auth])
    #resource = Resource('http://blobs.yottos.org/targets/stk3700-0.0.0.tar.gz', pool=connection_pool.getPool(), follow_redirect=True)
    response = resource.get()
    # there seems to be an issue with following the redirect using restkit:
    # follow redirect manually
    if response.status_int == 302 and 'Location' in response.headers:
        redirect_url = response.headers['Location']
        logger.debug('registry: redirect to: %s' % redirect_url)
        resource = Resource(redirect_url, pool=connection_pool.getPool())
        response = resource.get()
    return access_common.unpackTarballStream(response.body_stream(), directory, ('sha256', sha256))
Example #13
0
def _getTarball(url, directory, sha256):
    logger.debug('registry: get: %s' % url)

    if not sha256:
        logger.warn('tarball %s has no hash to check' % url)

    auth_token = generate_jwt_token(_getPrivateKeyObject())

    request_headers = {'Authorization': 'Bearer %s' % auth_token}

    response = requests.get(url,
                            headers=request_headers,
                            allow_redirects=True,
                            stream=True)
    response.raise_for_status()

    return access_common.unpackTarballStream(response, directory,
                                             ('sha256', sha256))
Example #14
0
def _getTarball(url, directory, sha256):
    logger.debug("registry: get: %s" % url)

    if not sha256:
        logger.warn("tarball %s has no hash to check" % url)

    # figure out which registry we're fetching this tarball from (if any) and
    # add appropriate headers
    registry = Registry_Base_URL

    for source in _getSources():
        if "type" in source and source["type"] == "registry" and "url" in source and url.startswith(source["url"]):
            registry = source["url"]
            break

    request_headers = _headersForRegistry(registry)

    logger.debug("GET %s, %s", url, request_headers)
    response = requests.get(url, headers=request_headers, allow_redirects=True, stream=True)
    response.raise_for_status()

    return access_common.unpackTarballStream(response, directory, ("sha256", sha256))