Beispiel #1
0
def _getTarball(url, into_directory, cache_key, origin_info=None):
    '''unpack the specified tarball url into the specified directory'''

    try:
        access_common.unpackFromCache(cache_key, into_directory)
    except KeyError as e:
        tok = settings.getProperty('github', 'authtoken')
        headers = {}
        if tok is not None:
            headers['Authorization'] = 'token ' + str(tok)

        logger.debug('GET %s', url)
        response = requests.get(url, allow_redirects=True, stream=True, headers=headers)
        response.raise_for_status()

        logger.debug('getting file: %s', url)
        logger.debug('headers: %s', response.headers)
        response.raise_for_status()

        # github doesn't exposes hashes of the archives being downloaded as far
        # as I can tell :(
        access_common.unpackTarballStream(
                    stream = response,
            into_directory = into_directory,
                      hash = {},
                 cache_key = cache_key,
               origin_info = origin_info
        )
Beispiel #2
0
def _getTarball(url, directory, sha256):
    logger.debug('registry: get: %s' % url)

    if not sha256:
        logger.warn('tarball %s has no hash to check' % url)

    try:
        access_common.unpackFromCache(sha256, directory)
    except KeyError as e:
        # figure out which registry we're fetching this tarball from (if any)
        # and add appropriate headers
        registry = Registry_Base_URL

        for source in _getSources():
            if ('type' in source and source['type'] == 'registry'
                    and 'url' in source and url.startswith(source['url'])):
                registry = source['url']
                break

        request_headers = _headersForRegistry(registry)

        logger.debug('GET %s, %s', url, request_headers)
        response = requests.get(url,
                                headers=request_headers,
                                allow_redirects=True,
                                stream=True)
        response.raise_for_status()

        access_common.unpackTarballStream(stream=response,
                                          into_directory=directory,
                                          hash={'sha256': sha256},
                                          cache_key=sha256,
                                          origin_info={'url': url})
def _getTarball(url, directory, sha256):
    logger.debug('registry: get: %s' % url)

    if not sha256:
        logger.warn('tarball %s has no hash to check' % url)

    try:
        access_common.unpackFromCache(sha256, directory)
    except KeyError as e:
        # figure out which registry we're fetching this tarball from (if any)
        # and add appropriate headers
        registry = Registry_Base_URL

        for source in _getSources():
            if ('type' in source and source['type'] == 'registry' and
                 'url' in source and url.startswith(source['url'])):
                registry = source['url']
                break

        request_headers = _headersForRegistry(registry)

        logger.debug('GET %s, %s', url, request_headers)
        response = requests.get(url, headers=request_headers, allow_redirects=True, stream=True)
        response.raise_for_status()

        access_common.unpackTarballStream(
                    stream = response,
            into_directory = directory,
                      hash = {'sha256':sha256},
                 cache_key = sha256
        )
Beispiel #4
0
def _getTarball(url, into_directory, cache_key):
    '''unpack the specified tarball url into the specified directory'''

    try:
        access_common.unpackFromCache(cache_key, into_directory)
    except KeyError as e:
        tok = settings.getProperty('github', 'authtoken')
        headers = {}
        if tok is not None:
            headers['Authorization'] = 'token ' + str(tok)

        logger.debug('GET %s', url)
        response = requests.get(url,
                                allow_redirects=True,
                                stream=True,
                                headers=headers)
        response.raise_for_status()

        logger.debug('getting file: %s', url)
        logger.debug('headers: %s', response.headers)
        response.raise_for_status()

        # github doesn't exposes hashes of the archives being downloaded as far
        # as I can tell :(
        access_common.unpackTarballStream(stream=response,
                                          into_directory=into_directory,
                                          hash={},
                                          cache_key=cache_key)
Beispiel #5
0
def _getTarball(url, into_directory, cache_key):
    """unpack the specified tarball url into the specified directory"""

    try:
        access_common.unpackFromCache(cache_key, into_directory)
    except KeyError as e:
        tok = settings.getProperty("github", "authtoken")
        headers = {}
        if tok is not None:
            headers["Authorization"] = "token " + str(tok)

        logger.debug("GET %s", url)
        response = requests.get(url, allow_redirects=True, stream=True, headers=headers)
        response.raise_for_status()

        logger.debug("getting file: %s", url)
        logger.debug("headers: %s", response.headers)
        response.raise_for_status()

        # github doesn't exposes hashes of the archives being downloaded as far
        # as I can tell :(
        access_common.unpackTarballStream(stream=response, into_directory=into_directory, hash={}, cache_key=cache_key)