def _getTarball(url, into_directory, cache_key, origin_info=None):
'''unpack the specified tarball url into the specified directory'''
try:
access_common.unpackFromCache(cache_key, into_directory)
except KeyError as e:
tok = settings.getProperty('github', 'authtoken')
headers = {}
if tok is not None:
headers['Authorization'] = 'token ' + str(tok)
logger.debug('GET %s', url)
response = requests.get(url, allow_redirects=True, stream=True, headers=headers)
response.raise_for_status()
logger.debug('getting file: %s', url)
logger.debug('headers: %s', response.headers)
response.raise_for_status()
# github doesn't exposes hashes of the archives being downloaded as far
# as I can tell :(
access_common.unpackTarballStream(
stream = response,
into_directory = into_directory,
hash = {},
cache_key = cache_key,
origin_info = origin_info
)
def _getTarball(url, directory, sha256):
logger.debug('registry: get: %s' % url)
if not sha256:
logger.warn('tarball %s has no hash to check' % url)
try:
access_common.unpackFromCache(sha256, directory)
except KeyError as e:
# figure out which registry we're fetching this tarball from (if any)
# and add appropriate headers
registry = Registry_Base_URL
for source in _getSources():
if ('type' in source and source['type'] == 'registry' and
'url' in source and url.startswith(source['url'])):
registry = source['url']
break
request_headers = _headersForRegistry(registry)
logger.debug('GET %s, %s', url, request_headers)
response = requests.get(url, headers=request_headers, allow_redirects=True, stream=True)
response.raise_for_status()
access_common.unpackTarballStream(
stream = response,
into_directory = directory,
hash = {'sha256':sha256},
cache_key = sha256
)
def _getTarball(url, into_directory, cache_key):
'''unpack the specified tarball url into the specified directory'''
try:
access_common.unpackFromCache(cache_key, into_directory)
except KeyError as e:
tok = settings.getProperty('github', 'authtoken')
headers = {}
if tok is not None:
headers['Authorization'] = 'token ' + str(tok)
logger.debug('GET %s', url)
response = requests.get(url,
allow_redirects=True,
stream=True,
headers=headers)
response.raise_for_status()
logger.debug('getting file: %s', url)
logger.debug('headers: %s', response.headers)
response.raise_for_status()
# github doesn't exposes hashes of the archives being downloaded as far
# as I can tell :(
access_common.unpackTarballStream(stream=response,
into_directory=into_directory,
hash={},
cache_key=cache_key)
def _getTarball(url, directory, sha256):
logger.debug('registry: get: %s' % url)
if not sha256:
logger.warn('tarball %s has no hash to check' % url)
try:
access_common.unpackFromCache(sha256, directory)
except KeyError as e:
# figure out which registry we're fetching this tarball from (if any)
# and add appropriate headers
registry = Registry_Base_URL
for source in _getSources():
if ('type' in source and source['type'] == 'registry'
and 'url' in source and url.startswith(source['url'])):
registry = source['url']
break
request_headers = _headersForRegistry(registry)
logger.debug('GET %s, %s', url, request_headers)
response = requests.get(url,
headers=request_headers,
allow_redirects=True,
stream=True)
response.raise_for_status()
access_common.unpackTarballStream(stream=response,
into_directory=directory,
hash={'sha256': sha256},
cache_key=sha256,
origin_info={'url': url})
def _getTarball(url, into_directory):
'''unpack the specified tarball url into the specified directory'''
resource = Resource(url, pool=connection_pool.getPool(), follow_redirect=True)
response = resource.get(
headers = {'Authorization': 'token ' + settings.getProperty('github', 'authtoken')},
)
logger.debug('getting file: %s', url)
# TODO: there's an MD5 in the response headers, verify it
access_common.unpackTarballStream(response.body_stream(), into_directory)
def _getTarball(url, into_directory):
'''unpack the specified tarball url into the specified directory'''
headers = {'Authorization': 'token ' + str(settings.getProperty('github', 'authtoken'))}
response = requests.get(url, allow_redirects=True, stream=True, headers=headers)
logger.debug('getting file: %s', url)
# TODO: there's an MD5 in the response headers, verify it
access_common.unpackTarballStream(response, into_directory)
def _getTarball(url, into_directory):
'''unpack the specified tarball url into the specified directory'''
tok = settings.getProperty('github', 'authtoken')
headers = {}
if tok is not None:
headers['Authorization'] = 'token ' + str(tok)
logger.debug('GET %s', url)
response = requests.get(url, allow_redirects=True, stream=True, headers=headers)
response.raise_for_status()
logger.debug('getting file: %s', url)
# TODO: there's an MD5 in the response headers, verify it
access_common.unpackTarballStream(response, into_directory)
def _getTarball(url, directory, sha256):
logger.debug('registry: get: %s' % url)
if not sha256:
logger.warn('tarball %s has no hash to check' % url)
# figure out which registry we're fetching this tarball from (if any) and
# add appropriate headers
registry = Registry_Base_URL
for source in _getSources():
if ('type' in source and source['type'] == 'registry'
and 'url' in source and url.startswith(source['url'])):
registry = source['url']
break
request_headers = _headersForRegistry(registry)
logger.debug('GET %s, %s', url, request_headers)
response = requests.get(url,
headers=request_headers,
allow_redirects=True,
stream=True)
response.raise_for_status()
return access_common.unpackTarballStream(response, directory,
('sha256', sha256))
def _getTarball(url, into_directory):
'''unpack the specified tarball url into the specified directory'''
headers = {
'Authorization':
'token ' + settings.getProperty('github', 'authtoken')
}
response = requests.get(url,
allow_redirects=True,
stream=True,
headers=headers)
logger.debug('getting file: %s', url)
# TODO: there's an MD5 in the response headers, verify it
access_common.unpackTarballStream(response, into_directory)
def _getTarball(url, into_directory):
'''unpack the specified tarball url into the specified directory'''
tok = settings.getProperty('github', 'authtoken')
headers = {}
if tok is not None:
headers['Authorization'] = 'token ' + str(tok)
logger.debug('GET %s', url)
response = requests.get(url,
allow_redirects=True,
stream=True,
headers=headers)
response.raise_for_status()
logger.debug('getting file: %s', url)
# TODO: there's an MD5 in the response headers, verify it
access_common.unpackTarballStream(response, into_directory)
def _getTarball(url, into_directory, cache_key):
"""unpack the specified tarball url into the specified directory"""
try:
access_common.unpackFromCache(cache_key, into_directory)
except KeyError as e:
tok = settings.getProperty("github", "authtoken")
headers = {}
if tok is not None:
headers["Authorization"] = "token " + str(tok)
logger.debug("GET %s", url)
response = requests.get(url, allow_redirects=True, stream=True, headers=headers)
response.raise_for_status()
logger.debug("getting file: %s", url)
logger.debug("headers: %s", response.headers)
response.raise_for_status()
# github doesn't exposes hashes of the archives being downloaded as far
# as I can tell :(
access_common.unpackTarballStream(stream=response, into_directory=into_directory, hash={}, cache_key=cache_key)
def _getTarball(url, directory, sha256):
auth = _registryAuthFilter()
logger.debug('registry: get: %s' % url)
if not sha256:
logger.warn('tarball %s has no hash to check' % url)
resource = Resource(url, pool=connection_pool.getPool(), filters=[auth])
#resource = Resource('http://blobs.yottos.org/targets/stk3700-0.0.0.tar.gz', pool=connection_pool.getPool(), follow_redirect=True)
response = resource.get()
# there seems to be an issue with following the redirect using restkit:
# follow redirect manually
if response.status_int == 302 and 'Location' in response.headers:
redirect_url = response.headers['Location']
logger.debug('registry: redirect to: %s' % redirect_url)
resource = Resource(redirect_url, pool=connection_pool.getPool())
response = resource.get()
return access_common.unpackTarballStream(response.body_stream(), directory, ('sha256', sha256))
def _getTarball(url, directory, sha256):
logger.debug('registry: get: %s' % url)
if not sha256:
logger.warn('tarball %s has no hash to check' % url)
auth_token = generate_jwt_token(_getPrivateKeyObject())
request_headers = {'Authorization': 'Bearer %s' % auth_token}
response = requests.get(url,
headers=request_headers,
allow_redirects=True,
stream=True)
response.raise_for_status()
return access_common.unpackTarballStream(response, directory,
('sha256', sha256))
def _getTarball(url, directory, sha256):
logger.debug("registry: get: %s" % url)
if not sha256:
logger.warn("tarball %s has no hash to check" % url)
# figure out which registry we're fetching this tarball from (if any) and
# add appropriate headers
registry = Registry_Base_URL
for source in _getSources():
if "type" in source and source["type"] == "registry" and "url" in source and url.startswith(source["url"]):
registry = source["url"]
break
request_headers = _headersForRegistry(registry)
logger.debug("GET %s, %s", url, request_headers)
response = requests.get(url, headers=request_headers, allow_redirects=True, stream=True)
response.raise_for_status()
return access_common.unpackTarballStream(response, directory, ("sha256", sha256))