def urlgrab(self, url, filename=None, **kwargs):
"""urlgrab(url) copy the file to the local filesystem."""
request = self._request(url)
if filename is None:
filename = request.get_selector()
if filename.startswith('/'):
filename = filename[1:]
response = None
retries = self.retries
delay = self.delay
out = open(filename, 'w+')
while retries > 0:
try:
response = urllib2.urlopen(request)
buff = response.read(BUFFER_SIZE)
while buff:
out.write(buff)
buff = response.read(BUFFER_SIZE)
except urllib2.HTTPError, e:
if retries > 0:
time.sleep(delay)
delay *= self.backoff
else:
# Wrap exception as URLGrabError so that YumRepository catches it
from urlgrabber.grabber import URLGrabError
msg = '%s on %s tried' % (e, url)
if self.retries > 0:
msg += ' tried %d time(s)' % (self.retries)
new_e = URLGrabError(14, msg)
new_e.code = e.code
new_e.exception = e
new_e.url = url
raise new_e
finally:
def urlgrab(self, url, filename=None, **kwargs):
"""urlgrab(url) copy the file to the local filesystem."""
request = self._request(url)
if filename is None:
filename = request.get_selector()
if filename.startswith("/"):
filename = filename[1:]
response = None
try:
out = open(filename, "w+")
response = urllib2.urlopen(request)
buff = response.read(8192)
while buff:
out.write(buff)
buff = response.read(8192)
except urllib2.HTTPError, e:
# Wrap exception as URLGrabError so that YumRepository catches it
from urlgrabber.grabber import URLGrabError
new_e = URLGrabError(14, "%s on %s" % (e, url))
new_e.code = e.code
new_e.exception = e
new_e.url = url
raise new_e
def setUp(self):
self.snarfed_logs = []
self.db = urlgrabber.mirror.DEBUG
urlgrabber.mirror.DEBUG = FakeLogger()
self.mirrors = ['a', 'b', 'c', 'd', 'e', 'f']
self.g = FakeGrabber([URLGrabError(3), URLGrabError(3), 'filename'])
self.mg = MirrorGroup(self.g, self.mirrors)
def urlgrab(self, url, filename=None, **kwargs):
"""urlgrab(url) copy the file to the local filesystem."""
request = self._request(url)
if filename is None:
filename = request.get_selector()
if filename.startswith('/'):
filename = filename[1:]
response = None
retries = self.retries
delay = self.delay
out = open(filename, 'w+')
while retries > 0:
try:
response = urllib2.urlopen(request)
buff = response.read(BUFFER_SIZE)
while buff:
out.write(buff)
buff = response.read(BUFFER_SIZE)
except urllib2.HTTPError, e:
if retries > 0:
time.sleep(delay)
delay *= self.backoff
else:
# Wrap exception as URLGrabError so that YumRepository catches it
from urlgrabber.grabber import URLGrabError
msg = '%s on %s tried' % (e, url)
if self.retries > 0:
msg += ' tried %d time(s)' % (self.retries)
new_e = URLGrabError(14, msg)
new_e.code = e.code
new_e.exception = e
new_e.url = url
raise new_e
finally:
def _get_mirror(self, gr):
# OVERRIDE IDEAS:
# return a random mirror so that multiple mirrors get used
# even without failures.
if not gr.mirrors:
e = URLGrabError(256, _('No more mirrors to try.'))
e.errors = gr.errors
raise e
return gr.mirrors[gr._next]
def _get_mirror(self, gr):
# OVERRIDE IDEAS:
# return a random mirror so that multiple mirrors get used
# even without failures.
if not gr.mirrors:
e = URLGrabError(256, _('No more mirrors to try.'))
e.errors = gr.errors
raise e
return gr.mirrors[gr._next]
def checkfunc_grab(obj):
with open('foo') as f:
data = f.read()
if data == 'version1':
raise URLGrabError(-1, 'Outdated version of foo')
elif data != 'version2':
self.fail('Unexpected file content')
def setup_credentials(self):
"""
Attempts to setup credentials in the following strict priority order, from explicit to implicit,
1. Access key, secret and token set in repository configuration
2. Environment variables
3. IAM role(3rd party cross-account), if set in the repo configuration(delegated_role=<rolearn>)
4. Instance profile IAM role
"""
repo = self.repo
if repo.key_id and repo.secret_key and repo.token:
self.set_credentials(repo.key_id, repo.secret_key, repo.token)
elif repo.key_id and repo.secret_key:
self.set_credentials(repo.key_id, repo.secret_key)
else:
self.setup_environment_credentials()
if not self.credentials_set:
if repo.delegated_role:
self.setup_delegated_role_credentials(repo.delegated_role)
else:
self.setup_instance_profile_role_credentials()
if not self.credentials_set:
if hasattr(repo, 'name'):
msg = "Could not access AWS credentials, skipping repository '%s'" % (
repo.name)
else:
msg = "Could not access AWS credentials"
print msg
from urlgrabber.grabber import URLGrabError
raise URLGrabError(7, msg)
def urlgrab(self, url, filename=None, **kwargs):
print("urlgrab starting %s" % (url))
"""urlgrab(url) copy the file to the local filesystem."""
request = self._request(url)
self.pretty_print_POST(request)
if filename is None:
filename = request.get_selector()
if filename.startswith('/'):
filename = filename[1:]
response = None
retries = self.retries
delay = self.delay
out = open(filename, 'w+')
while retries > 0:
try:
print("request to open %s:" % (request.get_full_url()))
print("request header %s:" % (request.header_items()))
response = urllib2.urlopen(request)
buff = response.read(BUFFER_SIZE)
while buff:
out.write(buff)
buff = response.read(BUFFER_SIZE)
except urllib2.HTTPError, e:
print("error opening url %s %s %s" % (e, e.code, e.reason))
print("error headers %s" % (e.headers))
if retries > 0:
print("retry...")
time.sleep(delay)
delay *= self.backoff
else:
print("no more retries. just give up")
# Wrap exception as URLGrabError so that YumRepository catches it
from urlgrabber.grabber import URLGrabError
msg = '%s on %s tried' % (e, url)
if self.retries > 0:
msg += ' tried %d time(s)' % (self.retries)
new_e = URLGrabError(14, msg)
new_e.code = e.code
new_e.exception = e
new_e.url = url
raise new_e
finally:
def urlgrab(self, url, filename, text = None, **kwargs):
if url.startswith('file://'):
file = url.replace('file://', '', 1)
if os.path.isfile(file):
return file
else:
raise URLGrabError(2, 'Local file \'%s\' does not exist' % file)
f = open(filename, 'wb')
try:
try:
for i in streamfile(url, progress_obj=self.progress_obj, text=text):
f.write(i)
except urllib2.HTTPError, e:
exc = URLGrabError(14, str(e))
exc.url = url
exc.exception = e
exc.code = e.code
raise exc
except IOError, e:
raise URLGrabError(4, str(e))
def urlgrab(self, url, filename, text=None, **kwargs):
if url.startswith("file://"):
f = url.replace("file://", "", 1)
if os.path.isfile(f):
return f
else:
raise URLGrabError(2, "Local file '%s' does not exist" % f)
with file(filename, "wb") as f:
try:
for i in streamfile(url, progress_obj=self.progress_obj, text=text):
f.write(i)
except HTTPError as e:
exc = URLGrabError(14, str(e))
exc.url = url
exc.exception = e
exc.code = e.code
raise exc
except IOError as e:
raise URLGrabError(4, str(e))
return filename
def checkheader(headerfile, name, arch):
"""check a header by opening it and comparing the results to the name and arch
we believe it to be for. if it fails raise URLGrabError(-1)"""
h = Header_Work(headerfile)
fail = 0
if h.hdr is None:
fail = 1
else:
if name != h.name() or arch != h.arch():
fail = 1
if fail:
raise URLGrabError(-1, _('Header cannot be opened or does not match %s, %s.') % (name, arch))
return
def _checkfunc(self, obj, *args, **kwargs):
self.obj = obj
self.args = args
self.kwargs = kwargs
if hasattr(obj, 'filename'):
# we used urlgrab
data = open(obj.filename, 'rb').read()
else:
# we used urlread
data = obj.data
if data == self.data: return
else: raise URLGrabError(-2, "data doesn't match")
def urlgrab(self, url, filename=None, **kwargs):
"""urlgrab(url) copy the file to the local filesystem."""
request = self._request(url)
if filename is None:
filename = request.get_selector()
if filename.startswith('/'):
filename = filename[1:]
response = None
try:
out = open(filename, 'w+')
response = urllib2.urlopen(request)
buff = response.read(8192)
while buff:
out.write(buff)
buff = response.read(8192)
except urllib2.HTTPError, e:
# Wrap exception as URLGrabError so that YumRepository catches it
from urlgrabber.grabber import URLGrabError
new_e = URLGrabError(14, '%s on %s' % (e, url))
new_e.code = e.code
new_e.exception = e
new_e.url = url
raise new_e
def patches_checksum_func(self, callback_obj, checksum_type, checksum):
"""Simple function to checksum patches for urlgrabber
"""
import types
# this is ugly code copy&pasted from yumRepo.YumRepository._checkMD
if type(callback_obj) == types.InstanceType: # urlgrabber check
filename = callback_obj.filename
else:
filename = callback_obj
local_checksum = self.repo._checksum(checksum_type, filename)
if local_checksum == checksum:
return 1
else:
raise URLGrabError(-1, 'Metadata file does not match checksum')
def get_credentials(self):
print("get_credentials")
"""Read IAM credentials from AWS metadata store.
Note: This method should be explicitly called after constructing new
object, as in 'explicit is better than implicit'.
"""
request = urllib2.Request(
urlparse.urljoin(
urlparse.urljoin(
"http://169.254.169.254/",
"latest/meta-data/iam/security-credentials/",
), self.iamrole))
try:
print("Opening url... %s" % (request))
response = urllib2.urlopen(request)
data = json.loads(response.read())
print("response urllib2: %s" % (data))
self.access_key = data['AccessKeyId']
self.secret_key = data['SecretAccessKey']
self.token = data['Token']
except Exception:
print("error in library")
response = None
finally:
if response:
response.close()
if self.access_key is None and self.secret_key is None:
if "AWS_ACCESS_KEY_ID" in os.environ:
self.access_key = os.environ['AWS_ACCESS_KEY_ID']
if "AWS_SECRET_ACCESS_KEY" in os.environ:
self.secret_key = os.environ['AWS_SECRET_ACCESS_KEY']
if "AWS_SESSION_TOKEN" in os.environ:
self.token = os.environ['AWS_SESSION_TOKEN']
if self.access_key is None and self.secret_key is None:
if hasattr(self, 'name'):
msg = "Could not access AWS credentials, skipping repository '%s'" % (
self.name)
else:
msg = "Could not access AWS credentials"
print msg
from urlgrabber.grabber import URLGrabError
raise URLGrabError(7, msg)
def get_credentials(self):
"""Read IAM credentials from AWS metadata store.
Note: This method should be explicitly called after constructing new
object, as in 'explicit is better than implicit'.
"""
credentials = self.session.get_credentials()
self.access_key = credentials.access_key
self.secret_key = credentials.secret_key
self.token = credentials.token
if self.access_key is None and self.secret_key is None:
if hasattr(self, 'name'):
msg = "Could not access AWS credentials, skipping repository '%s'" % (self.name)
else:
msg = "Could not access AWS credentials"
print msg
from urlgrabber.grabber import URLGrabError
raise URLGrabError(7, msg)
def checkRpmMD5(package, urlgraberror=0):
"""take a package, check it out by trying to open it, return 1 if it's good
return 0 if it's not"""
ts.sigChecking('md5')
fdno = os.open(package, os.O_RDONLY)
try:
ts.hdrFromFdno(fdno)
except rpm.error as e:
good = 0
else:
good = 1
os.close(fdno)
ts.sigChecking('default')
if urlgraberror:
if not good:
raise URLGrabError(-1, _('RPM %s fails md5 check') % (package))
else:
return
else:
return good
def urlgrab(self, url, filename, text = None, **kwargs):
if url.startswith('file://'):
file = url.replace('file://', '', 1)
if os.path.isfile(file):
return file
else:
raise URLGrabError(2, 'Local file \'%s\' does not exist' % file)
f = open(filename, 'wb')
try:
try:
for i in streamfile(url, progress_obj=self.progress_obj, text=text):
f.write(i)
except urllib2.HTTPError, e:
exc = URLGrabError(14, str(e))
exc.url = url
exc.exception = e
exc.code = e.code
raise exc
except IOError, e:
raise URLGrabError(4, str(e))
def checkfunc_read(obj):
if obj.data == 'version1':
raise URLGrabError(-1, 'Outdated version of foo')
def checkfunc_grab(obj):
with open('foo') as f:
if f.read() == 'version1':
raise URLGrabError(-1, 'Outdated version of foo')
def checkfunc_read(obj):
if obj.data == b'version1':
raise URLGrabError(-1, 'Outdated version of foo')
elif obj.data != b'version2':
self.fail('Unexpected file content')
def run(self, buildinfo):
cached = 0
all = len(buildinfo.deps)
for i in buildinfo.deps:
i.makeurls(self.cachedir, self.urllist)
if os.path.exists(i.fullfilename):
cached += 1
miss = 0
needed = all - cached
if all:
miss = 100.0 * needed / all
print "%.1f%% cache miss. %d/%d dependencies cached.\n" % (miss,
cached, all)
done = 1
for i in buildinfo.deps:
i.makeurls(self.cachedir, self.urllist)
if not os.path.exists(i.fullfilename):
if self.offline:
raise oscerr.OscIOError(
None,
'Missing package \'%s\' in cache: --offline not possible.'
% i.fullfilename)
self.dirSetup(i)
try:
# if there isn't a progress bar, there is no output at all
if not self.progress_obj:
print '%d/%d (%s) %s' % (done, needed, i.project,
i.filename)
self.fetch(i)
if self.progress_obj:
print " %d/%d\r" % (done, needed),
sys.stdout.flush()
except KeyboardInterrupt:
print 'Cancelled by user (ctrl-c)'
print 'Exiting.'
sys.exit(0)
done += 1
self.__fetch_cpio(buildinfo.apiurl)
prjs = buildinfo.projects.keys()
for i in prjs:
dest = "%s/%s" % (self.cachedir, i)
if not os.path.exists(dest):
os.makedirs(dest, mode=0755)
dest += '/_pubkey'
url = makeurl(buildinfo.apiurl, ['source', i, '_pubkey'])
try:
if self.offline and not os.path.exists(dest):
# may need to try parent
raise URLGrabError(2)
elif not self.offline:
OscFileGrabber().urlgrab(url, dest)
if not i in buildinfo.prjkeys: # not that many keys usually
buildinfo.keys.append(dest)
buildinfo.prjkeys.append(i)
except KeyboardInterrupt:
print 'Cancelled by user (ctrl-c)'
print 'Exiting.'
if os.path.exists(dest):
os.unlink(dest)
sys.exit(0)
except URLGrabError, e:
if self.http_debug:
print >> sys.stderr, "can't fetch key for %s: %s" % (
i, e.strerror)
print >> sys.stderr, "url: %s" % url
if os.path.exists(dest):
os.unlink(dest)
l = i.rsplit(':', 1)
# try key from parent project
if len(l) > 1 and l[1] and not l[0] in buildinfo.projects:
prjs.append(l[0])
def run(self, buildinfo):
cached = 0
all = len(buildinfo.deps)
for i in buildinfo.deps:
i.makeurls(self.cachedir, self.urllist)
if os.path.exists(i.fullfilename):
cached += 1
if i.hdrmd5:
from .util import packagequery
hdrmd5 = packagequery.PackageQuery.queryhdrmd5(i.fullfilename)
if not hdrmd5 or hdrmd5 != i.hdrmd5:
os.unlink(i.fullfilename)
cached -= 1
miss = 0
needed = all - cached
if all:
miss = 100.0 * needed / all
print("%.1f%% cache miss. %d/%d dependencies cached.\n" % (miss, cached, all))
done = 1
for i in buildinfo.deps:
i.makeurls(self.cachedir, self.urllist)
if not os.path.exists(i.fullfilename):
if self.offline:
raise oscerr.OscIOError(None,
'Missing \'%s\' in cache: '
'--offline not possible.' %
i.fullfilename)
self.dirSetup(i)
if i.hdrmd5 and self.enable_cpio:
self.__add_cpio(i)
done += 1
continue
try:
# if there isn't a progress bar, there is no output at all
if not self.progress_obj:
print('%d/%d (%s) %s' % (done, needed, i.project, i.filename))
self.fetch(i)
if self.progress_obj:
print(" %d/%d\r" % (done, needed), end=' ')
sys.stdout.flush()
except KeyboardInterrupt:
print('Cancelled by user (ctrl-c)')
print('Exiting.')
sys.exit(0)
done += 1
self.__fetch_cpio(buildinfo.apiurl)
prjs = list(buildinfo.projects.keys())
for i in prjs:
dest = "%s/%s" % (self.cachedir, i)
if not os.path.exists(dest):
os.makedirs(dest, mode=0o755)
dest += '/_pubkey'
url = makeurl(buildinfo.apiurl, ['source', i, '_pubkey'])
try:
if self.offline and not os.path.exists(dest):
# may need to try parent
raise URLGrabError(2)
elif not self.offline:
OscFileGrabber().urlgrab(url, dest)
# not that many keys usually
if i not in buildinfo.prjkeys:
buildinfo.keys.append(dest)
buildinfo.prjkeys.append(i)
except KeyboardInterrupt:
print('Cancelled by user (ctrl-c)')
print('Exiting.')
if os.path.exists(dest):
os.unlink(dest)
sys.exit(0)
except URLGrabError as e:
# Not found is okay, let's go to the next project
if e.errno == 14 and e.code != 404:
print("Invalid answer from server", e, file=sys.stderr)
sys.exit(1)
if self.http_debug:
print("can't fetch key for %s: %s" % (i, e.strerror), file=sys.stderr)
print("url: %s" % url, file=sys.stderr)
if os.path.exists(dest):
os.unlink(dest)
l = i.rsplit(':', 1)
# try key from parent project
if len(l) > 1 and l[1] and not l[0] in buildinfo.projects:
prjs.append(l[0])
"""take a package, check it out by trying to open it, return 1 if its good
return 0 if it's not"""
ts.sigChecking('md5')
fdno = os.open(package, os.O_RDONLY)
try:
ts.hdrFromFdno(fdno)
except rpm.error, e:
good = 0
else:
good = 1
os.close(fdno)
ts.sigChecking('default')
if urlgraberror:
if not good:
raise URLGrabError(-1, _('RPM %s fails md5 check') % (package))
else:
return
else:
return good
def checkSig(package):
""" take a package, check it's sigs, return 0 if they are all fine, return
1 if the gpg key can't be found, 2 if the header is in someway damaged,
3 if the key is not trusted, 4 if the pkg is not gpg or pgp signed"""
ts.sigChecking('default')
fdno = os.open(package, os.O_RDONLY)
try:
hdr = ts.hdrFromFdno(fdno)
except rpm.error, e:
if str(e) == "public key not availaiable":
