def urlgrab(self, url, filename=None, **kwargs): """urlgrab(url) copy the file to the local filesystem.""" request = self._request(url) if filename is None: filename = request.get_selector() if filename.startswith('/'): filename = filename[1:] response = None retries = self.retries delay = self.delay out = open(filename, 'w+') while retries > 0: try: response = urllib2.urlopen(request) buff = response.read(BUFFER_SIZE) while buff: out.write(buff) buff = response.read(BUFFER_SIZE) except urllib2.HTTPError, e: if retries > 0: time.sleep(delay) delay *= self.backoff else: # Wrap exception as URLGrabError so that YumRepository catches it from urlgrabber.grabber import URLGrabError msg = '%s on %s tried' % (e, url) if self.retries > 0: msg += ' tried %d time(s)' % (self.retries) new_e = URLGrabError(14, msg) new_e.code = e.code new_e.exception = e new_e.url = url raise new_e finally:
def urlgrab(self, url, filename=None, **kwargs): """urlgrab(url) copy the file to the local filesystem.""" request = self._request(url) if filename is None: filename = request.get_selector() if filename.startswith("/"): filename = filename[1:] response = None try: out = open(filename, "w+") response = urllib2.urlopen(request) buff = response.read(8192) while buff: out.write(buff) buff = response.read(8192) except urllib2.HTTPError, e: # Wrap exception as URLGrabError so that YumRepository catches it from urlgrabber.grabber import URLGrabError new_e = URLGrabError(14, "%s on %s" % (e, url)) new_e.code = e.code new_e.exception = e new_e.url = url raise new_e
def setUp(self): self.snarfed_logs = [] self.db = urlgrabber.mirror.DEBUG urlgrabber.mirror.DEBUG = FakeLogger() self.mirrors = ['a', 'b', 'c', 'd', 'e', 'f'] self.g = FakeGrabber([URLGrabError(3), URLGrabError(3), 'filename']) self.mg = MirrorGroup(self.g, self.mirrors)
def _get_mirror(self, gr): # OVERRIDE IDEAS: # return a random mirror so that multiple mirrors get used # even without failures. if not gr.mirrors: e = URLGrabError(256, _('No more mirrors to try.')) e.errors = gr.errors raise e return gr.mirrors[gr._next]
def checkfunc_grab(obj): with open('foo') as f: data = f.read() if data == 'version1': raise URLGrabError(-1, 'Outdated version of foo') elif data != 'version2': self.fail('Unexpected file content')
def setup_credentials(self): """ Attempts to setup credentials in the following strict priority order, from explicit to implicit, 1. Access key, secret and token set in repository configuration 2. Environment variables 3. IAM role(3rd party cross-account), if set in the repo configuration(delegated_role=<rolearn>) 4. Instance profile IAM role """ repo = self.repo if repo.key_id and repo.secret_key and repo.token: self.set_credentials(repo.key_id, repo.secret_key, repo.token) elif repo.key_id and repo.secret_key: self.set_credentials(repo.key_id, repo.secret_key) else: self.setup_environment_credentials() if not self.credentials_set: if repo.delegated_role: self.setup_delegated_role_credentials(repo.delegated_role) else: self.setup_instance_profile_role_credentials() if not self.credentials_set: if hasattr(repo, 'name'): msg = "Could not access AWS credentials, skipping repository '%s'" % ( repo.name) else: msg = "Could not access AWS credentials" print msg from urlgrabber.grabber import URLGrabError raise URLGrabError(7, msg)
def urlgrab(self, url, filename=None, **kwargs): print("urlgrab starting %s" % (url)) """urlgrab(url) copy the file to the local filesystem.""" request = self._request(url) self.pretty_print_POST(request) if filename is None: filename = request.get_selector() if filename.startswith('/'): filename = filename[1:] response = None retries = self.retries delay = self.delay out = open(filename, 'w+') while retries > 0: try: print("request to open %s:" % (request.get_full_url())) print("request header %s:" % (request.header_items())) response = urllib2.urlopen(request) buff = response.read(BUFFER_SIZE) while buff: out.write(buff) buff = response.read(BUFFER_SIZE) except urllib2.HTTPError, e: print("error opening url %s %s %s" % (e, e.code, e.reason)) print("error headers %s" % (e.headers)) if retries > 0: print("retry...") time.sleep(delay) delay *= self.backoff else: print("no more retries. just give up") # Wrap exception as URLGrabError so that YumRepository catches it from urlgrabber.grabber import URLGrabError msg = '%s on %s tried' % (e, url) if self.retries > 0: msg += ' tried %d time(s)' % (self.retries) new_e = URLGrabError(14, msg) new_e.code = e.code new_e.exception = e new_e.url = url raise new_e finally:
def urlgrab(self, url, filename, text = None, **kwargs): if url.startswith('file://'): file = url.replace('file://', '', 1) if os.path.isfile(file): return file else: raise URLGrabError(2, 'Local file \'%s\' does not exist' % file) f = open(filename, 'wb') try: try: for i in streamfile(url, progress_obj=self.progress_obj, text=text): f.write(i) except urllib2.HTTPError, e: exc = URLGrabError(14, str(e)) exc.url = url exc.exception = e exc.code = e.code raise exc except IOError, e: raise URLGrabError(4, str(e))
def urlgrab(self, url, filename, text=None, **kwargs): if url.startswith("file://"): f = url.replace("file://", "", 1) if os.path.isfile(f): return f else: raise URLGrabError(2, "Local file '%s' does not exist" % f) with file(filename, "wb") as f: try: for i in streamfile(url, progress_obj=self.progress_obj, text=text): f.write(i) except HTTPError as e: exc = URLGrabError(14, str(e)) exc.url = url exc.exception = e exc.code = e.code raise exc except IOError as e: raise URLGrabError(4, str(e)) return filename
def checkheader(headerfile, name, arch): """check a header by opening it and comparing the results to the name and arch we believe it to be for. if it fails raise URLGrabError(-1)""" h = Header_Work(headerfile) fail = 0 if h.hdr is None: fail = 1 else: if name != h.name() or arch != h.arch(): fail = 1 if fail: raise URLGrabError(-1, _('Header cannot be opened or does not match %s, %s.') % (name, arch)) return
def _checkfunc(self, obj, *args, **kwargs): self.obj = obj self.args = args self.kwargs = kwargs if hasattr(obj, 'filename'): # we used urlgrab data = open(obj.filename, 'rb').read() else: # we used urlread data = obj.data if data == self.data: return else: raise URLGrabError(-2, "data doesn't match")
def urlgrab(self, url, filename=None, **kwargs): """urlgrab(url) copy the file to the local filesystem.""" request = self._request(url) if filename is None: filename = request.get_selector() if filename.startswith('/'): filename = filename[1:] response = None try: out = open(filename, 'w+') response = urllib2.urlopen(request) buff = response.read(8192) while buff: out.write(buff) buff = response.read(8192) except urllib2.HTTPError, e: # Wrap exception as URLGrabError so that YumRepository catches it from urlgrabber.grabber import URLGrabError new_e = URLGrabError(14, '%s on %s' % (e, url)) new_e.code = e.code new_e.exception = e new_e.url = url raise new_e
def patches_checksum_func(self, callback_obj, checksum_type, checksum): """Simple function to checksum patches for urlgrabber """ import types # this is ugly code copy&pasted from yumRepo.YumRepository._checkMD if type(callback_obj) == types.InstanceType: # urlgrabber check filename = callback_obj.filename else: filename = callback_obj local_checksum = self.repo._checksum(checksum_type, filename) if local_checksum == checksum: return 1 else: raise URLGrabError(-1, 'Metadata file does not match checksum')
def get_credentials(self): print("get_credentials") """Read IAM credentials from AWS metadata store. Note: This method should be explicitly called after constructing new object, as in 'explicit is better than implicit'. """ request = urllib2.Request( urlparse.urljoin( urlparse.urljoin( "http://169.254.169.254/", "latest/meta-data/iam/security-credentials/", ), self.iamrole)) try: print("Opening url... %s" % (request)) response = urllib2.urlopen(request) data = json.loads(response.read()) print("response urllib2: %s" % (data)) self.access_key = data['AccessKeyId'] self.secret_key = data['SecretAccessKey'] self.token = data['Token'] except Exception: print("error in library") response = None finally: if response: response.close() if self.access_key is None and self.secret_key is None: if "AWS_ACCESS_KEY_ID" in os.environ: self.access_key = os.environ['AWS_ACCESS_KEY_ID'] if "AWS_SECRET_ACCESS_KEY" in os.environ: self.secret_key = os.environ['AWS_SECRET_ACCESS_KEY'] if "AWS_SESSION_TOKEN" in os.environ: self.token = os.environ['AWS_SESSION_TOKEN'] if self.access_key is None and self.secret_key is None: if hasattr(self, 'name'): msg = "Could not access AWS credentials, skipping repository '%s'" % ( self.name) else: msg = "Could not access AWS credentials" print msg from urlgrabber.grabber import URLGrabError raise URLGrabError(7, msg)
def get_credentials(self): """Read IAM credentials from AWS metadata store. Note: This method should be explicitly called after constructing new object, as in 'explicit is better than implicit'. """ credentials = self.session.get_credentials() self.access_key = credentials.access_key self.secret_key = credentials.secret_key self.token = credentials.token if self.access_key is None and self.secret_key is None: if hasattr(self, 'name'): msg = "Could not access AWS credentials, skipping repository '%s'" % (self.name) else: msg = "Could not access AWS credentials" print msg from urlgrabber.grabber import URLGrabError raise URLGrabError(7, msg)
def checkRpmMD5(package, urlgraberror=0): """take a package, check it out by trying to open it, return 1 if it's good return 0 if it's not""" ts.sigChecking('md5') fdno = os.open(package, os.O_RDONLY) try: ts.hdrFromFdno(fdno) except rpm.error as e: good = 0 else: good = 1 os.close(fdno) ts.sigChecking('default') if urlgraberror: if not good: raise URLGrabError(-1, _('RPM %s fails md5 check') % (package)) else: return else: return good
def checkfunc_read(obj): if obj.data == 'version1': raise URLGrabError(-1, 'Outdated version of foo')
def checkfunc_grab(obj): with open('foo') as f: if f.read() == 'version1': raise URLGrabError(-1, 'Outdated version of foo')
def checkfunc_read(obj): if obj.data == b'version1': raise URLGrabError(-1, 'Outdated version of foo') elif obj.data != b'version2': self.fail('Unexpected file content')
def run(self, buildinfo): cached = 0 all = len(buildinfo.deps) for i in buildinfo.deps: i.makeurls(self.cachedir, self.urllist) if os.path.exists(i.fullfilename): cached += 1 miss = 0 needed = all - cached if all: miss = 100.0 * needed / all print "%.1f%% cache miss. %d/%d dependencies cached.\n" % (miss, cached, all) done = 1 for i in buildinfo.deps: i.makeurls(self.cachedir, self.urllist) if not os.path.exists(i.fullfilename): if self.offline: raise oscerr.OscIOError( None, 'Missing package \'%s\' in cache: --offline not possible.' % i.fullfilename) self.dirSetup(i) try: # if there isn't a progress bar, there is no output at all if not self.progress_obj: print '%d/%d (%s) %s' % (done, needed, i.project, i.filename) self.fetch(i) if self.progress_obj: print " %d/%d\r" % (done, needed), sys.stdout.flush() except KeyboardInterrupt: print 'Cancelled by user (ctrl-c)' print 'Exiting.' sys.exit(0) done += 1 self.__fetch_cpio(buildinfo.apiurl) prjs = buildinfo.projects.keys() for i in prjs: dest = "%s/%s" % (self.cachedir, i) if not os.path.exists(dest): os.makedirs(dest, mode=0755) dest += '/_pubkey' url = makeurl(buildinfo.apiurl, ['source', i, '_pubkey']) try: if self.offline and not os.path.exists(dest): # may need to try parent raise URLGrabError(2) elif not self.offline: OscFileGrabber().urlgrab(url, dest) if not i in buildinfo.prjkeys: # not that many keys usually buildinfo.keys.append(dest) buildinfo.prjkeys.append(i) except KeyboardInterrupt: print 'Cancelled by user (ctrl-c)' print 'Exiting.' if os.path.exists(dest): os.unlink(dest) sys.exit(0) except URLGrabError, e: if self.http_debug: print >> sys.stderr, "can't fetch key for %s: %s" % ( i, e.strerror) print >> sys.stderr, "url: %s" % url if os.path.exists(dest): os.unlink(dest) l = i.rsplit(':', 1) # try key from parent project if len(l) > 1 and l[1] and not l[0] in buildinfo.projects: prjs.append(l[0])
def run(self, buildinfo): cached = 0 all = len(buildinfo.deps) for i in buildinfo.deps: i.makeurls(self.cachedir, self.urllist) if os.path.exists(i.fullfilename): cached += 1 if i.hdrmd5: from .util import packagequery hdrmd5 = packagequery.PackageQuery.queryhdrmd5(i.fullfilename) if not hdrmd5 or hdrmd5 != i.hdrmd5: os.unlink(i.fullfilename) cached -= 1 miss = 0 needed = all - cached if all: miss = 100.0 * needed / all print("%.1f%% cache miss. %d/%d dependencies cached.\n" % (miss, cached, all)) done = 1 for i in buildinfo.deps: i.makeurls(self.cachedir, self.urllist) if not os.path.exists(i.fullfilename): if self.offline: raise oscerr.OscIOError(None, 'Missing \'%s\' in cache: ' '--offline not possible.' % i.fullfilename) self.dirSetup(i) if i.hdrmd5 and self.enable_cpio: self.__add_cpio(i) done += 1 continue try: # if there isn't a progress bar, there is no output at all if not self.progress_obj: print('%d/%d (%s) %s' % (done, needed, i.project, i.filename)) self.fetch(i) if self.progress_obj: print(" %d/%d\r" % (done, needed), end=' ') sys.stdout.flush() except KeyboardInterrupt: print('Cancelled by user (ctrl-c)') print('Exiting.') sys.exit(0) done += 1 self.__fetch_cpio(buildinfo.apiurl) prjs = list(buildinfo.projects.keys()) for i in prjs: dest = "%s/%s" % (self.cachedir, i) if not os.path.exists(dest): os.makedirs(dest, mode=0o755) dest += '/_pubkey' url = makeurl(buildinfo.apiurl, ['source', i, '_pubkey']) try: if self.offline and not os.path.exists(dest): # may need to try parent raise URLGrabError(2) elif not self.offline: OscFileGrabber().urlgrab(url, dest) # not that many keys usually if i not in buildinfo.prjkeys: buildinfo.keys.append(dest) buildinfo.prjkeys.append(i) except KeyboardInterrupt: print('Cancelled by user (ctrl-c)') print('Exiting.') if os.path.exists(dest): os.unlink(dest) sys.exit(0) except URLGrabError as e: # Not found is okay, let's go to the next project if e.errno == 14 and e.code != 404: print("Invalid answer from server", e, file=sys.stderr) sys.exit(1) if self.http_debug: print("can't fetch key for %s: %s" % (i, e.strerror), file=sys.stderr) print("url: %s" % url, file=sys.stderr) if os.path.exists(dest): os.unlink(dest) l = i.rsplit(':', 1) # try key from parent project if len(l) > 1 and l[1] and not l[0] in buildinfo.projects: prjs.append(l[0])
"""take a package, check it out by trying to open it, return 1 if its good return 0 if it's not""" ts.sigChecking('md5') fdno = os.open(package, os.O_RDONLY) try: ts.hdrFromFdno(fdno) except rpm.error, e: good = 0 else: good = 1 os.close(fdno) ts.sigChecking('default') if urlgraberror: if not good: raise URLGrabError(-1, _('RPM %s fails md5 check') % (package)) else: return else: return good def checkSig(package): """ take a package, check it's sigs, return 0 if they are all fine, return 1 if the gpg key can't be found, 2 if the header is in someway damaged, 3 if the key is not trusted, 4 if the pkg is not gpg or pgp signed""" ts.sigChecking('default') fdno = os.open(package, os.O_RDONLY) try: hdr = ts.hdrFromFdno(fdno) except rpm.error, e: if str(e) == "public key not availaiable":