def test_interrupt_callback_raises(self): "interrupt callback raises an exception" ic = (self._interrupt_callback, (), {'exception': self.TestException()}) g = grabber.URLGrabber(progress_obj=self.InterruptProgress(), interrupt_callback=ic) self.assertRaises(self.TestException, g.urlgrab, ref_http)
def myurlgrab(url, filename, proxies, progress_obj=None, ignore_404=False): g = grabber.URLGrabber(netrc_optional=True) if progress_obj is None: progress_obj = TextProgress() if url.startswith("file:/"): file = url.replace("file:", "") if not os.path.exists(file): if ignore_404: return None else: raise CreatorError("URLGrabber error: can't find file %s" % file) runner.show(['cp', "-f", file, filename]) else: try: filename = g.urlgrab(url=url, filename=filename, ssl_verify_host=False, ssl_verify_peer=False, proxies=proxies, http_headers=(('Pragma', 'no-cache'), ), quote=0, progress_obj=progress_obj) except grabber.URLGrabError as e: if e.errno == 14 and e.code in [404, 503] and ignore_404: return None else: raise CreatorError("URLGrabber error: %s - %s" % (url, e)) return filename
def myurlgrab(url, filename, proxies, progress_obj=None): g = grabber.URLGrabber() if progress_obj is None: progress_obj = TextProgress() if url.startswith("file:/"): filepath = "/%s" % url.replace("file:", "").lstrip('/') if not os.path.exists(filepath): raise CreatorError("URLGrabber error: can't find file %s" % url) if url.endswith('.rpm'): return filepath else: # untouch repometadata in source path runner.show(['cp', '-f', filepath, filename]) else: try: filename = g.urlgrab(url=str(url), filename=filename, ssl_verify_host=False, ssl_verify_peer=False, proxies=proxies, http_headers=(('Pragma', 'no-cache'), ), quote=0, progress_obj=progress_obj) except grabber.URLGrabError, err: msg = str(err) if msg.find(url) < 0: msg += ' on %s' % url raise CreatorError(msg)
def test_failure_callback_called(self): "failure callback is called on retry" self.failure_callback_called = 0 g = grabber.URLGrabber(retry=2, retrycodes=[14], failure_callback=self._failure_callback) try: g.urlgrab(ref_404) except URLGrabError: pass self.assertEquals(self.failure_callback_called, 1)
def test_interrupt_callback_called(self): "interrupt callback is called on retry" self.interrupt_callback_called = 0 ic = (self._interrupt_callback, (), {}) g = grabber.URLGrabber(progress_obj=self.InterruptProgress(), interrupt_callback=ic) try: g.urlgrab(ref_http) except KeyboardInterrupt: pass self.assertEquals(self.interrupt_callback_called, 1)
def _download_repo(cls, project_name, repo_filename, chroot=None): if chroot is None: chroot = cls._guess_chroot() #http://copr.fedoraproject.org/coprs/larsks/rcm/repo/epel-7-x86_64/ api_path = "/coprs/{0}/repo/{1}/".format(project_name, chroot) ug = grabber.URLGrabber() # FIXME when we are full on python2 urllib.parse try: ug.urlgrab(cls.copr_url + api_path, filename=repo_filename) except grabber.URLGrabError as e: cls._remove_repo(repo_filename) raise YError(str(e))
def setUp(self): self.ref = short_reference_data tmp = tempfile.mktemp() with open(tmp, 'wb') as tmpfo: tmpfo.write(self.ref) self.tmp = tmp (url, parts) = grabber.default_grabber.opts.urlparser.parse( tmp, grabber.default_grabber.opts) self.url = url self.grabber = grabber.URLGrabber(reget='check_timestamp', copy_local=1) self.filename = tempfile.mktemp() self.hl = len(self.ref) // 2
def setUp(self): self.ref = short_reference_data _, tmp = tempfile.mkstemp() tmpfo = open(tmp, 'wb' if not six.PY3 else 'w') tmpfo.write(self.ref) tmpfo.close() self.tmp = tmp (url, parts) = grabber.default_grabber.opts.urlparser.parse( tmp, grabber.default_grabber.opts) self.url = url self.grabber = grabber.URLGrabber(reget='check_timestamp', copy_local=1) _, self.filename = tempfile.mkstemp() self.hl = len(self.ref) / 2
def test_failure_callback_args(self): "failure callback is called with the proper args" fc = (self._failure_callback, ('foo',), {'bar': 'baz'}) g = grabber.URLGrabber(retry=2, retrycodes=[14], failure_callback=fc) try: g.urlgrab(ref_404) except URLGrabError: pass self.assert_(hasattr(self, 'obj')) self.assert_(hasattr(self, 'args')) self.assert_(hasattr(self, 'kwargs')) self.assertEquals(self.args, ('foo',)) self.assertEquals(self.kwargs, {'bar': 'baz'}) self.assert_(isinstance(self.obj, CallbackObject)) self.assertEquals(self.obj.url, ref_404) self.assert_(isinstance(self.obj.exception, URLGrabError)) del self.obj
def myurlgrab(url, filename, proxies, progress_obj=None): g = grabber.URLGrabber() if progress_obj is None: progress_obj = TextProgress() if url.startswith("file:/"): file = url.replace("file:", "") if not os.path.exists(file): raise CreatorError("URLGrabber error: can't find file %s" % file) runner.show(['cp', "-f", file, filename]) else: try: filename = g.urlgrab(url=url, filename=filename, ssl_verify_host=False, ssl_verify_peer=False, proxies=proxies, http_headers=(('Pragma', 'no-cache'), ), quote=0, progress_obj=progress_obj) except grabber.URLGrabError, e: raise CreatorError("URLGrabber error: %s" % url)
to_be_built = pkgs while try_again: failed = [] for pkg in to_be_built: if not pkg.endswith('.rpm'): log(opts.logfile, "%s doesn't appear to be an rpm - skipping" % pkg) failed.append(pkg) continue elif pkg.startswith('http://') or pkg.startswith('https://'): url = pkg cwd = os.getcwd() os.chdir(download_dir) try: log(opts.logfile, 'Fetching %s' % url) ug = grabber.URLGrabber() fn = ug.urlgrab(url) pkg = download_dir + '/' + fn except Exception, e: log(opts.logfile, 'Error Downloading %s: %s' % (url, str(e))) failed.append(url) os.chdir(cwd) continue else: os.chdir(cwd) downloaded_pkgs[pkg] = url log(opts.logfile, "Start build: %s" % pkg) ret, cmd, out, err = do_build(opts, cfg, pkg) log(opts.logfile, "End build: %s" % pkg) if ret == 0: if opts.recurse:
def test_restart_workaround(self): inst = grabber.URLGrabber() rslt = inst.urlread(self.url, range=(500, 1000))
def setUp(self): self.ref = short_reference_data self.grabber = grabber.URLGrabber(reget='check_timestamp') _, self.filename = tempfile.mkstemp() self.hl = len(self.ref) / 2 self.url = 'OVERRIDE THIS'
def setUp(self): cf = (self._checkfunc, ('foo',), {'bar': 'baz'}) self.g = grabber.URLGrabber(checkfunc=cf) _, self.filename = tempfile.mkstemp() self.data = short_reference_data
def main(args): opts, args = parse_args(args) # take mock config + list of pkgs cfg = opts.chroot pkgs = args[1:] mockcfg = mockconfig_path + '/' + cfg + '.cfg' if not os.path.exists(mockcfg): print "could not find config: %s" % mockcfg sys.exit(1) username = os.getlogin() pid = os.getpid() opts.uniqueext = '%s-%s' % (username, pid) # create a tempdir for our local info if opts.localrepo: local_tmp_dir = os.path.abspath(opts.localrepo) if not os.path.exists(local_tmp_dir): os.makedirs(local_tmp_dir) else: pre = 'mock-chain-%s-' % opts.uniqueext local_tmp_dir = tempfile.mkdtemp(prefix=pre, dir='/var/tmp') os.chmod(local_tmp_dir, 0755) if opts.logfile: opts.logfile = os.path.join(local_tmp_dir, opts.logfile) if os.path.exists(opts.logfile): os.unlink(opts.logfile) log(opts.logfile, "starting logfile: %s" % opts.logfile) opts.local_repo_dir = os.path.normpath(local_tmp_dir + '/results/' + cfg + '/') if not os.path.exists(opts.local_repo_dir): os.makedirs(opts.local_repo_dir, mode=0755) local_baseurl = "file://%s" % opts.local_repo_dir log(opts.logfile, "results dir: %s" % opts.local_repo_dir) opts.config_path = os.path.normpath(local_tmp_dir + '/configs/' + cfg + '/') if not os.path.exists(opts.config_path): os.makedirs(opts.config_path, mode=0755) log(opts.logfile, "config dir: %s" % opts.config_path) my_mock_config = opts.config_path + '/' + os.path.basename(mockcfg) # modify with localrepo res, msg = add_local_repo(mockcfg, my_mock_config, local_baseurl, 'local_build_repo') if not res: log(opts.logfile, "Error: Could not write out local config: %s" % msg) sys.exit(1) for baseurl in opts.repos: res, msg = add_local_repo(my_mock_config, my_mock_config, baseurl) if not res: log( opts.logfile, "Error: Could not add: %s to yum config in mock chroot: %s" % (baseurl, msg)) sys.exit(1) # these files needed from the mock.config dir to make mock run for fn in ['site-defaults.cfg', 'logging.ini']: pth = mockconfig_path + '/' + fn shutil.copyfile(pth, opts.config_path + '/' + fn) # createrepo on it out, err = createrepo(opts.local_repo_dir) if err.strip(): log(opts.logfile, "Error making local repo: %s" % opts.local_repo_dir) log(opts.logfile, "Err: %s" % err) sys.exit(1) download_dir = tempfile.mkdtemp() downloaded_pkgs = {} built_pkgs = [] try_again = True to_be_built = pkgs while try_again: failed = [] for pkg in to_be_built: if not pkg.endswith('.rpm'): log(opts.logfile, "%s doesn't appear to be an rpm - skipping" % pkg) failed.append(pkg) continue elif pkg.startswith('http://') or pkg.startswith('https://'): url = pkg cwd = os.getcwd() os.chdir(download_dir) try: log(opts.logfile, 'Fetching %s' % url) ug = grabber.URLGrabber() fn = ug.urlgrab(url) pkg = download_dir + '/' + fn except Exception, e: log(opts.logfile, 'Error Downloading %s: %s' % (url, str(e))) failed.append(url) os.chdir(cwd) continue else: os.chdir(cwd) downloaded_pkgs[pkg] = url log(opts.logfile, "Start build: %s" % pkg) ret, cmd, out, err = do_build(opts, cfg, pkg) log(opts.logfile, "End build: %s" % pkg) if ret == 0: if opts.recurse: failed.append(pkg) log( opts.logfile, "Error building %s, will try again" % os.path.basename(pkg)) else: log(opts.logfile, "Error building %s" % os.path.basename(pkg)) log(opts.logfile, "See logs/results in %s" % opts.local_repo_dir) if not opts.cont: sys.exit(1) elif ret == 1: log(opts.logfile, "Success building %s" % os.path.basename(pkg)) built_pkgs.append(pkg) # createrepo with the new pkgs out, err = createrepo(opts.local_repo_dir) if err.strip(): log(opts.logfile, "Error making local repo: %s" % opts.local_repo_dir) log(opts.logfile, "Err: %s" % err) elif ret == 2: log(opts.logfile, "Skipping already built pkg %s" % os.path.basename(pkg)) if failed: if len(failed) != len(to_be_built): to_be_built = failed try_again = True log(opts.logfile, 'Trying to rebuild %s failed pkgs' % len(failed)) else: log( opts.logfile, "Tried twice - following pkgs could not be successfully built:" ) for pkg in failed: msg = pkg if pkg in downloaded_pkgs: msg = downloaded_pkgs[pkg] log(opts.logfile, msg) try_again = False else: try_again = False