def setUp(self): def server(): import socket s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) s.bind(('localhost', 2000)) s.listen(1) while 1: c, a = s.accept() while not c.recv(4096).endswith('\r\n\r\n'): pass c.sendall('HTTP/1.1 %d %s\r\n' % self.reply) c.close() import thread self.reply = 503, "Busy" thread.start_new_thread(server, ()) def failure(obj): self.code = getattr(obj.exception, 'code', None) return {} self.g = URLGrabber() self.mg = MirrorGroup(self.g, ['http://localhost:2000/'], failure_callback=failure)
def fetch(self, pac, prefix=''): # for use by the failure callback self.curpac = pac MirrorGroup._join_url = join_url mg = MirrorGroup(self.gr, pac.urllist, failure_callback=(self.failureReport, (), {})) if self.http_debug: print('\nURLs to try for package \'%s\':' % pac, file=sys.stderr) print('\n'.join(pac.urllist), file=sys.stderr) print(file=sys.stderr) try: with tempfile.NamedTemporaryFile(prefix='osc_build', delete=False) as tmpfile: mg.urlgrab(pac.filename, filename=tmpfile.name, text='%s(%s) %s' % (prefix, pac.project, pac.filename)) self.move_package(tmpfile.name, pac.localdir, pac) except URLGrabError as e: if self.enable_cpio and e.errno == 256: self.__add_cpio(pac) return print() print('Error:', e.strerror, file=sys.stderr) print('Failed to retrieve %s from the following locations ' '(in order):' % pac.filename, file=sys.stderr) print('\n'.join(pac.urllist), file=sys.stderr) sys.exit(1) finally: if os.path.exists(tmpfile.name): os.unlink(tmpfile.name)
def setUp(self): self.snarfed_logs = [] self.db = urlgrabber.mirror.DEBUG urlgrabber.mirror.DEBUG = FakeLogger() self.mirrors = ['a', 'b', 'c', 'd', 'e', 'f'] self.g = FakeGrabber([URLGrabError(3), URLGrabError(3), 'filename']) self.mg = MirrorGroup(self.g, self.mirrors)
def fetch(self, pac, prefix=''): # for use by the failure callback self.curpac = pac MirrorGroup._join_url = join_url mg = MirrorGroup(self.gr, pac.urllist, failure_callback=(self.failureReport, (), {})) if self.http_debug: print >> sys.stderr, '\nURLs to try for package \'%s\':' % pac print >> sys.stderr, '\n'.join(pac.urllist) print >> sys.stderr (fd, tmpfile) = tempfile.mkstemp(prefix='osc_build') try: try: mg.urlgrab(pac.filename, filename=tmpfile, text='%s(%s) %s' % (prefix, pac.project, pac.filename)) self.move_package(tmpfile, pac.localdir, pac) except URLGrabError, e: if self.enable_cpio and e.errno == 256: self.__add_cpio(pac) return print print >> sys.stderr, 'Error:', e.strerror print >> sys.stderr, 'Failed to retrieve %s from the following locations (in order):' % pac.filename print >> sys.stderr, '\n'.join(pac.urllist) sys.exit(1) finally: os.close(fd) if os.path.exists(tmpfile): os.unlink(tmpfile)
def setUp(self): self.g = URLGrabber() fullmirrors = [base_mirror_url + m + '/' for m in \ (bad_mirrors + good_mirrors)] if hasattr(urlgrabber.grabber, '_TH'): # test assumes mirrors are not re-ordered urlgrabber.grabber._TH.hosts.clear() self.mg = MirrorGroup(self.g, fullmirrors)
def setUp(self): # start the server self.exit = False self.process = lambda data: None s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) s.bind(('localhost', 0)) s.listen(1) self.port = s.getsockname()[1] def server(): while True: c, a = s.accept() if self.exit: c.close() break data = b'' while not data.endswith(b'\r\n\r\n'): data = c.recv(4096) self.process(data) c.sendall(b'HTTP/1.1 %d %s\r\n' % self.reply) if self.content is not None: c.sendall(b'Content-Length: %d\r\n\r\n' % len(self.content)) c.sendall(self.content) c.close() s.close() self.exit = False self.thread = threading.Thread(target=server) self.thread.start() # create grabber and mirror group objects def failure(obj): self.code = getattr(obj.exception, 'code', None) return {} self.g = URLGrabber() self.mg = MirrorGroup(self.g, ['http://localhost:%d' % self.port], failure_callback=failure)
def setUp(self): # start the server self.exit = False def server(): s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) s.bind(LOCALPORT) s.listen(1) while 1: c, a = s.accept() if self.exit: c.close() break ending_compat = '\r\n\r\n' if not six.PY3 else b'\r\n\r\n' while not c.recv(4096).endswith(ending_compat): pass http_compat = 'HTTP/1.1 %d %s\r\n' % self.reply c.sendall(http_compat if not six.PY3 else http_compat. encode('utf-8')) if self.content is not None: cont_length_compat = 'Content-Length: %d\r\n\r\n' % len( self.content) c.sendall(cont_length_compat if not six.PY3 else cont_length_compat.encode('utf-8')) c.sendall(self.content if not six.PY3 else self.content. encode('utf-8')) c.close() s.close() self.exit = False thread.start_new_thread(server, ()) # create grabber and mirror group objects def failure(obj): self.code = getattr(obj.exception, 'code', None) return {} self.g = URLGrabber() self.mg = MirrorGroup(self.g, ['http://%s:%d' % LOCALPORT], failure_callback=failure)
def setUp(self): self.g = URLGrabber() fullmirrors = [base_mirror_url + m + '/' for m in good_mirrors] self.mg = MirrorGroup(self.g, fullmirrors)
#!/usr/bin/python3 from urlgrabber import urlopen from urlgrabber.grabber import URLGrabber from urlgrabber.mirror import MirrorGroup fo = urlopen('http://localhost') data = fo.read() print(data) gr = URLGrabber() mg = MirrorGroup(gr, ['http://localhost2/', 'http://me.myself/']) mg.urlgrab('test.txt')