def setUp(self):
def server():
import socket
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
s.bind(('localhost', 2000))
s.listen(1)
while 1:
c, a = s.accept()
while not c.recv(4096).endswith('\r\n\r\n'):
pass
c.sendall('HTTP/1.1 %d %s\r\n' % self.reply)
c.close()
import thread
self.reply = 503, "Busy"
thread.start_new_thread(server, ())
def failure(obj):
self.code = getattr(obj.exception, 'code', None)
return {}
self.g = URLGrabber()
self.mg = MirrorGroup(self.g, ['http://localhost:2000/'],
failure_callback=failure)
def fetch(self, pac, prefix=''):
# for use by the failure callback
self.curpac = pac
MirrorGroup._join_url = join_url
mg = MirrorGroup(self.gr, pac.urllist, failure_callback=(self.failureReport, (), {}))
if self.http_debug:
print('\nURLs to try for package \'%s\':' % pac, file=sys.stderr)
print('\n'.join(pac.urllist), file=sys.stderr)
print(file=sys.stderr)
try:
with tempfile.NamedTemporaryFile(prefix='osc_build',
delete=False) as tmpfile:
mg.urlgrab(pac.filename, filename=tmpfile.name,
text='%s(%s) %s' % (prefix, pac.project, pac.filename))
self.move_package(tmpfile.name, pac.localdir, pac)
except URLGrabError as e:
if self.enable_cpio and e.errno == 256:
self.__add_cpio(pac)
return
print()
print('Error:', e.strerror, file=sys.stderr)
print('Failed to retrieve %s from the following locations '
'(in order):' % pac.filename, file=sys.stderr)
print('\n'.join(pac.urllist), file=sys.stderr)
sys.exit(1)
finally:
if os.path.exists(tmpfile.name):
os.unlink(tmpfile.name)
def setUp(self):
self.snarfed_logs = []
self.db = urlgrabber.mirror.DEBUG
urlgrabber.mirror.DEBUG = FakeLogger()
self.mirrors = ['a', 'b', 'c', 'd', 'e', 'f']
self.g = FakeGrabber([URLGrabError(3), URLGrabError(3), 'filename'])
self.mg = MirrorGroup(self.g, self.mirrors)
def fetch(self, pac, prefix=''):
# for use by the failure callback
self.curpac = pac
MirrorGroup._join_url = join_url
mg = MirrorGroup(self.gr,
pac.urllist,
failure_callback=(self.failureReport, (), {}))
if self.http_debug:
print >> sys.stderr, '\nURLs to try for package \'%s\':' % pac
print >> sys.stderr, '\n'.join(pac.urllist)
print >> sys.stderr
(fd, tmpfile) = tempfile.mkstemp(prefix='osc_build')
try:
try:
mg.urlgrab(pac.filename,
filename=tmpfile,
text='%s(%s) %s' %
(prefix, pac.project, pac.filename))
self.move_package(tmpfile, pac.localdir, pac)
except URLGrabError, e:
if self.enable_cpio and e.errno == 256:
self.__add_cpio(pac)
return
print
print >> sys.stderr, 'Error:', e.strerror
print >> sys.stderr, 'Failed to retrieve %s from the following locations (in order):' % pac.filename
print >> sys.stderr, '\n'.join(pac.urllist)
sys.exit(1)
finally:
os.close(fd)
if os.path.exists(tmpfile):
os.unlink(tmpfile)
def setUp(self):
self.g = URLGrabber()
fullmirrors = [base_mirror_url + m + '/' for m in \
(bad_mirrors + good_mirrors)]
if hasattr(urlgrabber.grabber, '_TH'):
# test assumes mirrors are not re-ordered
urlgrabber.grabber._TH.hosts.clear()
self.mg = MirrorGroup(self.g, fullmirrors)
def setUp(self):
# start the server
self.exit = False
self.process = lambda data: None
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
s.bind(('localhost', 0))
s.listen(1)
self.port = s.getsockname()[1]
def server():
while True:
c, a = s.accept()
if self.exit:
c.close()
break
data = b''
while not data.endswith(b'\r\n\r\n'):
data = c.recv(4096)
self.process(data)
c.sendall(b'HTTP/1.1 %d %s\r\n' % self.reply)
if self.content is not None:
c.sendall(b'Content-Length: %d\r\n\r\n' %
len(self.content))
c.sendall(self.content)
c.close()
s.close()
self.exit = False
self.thread = threading.Thread(target=server)
self.thread.start()
# create grabber and mirror group objects
def failure(obj):
self.code = getattr(obj.exception, 'code', None)
return {}
self.g = URLGrabber()
self.mg = MirrorGroup(self.g, ['http://localhost:%d' % self.port],
failure_callback=failure)
def setUp(self):
# start the server
self.exit = False
def server():
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
s.bind(LOCALPORT)
s.listen(1)
while 1:
c, a = s.accept()
if self.exit:
c.close()
break
ending_compat = '\r\n\r\n' if not six.PY3 else b'\r\n\r\n'
while not c.recv(4096).endswith(ending_compat):
pass
http_compat = 'HTTP/1.1 %d %s\r\n' % self.reply
c.sendall(http_compat if not six.PY3 else http_compat.
encode('utf-8'))
if self.content is not None:
cont_length_compat = 'Content-Length: %d\r\n\r\n' % len(
self.content)
c.sendall(cont_length_compat if not six.PY3 else
cont_length_compat.encode('utf-8'))
c.sendall(self.content if not six.PY3 else self.content.
encode('utf-8'))
c.close()
s.close()
self.exit = False
thread.start_new_thread(server, ())
# create grabber and mirror group objects
def failure(obj):
self.code = getattr(obj.exception, 'code', None)
return {}
self.g = URLGrabber()
self.mg = MirrorGroup(self.g, ['http://%s:%d' % LOCALPORT],
failure_callback=failure)
def setUp(self):
self.g = URLGrabber()
fullmirrors = [base_mirror_url + m + '/' for m in good_mirrors]
self.mg = MirrorGroup(self.g, fullmirrors)
#!/usr/bin/python3
from urlgrabber import urlopen
from urlgrabber.grabber import URLGrabber
from urlgrabber.mirror import MirrorGroup
fo = urlopen('http://localhost')
data = fo.read()
print(data)
gr = URLGrabber()
mg = MirrorGroup(gr, ['http://localhost2/', 'http://me.myself/'])
mg.urlgrab('test.txt')
