def process(path, file, hddsem):
with open(path) as f:
links = container[path.ext](f.read())
core.add_links(links)
if config.delete_after_process is not None:
delete = config.delete_after_process
else:
result = dict()
delete = False
elements = [
input.Text('Added links of container {}.\nDelete container now?'.format(os.path.basename(path))),
input.Input('remember', 'checkbox', label='Remember decision?'),
input.Choice('delete', choices=[
{"value": True, "content": "Yes"},
{"value": False, "content": "No"},
])]
try:
result = input.get(elements, type='delete_container', parent=file, timeout=120)
delete = result['delete']
except input.InputTimeout:
log.warning('input timed out')
except input.InputError:
log.exception('input was aborted')
if result.get('remember', False):
config.delete_after_process = delete
if delete:
if file:
file.erase_after_greenlet()
else:
os.remove(path)
raise gevent.GreenletExit()
def __url_crawler__(self, url, idx):
"""Worker to crawl a url. Downloads the url, parses it and stores any new outgoing links. #TODO add a seperate pipeline for postprocessing."""
LOGGER = get_logger("__url_crawler__")
LOGGER.debug("worker {idx}: starting to crawl {url}".format(url=url,
idx=idx))
try:
req = requests.get(url)
LOGGER.debug("worker {idx}: download_success {url}".format(
url=url, idx=idx))
except Exception as e:
LOGGER.warn("worker {idx}: download_failed {url}".format(url=url,
idx=idx))
req = None
pass
if req and req.status_code == 200:
try:
soup = BeautifulSoup(req.text, "html.parser")
LOGGER.debug("worker {idx}: parsed {url}".format(url=url,
idx=idx))
anchors = soup.find_all('a', href=True)
LOGGER.debug("worker {idx}: found {anchors}".format(
idx=idx, anchors=len(anchors)))
anchors = [
self.__add_url_to_q__(anchor['href']) for anchor in anchors
]
LOGGER.debug("worker {idx}: added {anchors}".format(
idx=idx, anchors=sum(anchors)))
except Exception as e:
LOGGER.warn("adding urls failed")
LOGGER.debug("exiting worker {idx}".format(idx=idx))
self.url_crawled.set()
raise gevent.GreenletExit('success')
def worker(url):
html = (D.post if post else D.get)(url, **kwargs)
if cb:
for url in (cb(D, url, html) or []):
inq.put(url)
worker_finished.set()
raise gevent.GreenletExit('success')
def send(self, msg):
"""Wrapper for self.io.send, to detect any sending problem and
call the destroy callbacks"""
if not self.connected:
self.kill()
self.debug("not connected on send(): exiting greenlet")
raise gevent.GreenletExit()
self.io.send(msg)
def wrapped(ctx, *args, **kwargs):
io = ctx.io
if not io.session.connected:
ctx.kill()
ctx.debug("not connected on %s: exiting greenlet", fn.__name__)
raise gevent.GreenletExit()
return fn(ctx, *args, **kwargs)
def test_together_kill(self):
sock = mock.Mock()
with self.assertRaises(gevent.GreenletExit):
with utils.SocketCloser(sock):
sock.make_call()
raise gevent.GreenletExit()
sock.make_call.assert_called_once_with()
sock.close.assert_called_once_with()
def test_exit_kill_thresh(self):
sock = mock.Mock()
closer = utils.SocketCloser(sock, 10)
result = closer.__exit__(gevent.GreenletExit, gevent.GreenletExit(),
[])
self.assertEqual(result, None)
sock.close.assert_called_once_with()
self.assertEqual(closer.errors, 0)
def fatal(self, msg, abort_greenlet=True):
if self.input:
interface.call('input', 'abort', id=self.input.id)
self.input = None
with transaction:
self.last_error = msg
self.enabled = False
self.log.error(msg)
event.fire('file:fatal_error', self)
if abort_greenlet:
raise gevent.GreenletExit()
def run(self):
self.setup_in_process()
last_clean = datetime.utcnow()
while True: # Run forever and ever
# Clean out visited every 5 minutes
if datetime.utcnow() - last_clean > timedelta(minutes=5):
self._log.debug("Cleaning cache...")
self.__cache.clean_and_save()
last_clean = datetime.utcnow()
try: # Get next object to process
event = self.__queue.get(block=True, timeout=5)
except gevent.queue.Empty:
# Check if the process should exit process
if self.__event.is_set():
break
# Explict context yield
gevent.sleep(0)
continue
try:
kind = type(event)
self._log.debug("Processing event: %s", event.id)
if kind == Events.MonEvent:
self.process_monster(event)
elif kind == Events.StopEvent:
self.process_stop(event)
elif kind == Events.GruntEvent:
self.process_grunt(event)
elif kind == Events.GymEvent:
self.process_gym(event)
elif kind == Events.EggEvent:
self.process_egg(event)
elif kind == Events.RaidEvent:
self.process_raid(event)
elif kind == Events.WeatherEvent:
self.process_weather(event)
elif kind == Events.QuestEvent:
self.process_quest(event)
else:
self._log.error(
"!!! Manager does not support {} events!".format(kind))
self._log.debug("Finished event: %s", event.id)
except Exception as e:
self._log.error("Encountered error during processing: "
"{}: {}".format(type(e).__name__, e))
self._log.error("Stack trace: \n {}"
"".format(traceback.format_exc()))
# Explict context yield
gevent.sleep(0)
# Save cache and exit
self.__cache.clean_and_save()
raise gevent.GreenletExit()
def test_together_kill_thresh(self):
sock = mock.Mock()
closer = utils.SocketCloser(sock, 10)
with self.assertRaises(gevent.GreenletExit):
with closer:
sock.make_call()
raise gevent.GreenletExit()
sock.make_call.assert_called_once_with()
sock.close.assert_called_once_with()
self.assertEqual(closer.errors, 0)
def retry(self, msg, seconds, need_reconnect=False):
seconds = float(seconds)
self.log.info(u'retry in {} seconds: {}; reconnect: {}'.format(
seconds, msg, need_reconnect))
with transaction:
self.last_error = msg
self.need_reconnect = need_reconnect
self.next_try = time.time() + seconds
if self.input:
interface.call('input', 'abort', id=self.input.id)
self.input = None
raise gevent.GreenletExit()
def retry(
self,
msg,
seconds,
_=False
): # _ is a placeholder to make ErrorFunctions work. we have no need_reconnect
with transaction:
self.next_try = gevent.spawn_later(seconds, self.reset_retry)
self.next_try.eta = time.time() + seconds
self.last_error = msg
self.log.info('retry in {} seconds: {}'.format(seconds, msg))
raise gevent.GreenletExit()
def retry(self, msg, seconds, need_reconnect=False):
seconds = float(seconds)
if self.input:
interface.call('input', 'abort', id=self.input.id)
self.input = None
with transaction:
self.last_error = msg
self.need_reconnect = need_reconnect
self.next_try = gevent.spawn_later(seconds, self.reset_retry)
self.next_try.eta = time.time() + seconds
self.retry_num += 1
self.log.info(u'retry in {} seconds: {}; reconnect: {}'.format(
seconds, msg, need_reconnect))
event.fire('file:retry', self)
raise gevent.GreenletExit()
def add_torrent(t, file=None):
"""t = Torrent object instance or string with torrent data"""
if isinstance(t, str):
try:
t = Torrent(data=t,
options=dict(save_path=core.config.download_dir,
auto_managed=False))
except RuntimeError:
if file is not None:
file.fatal('torrent already exists')
else:
log.warning('torrent already exists')
raise gevent.GreenletExit()
try:
name = t.name
for package in core.packages():
if package.system == 'torrent' and package.name == name:
return
links = list()
#rename = list()
for i, f in enumerate(t.files):
fname = convert_torrent_name(name, f['path'])
#if fname != f['path']:
# rename.append((i, fname))
link = {}
link['url'] = 'torrent:///?{}#{}'.format(urllib.quote(t.id), i)
link['name'] = fname
link['size'] = f['size']
link['state'] = 'collect'
links.append(link)
#if rename:
# t.rename_files(rename)
t.save_torrent()
ids = core.add_links(links,
package_name=name,
package_id=t.id,
system='torrent')
for file in core.files():
if file.id in ids:
event.fire('file:checked', file)
finally:
t.remove()
def _upload(thread, queue, errors):
if verbose:
print 'Thread %s: start' % thread
while True:
try:
file = queue.get_nowait()
except gevent.queue.Empty:
if verbose:
if verbose > 1:
print 'Thread %3s: queue empty' % thread
print 'Thread %3s: exiting' % thread
raise gevent.GreenletExit()
else:
with open(file['path'], 'rb') as f:
body = f.read()
if verbose > 1:
print 'Thread %3s: uploading %s' % (thread,
file['name'])
try:
r = s.put('%s/%s/%s' %
(self.endpoint, container, file['name']),
data=body, headers={
'X-Auth-Token': self.token
})
except:
e = sys.exc_info()[1]
errors.append({
'name': file['name'],
'container': container,
'exception': str(e)
})
else:
if r.status_code == 401:
raise AuthenticationError
if r.status_code != 201:
errors.append({
'name': file['name'],
'container': container,
'status_code': r.status_code,
'headers': r.headers,
'response': json.loads(r.text)
})
finally:
if verbose > 1:
print ('Thread %3s: upload complete for %s'
% (thread, file['name']))
def test_thread_error_greenletexit(self, mock_closed, mock_close):
thread = mock.Mock(**{
'successful.return_value': False,
'exception': gevent.GreenletExit(),
})
tend = tcp.TCPTendril('manager', self.sock)
tend._send_thread = mock.Mock()
tend._recv_thread = mock.Mock()
tend._thread_error(thread)
mock_close.assert_called_once_with()
self.assertFalse(mock_closed.called)
# Ensure we didn't overwrite the threads
self.assertNotEqual(tend._send_thread, None)
self.assertNotEqual(tend._recv_thread, None)
def _recv(self):
"""
Implementation of the receive thread. Waits for data to
arrive on the socket, then passes the data through the defined
receive framer and sends it on to the application.
"""
# Outer loop: receive some data
while True:
# Wait until we can go
self._recv_lock.release()
gevent.sleep() # Yield to another thread
self._recv_lock.acquire()
recv_buf = self._sock.recv(self.recv_bufsize)
# If it's empty, the peer closed the other end
if not recv_buf:
# Manually kill the send thread; do this manually
# instead of calling close() because close() will kill
# us, and since close() would be running in our thread
# context, it would never get around to killing the
# send thread
if self._send_thread:
self._send_thread.kill()
self._send_thread = None
# Manually close the socket
self._sock.close()
self._sock = None
# Make sure the manager knows we're closed
super(TCPTendril, self).close()
# Notify the application
self.closed()
# As our last step, commit seppuku; this will keep
# _thread_error() from notifying the application of an
# erroneous exit from the receive thread
raise gevent.GreenletExit()
# Process the received data
self._recv_frameify(recv_buf)
def ask_buy_premium_dialog(hoster, url):
with premium_lock:
if hoster in config.ignore_ask_premium:
return
try:
elements = [
input.Text("Buy premium account for hoster {}?".format(hoster)),
input.Choice('answer', choices=[
{"value": 'yes', "content": "Yes", "link": url},
{"value": 'no', "content": "No"},
{"value": 'already', "content": "I already have an account"},
{"value": 'never', "content": "No and never ask again"}
]),
]
result = input.get(elements)
if result['answer'] == 'already':
elements = [
input.Text('Please enter your account details for {}'.format(hoster)),
input.Float('left'),
[input.Text('Username:'******'username')],
[input.Text('Password:'******'password', 'password')],
input.Float('center'),
input.Text(''),
input.Choice('answer', choices=[
{"value": 'add', "content": "Add Account", 'ok': True},
{"value": 'cancel', "content": "Cancel", 'cancel': True}
])
]
result = input.get(elements, close_aborts=True)
if result['answer'] == 'add' and result['username'] and result['password']:
interface.call('account', 'add', name=hoster, username=result['username'], password=result['password'])
raise gevent.GreenletExit()
elif result['answer'] == 'never':
if hoster not in config.ignore_ask_premium:
with transaction:
config.ignore_ask_premium.append(hoster)
except KeyError:
pass
except input.InputTimeout:
log.warning('input timed out')
except input.InputError:
log.error('input was aborted')
class TestUDPTendrilManager(unittest.TestCase):
@mock.patch.object(manager.TendrilManager, '__init__')
def test_init(self, mock_init):
manager = udp.UDPTendrilManager()
mock_init.assert_called_once_with(None)
self.assertEqual(manager._sock, None)
self.assertIsInstance(manager._sock_event, event.Event)
self.assertEqual(manager._sock_event.is_set(), False)
@mock.patch.object(manager.TendrilManager, 'start')
def test_start(self, mock_start):
manager = udp.UDPTendrilManager()
manager._sock = 'sock'
manager._sock_event = mock.Mock()
manager.start('acceptor', 'wrapper')
mock_start.assert_called_once_with('acceptor', 'wrapper')
self.assertEqual(manager._sock, None)
manager._sock_event.clear.assert_called_once_with()
@mock.patch.object(manager.TendrilManager, 'stop')
def test_stop(self, mock_stop):
manager = udp.UDPTendrilManager()
manager._sock = 'sock'
manager._sock_event = mock.Mock()
manager.stop('thread')
mock_stop.assert_called_once_with('thread')
self.assertEqual(manager._sock, None)
manager._sock_event.clear.assert_called_once_with()
@mock.patch.object(manager.TendrilManager, 'shutdown')
def test_shutdown(self, mock_shutdown):
manager = udp.UDPTendrilManager()
manager._sock = 'sock'
manager._sock_event = mock.Mock()
manager.shutdown()
mock_shutdown.assert_called_once_with()
self.assertEqual(manager._sock, None)
manager._sock_event.clear.assert_called_once_with()
@mock.patch.object(manager.TendrilManager, 'connect')
@mock.patch.object(manager.TendrilManager, '_track_tendril')
@mock.patch.object(udp, 'UDPTendril', return_value=mock.Mock())
@mock.patch.object(udp.UDPTendrilManager, 'local_addr', ('0.0.0.0', 8880))
def test_connect(self, mock_UDPTendril, mock_track_tendril, mock_connect):
acceptor = mock.Mock()
manager = udp.UDPTendrilManager()
tend = manager.connect(('127.0.0.1', 8080), acceptor)
mock_connect.assert_called_once_with(('127.0.0.1', 8080), acceptor,
None)
mock_UDPTendril.assert_called_once_with(manager, ('0.0.0.0', 8880),
('127.0.0.1', 8080))
acceptor.assert_called_once_with(mock_UDPTendril.return_value)
mock_track_tendril.assert_called_once_with(
mock_UDPTendril.return_value)
self.assertEqual(id(tend), id(mock_UDPTendril.return_value))
@mock.patch.object(manager.TendrilManager, 'connect')
@mock.patch.object(manager.TendrilManager, '_track_tendril')
@mock.patch.object(udp, 'UDPTendril', return_value=mock.Mock())
@mock.patch.object(udp.UDPTendrilManager, 'local_addr', ('0.0.0.0', 8880))
def test_connect_rejected(self, mock_UDPTendril, mock_track_tendril,
mock_connect):
acceptor = mock.Mock(side_effect=application.RejectConnection())
manager = udp.UDPTendrilManager()
tend = manager.connect(('127.0.0.1', 8080), acceptor)
mock_connect.assert_called_once_with(('127.0.0.1', 8080), acceptor,
None)
mock_UDPTendril.assert_called_once_with(manager, ('0.0.0.0', 8880),
('127.0.0.1', 8080))
acceptor.assert_called_once_with(mock_UDPTendril.return_value)
self.assertFalse(mock_track_tendril.called)
self.assertEqual(tend, None)
@mock.patch.object(socket,
'socket',
return_value=mock.Mock(
**{
'recvfrom.side_effect': TestException(),
'getsockname.return_value': ('127.0.0.1', 8080),
}))
@mock.patch.object(manager.TendrilManager, '_track_tendril')
@mock.patch.object(udp, 'UDPTendril', return_value=mock.Mock())
def test_listener_creator(self, mock_UDPTendril, mock_track_tendril,
mock_socket):
acceptor = mock.Mock()
manager = udp.UDPTendrilManager()
manager.running = True
with self.assertRaises(TestException):
manager.listener(acceptor, None)
mock_socket.assert_called_once_with(socket.AF_INET, socket.SOCK_DGRAM)
mock_socket.return_value.assert_has_calls([
mock.call.bind(('', 0)),
mock.call.getsockname(),
mock.call.recvfrom(4096),
mock.call.recvfrom(4096),
mock.call.recvfrom(4096),
mock.call.recvfrom(4096),
mock.call.recvfrom(4096),
mock.call.recvfrom(4096),
mock.call.recvfrom(4096),
mock.call.recvfrom(4096),
mock.call.recvfrom(4096),
mock.call.recvfrom(4096),
mock.call.recvfrom(4096),
mock.call.close(),
])
self.assertEqual(manager.local_addr, ('127.0.0.1', 8080))
self.assertFalse(mock_UDPTendril.called)
self.assertFalse(acceptor.called)
self.assertFalse(mock_track_tendril.called)
@mock.patch.object(socket,
'socket',
return_value=mock.Mock(
**{
'recvfrom.side_effect': TestException(),
'getsockname.return_value': ('127.0.0.1', 8080),
}))
@mock.patch.object(manager.TendrilManager, '_track_tendril')
@mock.patch.object(udp, 'UDPTendril', return_value=mock.Mock())
@mock.patch.object(udp.UDPTendrilManager, 'recv_bufsize', 8192)
def test_listener_creator_recv_bufsize(self, mock_UDPTendril,
mock_track_tendril, mock_socket):
acceptor = mock.Mock()
manager = udp.UDPTendrilManager()
manager.running = True
with self.assertRaises(TestException):
manager.listener(acceptor, None)
mock_socket.assert_called_once_with(socket.AF_INET, socket.SOCK_DGRAM)
mock_socket.return_value.assert_has_calls([
mock.call.bind(('', 0)),
mock.call.getsockname(),
mock.call.recvfrom(8192),
mock.call.recvfrom(8192),
mock.call.recvfrom(8192),
mock.call.recvfrom(8192),
mock.call.recvfrom(8192),
mock.call.recvfrom(8192),
mock.call.recvfrom(8192),
mock.call.recvfrom(8192),
mock.call.recvfrom(8192),
mock.call.recvfrom(8192),
mock.call.recvfrom(8192),
mock.call.close(),
])
self.assertEqual(manager.local_addr, ('127.0.0.1', 8080))
self.assertFalse(mock_UDPTendril.called)
self.assertFalse(acceptor.called)
self.assertFalse(mock_track_tendril.called)
@mock.patch.object(socket,
'socket',
return_value=mock.Mock(
**{
'recvfrom.side_effect': gevent.GreenletExit(),
'getsockname.return_value': ('127.0.0.1', 8080),
'close.side_effect': socket.error(),
}))
@mock.patch.object(manager.TendrilManager, '_track_tendril')
@mock.patch.object(udp, 'UDPTendril', return_value=mock.Mock())
def test_listener_killed(self, mock_UDPTendril, mock_track_tendril,
mock_socket):
acceptor = mock.Mock()
manager = udp.UDPTendrilManager()
manager.running = True
with self.assertRaises(gevent.GreenletExit):
manager.listener(acceptor, None)
mock_socket.assert_called_once_with(socket.AF_INET, socket.SOCK_DGRAM)
mock_socket.return_value.assert_has_calls([
mock.call.bind(('', 0)),
mock.call.getsockname(),
mock.call.recvfrom(4096),
mock.call.close(),
])
self.assertEqual(manager.local_addr, ('127.0.0.1', 8080))
self.assertFalse(mock_UDPTendril.called)
self.assertFalse(acceptor.called)
self.assertFalse(mock_track_tendril.called)
@mock.patch.object(socket,
'socket',
return_value=mock.Mock(
**{
'recvfrom.side_effect': TestException(),
'getsockname.return_value': ('127.0.0.1', 8080),
}))
@mock.patch.object(manager.TendrilManager, '_track_tendril')
@mock.patch.object(udp, 'UDPTendril', return_value=mock.Mock())
def test_listener_wrapper(self, mock_UDPTendril, mock_track_tendril,
mock_socket):
wrapped_sock = mock.Mock(**{
'recvfrom.side_effect': TestException(),
})
wrapper = mock.Mock(return_value=wrapped_sock)
acceptor = mock.Mock()
manager = udp.UDPTendrilManager()
manager.running = True
with self.assertRaises(TestException):
manager.listener(acceptor, wrapper)
mock_socket.assert_called_once_with(socket.AF_INET, socket.SOCK_DGRAM)
mock_socket.return_value.assert_has_calls([
mock.call.bind(('', 0)),
mock.call.getsockname(),
])
wrapper.assert_called_once_with(mock_socket.return_value)
wrapped_sock.assert_has_calls([
mock.call.recvfrom(4096),
mock.call.recvfrom(4096),
mock.call.recvfrom(4096),
mock.call.recvfrom(4096),
mock.call.recvfrom(4096),
mock.call.recvfrom(4096),
mock.call.recvfrom(4096),
mock.call.recvfrom(4096),
mock.call.recvfrom(4096),
mock.call.recvfrom(4096),
mock.call.recvfrom(4096),
mock.call.close(),
])
self.assertEqual(manager.local_addr, ('127.0.0.1', 8080))
self.assertFalse(mock_UDPTendril.called)
self.assertFalse(acceptor.called)
self.assertFalse(mock_track_tendril.called)
@mock.patch.object(socket,
'socket',
return_value=mock.Mock(**{
'getsockname.return_value': ('127.0.0.1', 8080),
}))
@mock.patch.object(manager.TendrilManager, '_track_tendril')
@mock.patch.object(udp, 'UDPTendril', return_value=mock.Mock())
def test_listener_noacceptor(self, mock_UDPTendril, mock_track_tendril,
mock_socket):
msgs = ['msg1', 'msg2', 'msg3']
mock_socket.return_value.recvfrom.side_effect = [
(msgs[0], ('127.0.0.2', 8082)),
(msgs[1], ('127.0.0.3', 8083)),
(msgs[2], ('127.0.0.4', 8084)),
TestException(),
TestException(),
TestException(),
TestException(),
TestException(),
TestException(),
TestException(),
TestException(),
TestException(),
TestException(),
TestException(),
]
tend = mock.Mock()
manager = udp.UDPTendrilManager()
manager.running = True
manager.tendrils[(('127.0.0.1', 8080), ('127.0.0.3', 8083))] = tend
with self.assertRaises(TestException):
manager.listener(None, None)
mock_socket.assert_called_once_with(socket.AF_INET, socket.SOCK_DGRAM)
mock_socket.return_value.assert_has_calls([
mock.call.bind(('', 0)),
mock.call.getsockname(),
mock.call.recvfrom(4096),
mock.call.recvfrom(4096),
mock.call.recvfrom(4096),
mock.call.recvfrom(4096),
mock.call.recvfrom(4096),
mock.call.recvfrom(4096),
mock.call.recvfrom(4096),
mock.call.recvfrom(4096),
mock.call.recvfrom(4096),
mock.call.recvfrom(4096),
mock.call.recvfrom(4096),
mock.call.recvfrom(4096),
mock.call.recvfrom(4096),
mock.call.recvfrom(4096),
mock.call.close(),
])
self.assertEqual(manager.local_addr, ('127.0.0.1', 8080))
self.assertFalse(mock_UDPTendril.called)
self.assertFalse(mock_track_tendril.called)
tend._recv_frameify.assert_called_once_with('msg2')
@mock.patch.object(socket,
'socket',
return_value=mock.Mock(**{
'getsockname.return_value': ('127.0.0.1', 8080),
}))
@mock.patch.object(manager.TendrilManager, '_track_tendril')
@mock.patch.object(udp, 'UDPTendril', return_value=mock.Mock())
def test_listener_recv_frameify_error(self, mock_UDPTendril,
mock_track_tendril, mock_socket):
mock_socket.return_value.recvfrom.side_effect = [
('frame', ('127.0.0.3', 8083)),
TestException(),
TestException(),
TestException(),
TestException(),
TestException(),
TestException(),
TestException(),
TestException(),
TestException(),
TestException(),
TestException(),
]
tend = mock.Mock(**{'_recv_frameify.side_effect': TestException()})
manager = udp.UDPTendrilManager()
manager.running = True
manager.tendrils[(('127.0.0.1', 8080), ('127.0.0.3', 8083))] = tend
with self.assertRaises(TestException):
manager.listener(None, None)
mock_socket.assert_called_once_with(socket.AF_INET, socket.SOCK_DGRAM)
mock_socket.return_value.assert_has_calls([
mock.call.bind(('', 0)),
mock.call.getsockname(),
mock.call.recvfrom(4096),
mock.call.recvfrom(4096),
mock.call.recvfrom(4096),
mock.call.recvfrom(4096),
mock.call.recvfrom(4096),
mock.call.recvfrom(4096),
mock.call.recvfrom(4096),
mock.call.recvfrom(4096),
mock.call.recvfrom(4096),
mock.call.recvfrom(4096),
mock.call.recvfrom(4096),
mock.call.recvfrom(4096),
mock.call.close(),
])
self.assertEqual(manager.local_addr, ('127.0.0.1', 8080))
self.assertFalse(mock_UDPTendril.called)
self.assertFalse(mock_track_tendril.called)
tend._recv_frameify.assert_called_once_with('frame')
tend.close.assert_called_once_with()
self.assertEqual(tend.closed.call_count, 1)
args = tend.closed.call_args[0]
self.assertEqual(len(args), 1)
self.assertIsInstance(args[0], TestException)
@mock.patch.object(socket,
'socket',
return_value=mock.Mock(**{
'getsockname.return_value': ('127.0.0.1', 8080),
}))
@mock.patch.object(manager.TendrilManager, '_track_tendril')
@mock.patch.object(udp, 'UDPTendril', return_value=mock.Mock())
def test_listener_acceptor(self, mock_UDPTendril, mock_track_tendril,
mock_socket):
msgs = ['msg1', 'msg2', 'msg3']
mock_socket.return_value.recvfrom.side_effect = [
(msgs[0], ('127.0.0.2', 8082)),
(msgs[1], ('127.0.0.3', 8083)),
(msgs[2], ('127.0.0.4', 8084)),
TestException(),
TestException(),
TestException(),
TestException(),
TestException(),
TestException(),
TestException(),
TestException(),
TestException(),
TestException(),
TestException(),
]
tendrils = [mock.Mock(), mock.Mock(), mock.Mock()]
mock_UDPTendril.side_effect = [tendrils[0], tendrils[2]]
acceptor = mock.Mock()
manager = udp.UDPTendrilManager()
manager.running = True
manager.tendrils[(('127.0.0.1', 8080), ('127.0.0.3',
8083))] = tendrils[1]
with self.assertRaises(TestException):
manager.listener(acceptor, None)
mock_socket.assert_called_once_with(socket.AF_INET, socket.SOCK_DGRAM)
mock_socket.return_value.assert_has_calls([
mock.call.bind(('', 0)),
mock.call.getsockname(),
mock.call.recvfrom(4096),
mock.call.recvfrom(4096),
mock.call.recvfrom(4096),
mock.call.recvfrom(4096),
mock.call.recvfrom(4096),
mock.call.recvfrom(4096),
mock.call.recvfrom(4096),
mock.call.recvfrom(4096),
mock.call.recvfrom(4096),
mock.call.recvfrom(4096),
mock.call.recvfrom(4096),
mock.call.recvfrom(4096),
mock.call.recvfrom(4096),
mock.call.recvfrom(4096),
mock.call.close(),
])
self.assertEqual(manager.local_addr, ('127.0.0.1', 8080))
mock_UDPTendril.assert_has_calls([
mock.call(manager, ('127.0.0.1', 8080), ('127.0.0.2', 8082)),
mock.call(manager, ('127.0.0.1', 8080), ('127.0.0.4', 8084)),
])
acceptor.assert_has_calls([
mock.call(tendrils[0]),
mock.call(tendrils[2]),
])
mock_track_tendril.assert_has_calls([
mock.call(tendrils[0]),
mock.call(tendrils[2]),
])
tendrils[0]._recv_frameify.assert_called_once_with('msg1')
tendrils[1]._recv_frameify.assert_called_once_with('msg2')
tendrils[2]._recv_frameify.assert_called_once_with('msg3')
@mock.patch.object(socket,
'socket',
return_value=mock.Mock(**{
'getsockname.return_value': ('127.0.0.1', 8080),
}))
@mock.patch.object(manager.TendrilManager, '_track_tendril')
@mock.patch.object(udp, 'UDPTendril', return_value=mock.Mock())
def test_listener_rejector(self, mock_UDPTendril, mock_track_tendril,
mock_socket):
mock_socket.return_value.recvfrom.side_effect = [
('msg1', ('127.0.0.2', 8082)),
('msg2', ('127.0.0.3', 8083)),
('msg3', ('127.0.0.4', 8084)),
TestException(),
TestException(),
TestException(),
TestException(),
TestException(),
TestException(),
TestException(),
TestException(),
TestException(),
TestException(),
TestException(),
]
tendrils = [mock.Mock(), mock.Mock(), mock.Mock()]
mock_UDPTendril.side_effect = tendrils[:]
acceptor = mock.Mock(side_effect=application.RejectConnection())
manager = udp.UDPTendrilManager()
manager.running = True
with self.assertRaises(TestException):
manager.listener(acceptor, None)
mock_socket.assert_called_once_with(socket.AF_INET, socket.SOCK_DGRAM)
mock_socket.return_value.assert_has_calls([
mock.call.bind(('', 0)),
mock.call.getsockname(),
mock.call.recvfrom(4096),
mock.call.recvfrom(4096),
mock.call.recvfrom(4096),
mock.call.recvfrom(4096),
mock.call.recvfrom(4096),
mock.call.recvfrom(4096),
mock.call.recvfrom(4096),
mock.call.recvfrom(4096),
mock.call.recvfrom(4096),
mock.call.recvfrom(4096),
mock.call.recvfrom(4096),
mock.call.recvfrom(4096),
mock.call.recvfrom(4096),
mock.call.recvfrom(4096),
mock.call.close(),
])
self.assertEqual(manager.local_addr, ('127.0.0.1', 8080))
mock_UDPTendril.assert_has_calls([
mock.call(manager, ('127.0.0.1', 8080), ('127.0.0.2', 8082)),
mock.call(manager, ('127.0.0.1', 8080), ('127.0.0.3', 8083)),
mock.call(manager, ('127.0.0.1', 8080), ('127.0.0.4', 8084)),
])
acceptor.assert_has_calls([
mock.call(tendrils[0]),
mock.call(tendrils[1]),
mock.call(tendrils[2]),
])
self.assertFalse(mock_track_tendril.called)
self.assertFalse(tendrils[0]._recv_frameify.called)
self.assertFalse(tendrils[1]._recv_frameify.called)
self.assertFalse(tendrils[2]._recv_frameify.called)
def test_sock_getter_notrunning(self):
manager = udp.UDPTendrilManager()
manager._sock = 'socket'
manager._sock_event = mock.Mock()
result = manager.sock
self.assertEqual(result, None)
self.assertFalse(manager._sock_event.wait.called)
def test_sock_getter(self):
manager = udp.UDPTendrilManager()
manager.running = True
manager._sock = 'socket'
manager._sock_event = mock.Mock()
result = manager.sock
self.assertEqual(result, 'socket')
manager._sock_event.wait.assert_called_once_with()
class Crawler(object):
"""Simple crawler based on gcrawler, but with a rolling inq that can be
added to. The crawler is done when the workers have no more jobs and
there are no more urls in the queue."""
seen_jobs = set()
def __init__(self, spider, timeout=2, worker_count=4, pipeline_size=100):
self.spider = spider
self.spider.crawler = self
self.timeout = timeout
self.count = worker_count
self.inq = queue.Queue(0)
self.outq = queue.Queue(pipeline_size)
self.jobq = queue.Queue()
self.pool = pool.Pool(worker_count)
self.worker_finished = event.Event()
for job in getattr(self.spider, 'jobs', []):
self.add_job(job)
self.cache = '.cache'
def start(self):
"""Start the crawler. Starts the scheduler and pipeline first, then
adds jobs to the pool and waits for the scheduler and pipeline to
finish. The scheduler itself shuts down the pool and waits on that,
so it's not required to watch the pool."""
# add the spider's jobs first
for url in self.spider.urls:
self.add_job(url)
# start the scheduler & the pipeline
self.scheduler_greenlet = gevent.spawn(self.scheduler)
self.pipeline_greenlet = gevent.spawn(self.pipeline)
self.scheduler_greenlet.join()
def add_job(self, job, **kwargs):
"""Add a job to the queue. The job argument can either be a Job object
or a url with keyword arguments to be passed to the Job constructor."""
if isinstance(job, basestring):
job = Job(job, **kwargs)
# do not visit previously viewed urls
if job in self.seen_jobs:
return
self.jobq.put(job)
def scheduler(self):
"""A simple job scheduler. Presuming it's started after there is at
least one job, it feeds jobs to the job queue to a synchronous worker
job queue one at a time. When the worker queue fills up, the
scheduler will block on the put() operation. When job queue is empty,
the scheduler will wait for the workers to finish. If the job queue
is empty and no workers are active, the pool's stopped."""
logger = logging.getLogger(__name__ + '.scheduler')
while True:
# join dead greenlets
for greenlet in list(self.pool):
if greenlet.dead:
self.pool.discard(greenlet)
try:
logger.debug("Fetching job from job queue.")
job = self.jobq.get_nowait()
except queue.Empty:
logger.debug("No jobs remaining.")
if self.pool.free_count() != self.pool.size:
logger.debug("%d workers remaining, waiting..." %
(self.pool.size - self.pool.free_count()))
self.worker_finished.wait()
self.worker_finished.clear()
continue
else:
logger.debug("No workers left, shutting down.")
return self.shutdown()
self.pool.spawn(self.worker, job)
def shutdown(self):
"""Shutdown the crawler after the pool has finished."""
self.pool.join()
self.outq.put(StopIteration)
self.pipeline_greenlet.join()
return True
def worker(self, job, logger=logging.getLogger(__name__ + '.worker')):
"""A simple worker that fetches urls based on jobs and puts the
ammended jobs on the outq for processing in the pipeline thread.
After each job is fetched, but before the worker sets the finished
event, the spider's preprocess method will be called on the job. This
is its opportunity to add urls to the job queue. Heavy processing
should be done via the pipeline in postprocess."""
logger.debug("starting: %r" % job)
# you need to create a new Http instance per greenlet, see ticket:
# http://code.google.com/p/httplib2/issues/detail?id=5
h = httplib2.Http(self.cache)
try:
job.response, job.data = h.request(job.url,
method=job.method,
headers=job.headers)
self.spider.preprocess(job)
except Exception, e:
logger.error("Preprocessing error:\n%s" % traceback.format_exc())
self.outq.put(job)
self.worker_finished.set()
logger.debug("finished: %r" % job)
raise gevent.GreenletExit('success')
def fatal(self, msg):
with transaction:
self.last_error = msg
self.enabled = False
self.log.error(msg)
raise gevent.GreenletExit()
def test_00_init(self, mock_SimpleConsumer, mock_KafkaClient,
mock_UVEServer):
self.test_spec = [
TestStage(i=PartHandlerInput(
redis_instances=set([("127.0.0.1", 44444, 0)]),
get_part={
"127.0.0.1:44444": {
"gen1": {
"ObjectXX:uve1": set(["type1"])
}
}
},
get_messages=[
OffsetAndMessage(
offset=0,
message=Message(
magic=0,
attributes=0,
key='',
value=(
'{"message":"UVEUpdate","key":"ObjectYY:uve2",'
'"type":"type2","gen":"gen1","coll":'
'"127.0.0.1:44444","deleted":false}')))
]),
o=PartHandlerOutput(callbacks=[
{
"ObjectXX:uve1": None
},
{
"ObjectYY:uve2": set(["type2"])
},
],
uvedb=None)),
TestStage(
i=PartHandlerInput(redis_instances=gevent.GreenletExit(),
get_part=None,
get_messages=None),
o=PartHandlerOutput(callbacks=[
{
"ObjectXX:uve1": None,
"ObjectYY:uve2": None
},
],
uvedb={
"127.0.0.1:44444": {
"gen1": {
"ObjectXX:uve1":
set(["type1"]),
"ObjectYY:uve2": set(["type2"])
}
}
}),
)
]
mock_UVEServer.return_value.redis_instances.side_effect = \
[x.i.redis_instances for x in self.test_spec]
mock_UVEServer.return_value.get_part.side_effect = \
[x.i.get_part for x in self.test_spec if x.i.get_part is not None]
mock_SimpleConsumer.return_value.get_messages.side_effect = \
[x.i.get_messages for x in self.test_spec]
self.ph = UveStreamProc('no-brokers', 1, "uve-1", logging,
self.callback_proc, "127.0.0.1",
mock_UVEServer.return_value)
self.ph.start()
res, db = self.ph.get(timeout=10)
if (isinstance(res, AssertionError)):
raise res
self.assertEqual(db, self.test_spec[-1].o.uvedb)
def run_step(self, step_config):
"""Runs a step.
Args:
step_config (StepConfig): The step configuration to run.
Returns:
A StepData object containing the result of the finished step.
"""
# TODO(iannucci): When subannotations are handled with annotee, move
# `allow_subannotations` into recipe_module/step.
name_tokens = self._record_step_name(step_config.name)
# TODO(iannucci): Start with had_exception=True and overwrite when we know
# we DIDN'T have an exception.
ret = StepData(name_tokens, ExecutionResult())
try:
self._step_runner.register_step_config(name_tokens, step_config)
except:
# Test data functions are not allowed to raise exceptions. Instead of
# letting user code catch these, we crash the test immediately.
_log_crash(self._stream_engine,
"register_step_config(%r)" % (ret.name, ))
raise CrashEngine("Registering step_config failed for %r." %
(ret.name))
step_stream = self._stream_engine.new_step_stream(
name_tokens, step_config.allow_subannotations)
debug_log = step_stream.new_log_stream('$debug')
caught = None
try:
# If there's a parent step on the stack, add `ret` to its children.
self._step_stack[-1].children_steps.append(ret)
# initialize presentation to show an exception.
ret.presentation = StepPresentation(step_config.name)
ret.presentation.status = 'EXCEPTION'
self._step_stack.append(_ActiveStep(ret, step_stream, False))
# _run_step should never raise an exception, except for GreenletExit
caught = _run_step(debug_log, ret, step_stream, self._step_runner,
self._resource, step_config, self._environ,
self._start_dir)
# TODO(iannucci): remove this trigger specs crap
if step_config.trigger_specs:
_trigger_builds(step_stream, step_config.trigger_specs)
# NOTE: See the accompanying note in stream.py.
step_stream.reset_subannotation_state()
ret.finalize()
# If there's a buffered exception, we raise it now.
if caught:
# TODO(iannucci): Python3 incompatible.
raise caught[0], caught[1], caught[2]
# If the step was cancelled, raise GreenletExit.
if ret.exc_result.was_cancelled:
raise gevent.GreenletExit()
if ret.presentation.status == 'SUCCESS':
return ret
# Otherwise, we raise an appropriate error based on
# ret.presentation.status
exc = {
'FAILURE': recipe_api.StepFailure,
'WARNING': recipe_api.StepWarning,
'EXCEPTION': recipe_api.InfraFailure,
}[ret.presentation.status]
# TODO(iannucci): Use '|' instead of '.'
raise exc('.'.join(name_tokens), ret)
finally:
# per sys.exc_info this is recommended in python 2.x to avoid creating
# garbage cycles.
del caught
debug_log.close()
def spawn_to_kill() -> None:
gevent.spawn(greenlet.throw, gevent.GreenletExit())
class TestTCPTendrilManager(unittest.TestCase):
@mock.patch.object(socket, 'socket', return_value=mock.Mock())
@mock.patch.object(manager.TendrilManager, 'connect')
@mock.patch.object(manager.TendrilManager, '_track_tendril')
@mock.patch.object(tcp, 'TCPTendril', return_value=mock.Mock())
def test_connect_basic(self, mock_TCPTendril, mock_track_tendril,
mock_connect, mock_socket):
acceptor = mock.Mock()
manager = tcp.TCPTendrilManager()
tend = manager.connect(('127.0.0.1', 8080), acceptor)
mock_connect.assert_called_once_with(('127.0.0.1', 8080), acceptor,
None)
mock_socket.assert_called_once_with(socket.AF_INET, socket.SOCK_STREAM)
mock_socket.return_value.assert_has_calls([
mock.call.bind(('', 0)),
mock.call.connect(('127.0.0.1', 8080)),
])
mock_TCPTendril.assert_called_once_with(manager,
mock_socket.return_value)
acceptor.assert_called_once_with(mock_TCPTendril.return_value)
mock_track_tendril.assert_called_once_with(
mock_TCPTendril.return_value)
mock_TCPTendril.return_value._start.assert_called_once_with()
self.assertEqual(id(tend), id(mock_TCPTendril.return_value))
@mock.patch.object(socket, 'socket', return_value=mock.Mock())
@mock.patch.object(manager.TendrilManager, 'connect')
@mock.patch.object(manager.TendrilManager, '_track_tendril')
@mock.patch.object(tcp, 'TCPTendril', return_value=mock.Mock())
def test_connect_wrapper(self, mock_TCPTendril, mock_track_tendril,
mock_connect, mock_socket):
wrapped_sock = mock.Mock()
wrapper = mock.Mock(return_value=wrapped_sock)
acceptor = mock.Mock()
manager = tcp.TCPTendrilManager(('127.0.0.2', 8880))
tend = manager.connect(('127.0.0.1', 8080), acceptor, wrapper)
mock_connect.assert_called_once_with(('127.0.0.1', 8080), acceptor,
wrapper)
mock_socket.assert_called_once_with(socket.AF_INET, socket.SOCK_STREAM)
mock_socket.return_value.assert_has_calls([
mock.call.bind(('127.0.0.2', 8880)),
mock.call.connect(('127.0.0.1', 8080)),
])
wrapper.assert_called_once_with(mock_socket.return_value)
mock_TCPTendril.assert_called_once_with(manager, wrapped_sock)
acceptor.assert_called_once_with(mock_TCPTendril.return_value)
mock_track_tendril.assert_called_once_with(
mock_TCPTendril.return_value)
mock_TCPTendril.return_value._start.assert_called_once_with()
self.assertEqual(id(tend), id(mock_TCPTendril.return_value))
@mock.patch.object(socket, 'socket', return_value=mock.Mock())
@mock.patch.object(manager.TendrilManager, 'connect')
@mock.patch.object(manager.TendrilManager, '_track_tendril')
@mock.patch.object(tcp, 'TCPTendril', return_value=mock.Mock())
def test_connect_failure(self, mock_TCPTendril, mock_track_tendril,
mock_connect, mock_socket):
acceptor = mock.Mock(side_effect=TestException())
manager = tcp.TCPTendrilManager()
with self.assertRaises(TestException):
tend = manager.connect(('127.0.0.1', 8080), acceptor)
mock_connect.assert_called_once_with(('127.0.0.1', 8080), acceptor,
None)
mock_socket.assert_called_once_with(socket.AF_INET, socket.SOCK_STREAM)
mock_socket.return_value.assert_has_calls([
mock.call.bind(('', 0)),
mock.call.connect(('127.0.0.1', 8080)),
mock.call.close(),
])
mock_TCPTendril.assert_called_once_with(manager,
mock_socket.return_value)
acceptor.assert_called_once_with(mock_TCPTendril.return_value)
self.assertFalse(mock_track_tendril.called)
self.assertFalse(mock_TCPTendril.return_value._start.called)
@mock.patch.object(
socket,
'socket',
return_value=mock.Mock(**{
'close.side_effect': socket.error(),
}))
@mock.patch.object(manager.TendrilManager, 'connect')
@mock.patch.object(manager.TendrilManager, '_track_tendril')
@mock.patch.object(tcp, 'TCPTendril', return_value=mock.Mock())
def test_connect_failure_closeerror(self, mock_TCPTendril,
mock_track_tendril, mock_connect,
mock_socket):
acceptor = mock.Mock(side_effect=TestException())
manager = tcp.TCPTendrilManager()
with self.assertRaises(TestException):
tend = manager.connect(('127.0.0.1', 8080), acceptor)
mock_connect.assert_called_once_with(('127.0.0.1', 8080), acceptor,
None)
mock_socket.assert_called_once_with(socket.AF_INET, socket.SOCK_STREAM)
mock_socket.return_value.assert_has_calls([
mock.call.bind(('', 0)),
mock.call.connect(('127.0.0.1', 8080)),
mock.call.close(),
])
mock_TCPTendril.assert_called_once_with(manager,
mock_socket.return_value)
acceptor.assert_called_once_with(mock_TCPTendril.return_value)
self.assertFalse(mock_track_tendril.called)
self.assertFalse(mock_TCPTendril.return_value._start.called)
@mock.patch.object(socket, 'socket', return_value=mock.Mock())
@mock.patch.object(manager.TendrilManager, 'connect')
@mock.patch.object(manager.TendrilManager, '_track_tendril')
@mock.patch.object(tcp, 'TCPTendril', return_value=mock.Mock())
def test_connect_reject_connection(self, mock_TCPTendril,
mock_track_tendril, mock_connect,
mock_socket):
acceptor = mock.Mock(side_effect=application.RejectConnection)
manager = tcp.TCPTendrilManager()
tend = manager.connect(('127.0.0.1', 8080), acceptor)
self.assertEqual(tend, None)
mock_connect.assert_called_once_with(('127.0.0.1', 8080), acceptor,
None)
mock_socket.assert_called_once_with(socket.AF_INET, socket.SOCK_STREAM)
mock_socket.return_value.assert_has_calls([
mock.call.bind(('', 0)),
mock.call.connect(('127.0.0.1', 8080)),
mock.call.close(),
])
mock_TCPTendril.assert_called_once_with(manager,
mock_socket.return_value)
acceptor.assert_called_once_with(mock_TCPTendril.return_value)
self.assertFalse(mock_track_tendril.called)
self.assertFalse(mock_TCPTendril.return_value._start.called)
@mock.patch.object(gevent, 'sleep', side_effect=TestException())
@mock.patch.object(socket, 'socket', return_value=mock.Mock())
@mock.patch.object(manager.TendrilManager, '_track_tendril')
@mock.patch.object(tcp, 'TCPTendril', return_value=mock.Mock())
def test_listener_noacceptor(self, mock_TCPTendril, mock_track_tendril,
mock_socket, mock_sleep):
manager = tcp.TCPTendrilManager()
with self.assertRaises(TestException):
manager.listener(None, None)
mock_sleep.assert_called_once_with(600)
self.assertFalse(mock_socket.called)
@mock.patch.object(gevent, 'sleep', side_effect=TestException())
@mock.patch.object(socket,
'socket',
return_value=mock.Mock(
**{
'accept.side_effect': TestException(),
'getsockname.return_value': ('127.0.0.1', 8080),
}))
@mock.patch.object(manager.TendrilManager, '_track_tendril')
@mock.patch.object(tcp, 'TCPTendril', return_value=mock.Mock())
def test_listener_creator(self, mock_TCPTendril, mock_track_tendril,
mock_socket, mock_sleep):
acceptor = mock.Mock()
manager = tcp.TCPTendrilManager()
manager.running = True
with self.assertRaises(TestException):
manager.listener(acceptor, None)
self.assertFalse(mock_sleep.called)
mock_socket.assert_called_once_with(socket.AF_INET, socket.SOCK_STREAM)
mock_socket.return_value.assert_has_calls([
mock.call.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1),
mock.call.bind(('', 0)),
mock.call.getsockname(),
mock.call.listen(1024),
mock.call.accept(),
mock.call.accept(),
mock.call.accept(),
mock.call.accept(),
mock.call.accept(),
mock.call.accept(),
mock.call.accept(),
mock.call.accept(),
mock.call.accept(),
mock.call.accept(),
mock.call.accept(),
mock.call.close(),
])
self.assertEqual(manager.local_addr, ('127.0.0.1', 8080))
self.assertFalse(mock_TCPTendril.called)
self.assertFalse(acceptor.called)
self.assertFalse(mock_track_tendril.called)
@mock.patch.object(gevent, 'sleep', side_effect=TestException())
@mock.patch.object(socket,
'socket',
return_value=mock.Mock(
**{
'accept.side_effect': TestException(),
'getsockname.return_value': ('127.0.0.1', 8080),
}))
@mock.patch.object(manager.TendrilManager, '_track_tendril')
@mock.patch.object(tcp, 'TCPTendril', return_value=mock.Mock())
@mock.patch.object(tcp.TCPTendrilManager, 'backlog', 4096)
def test_listener_creator_backlog(self, mock_TCPTendril,
mock_track_tendril, mock_socket,
mock_sleep):
acceptor = mock.Mock()
manager = tcp.TCPTendrilManager()
manager.running = True
with self.assertRaises(TestException):
manager.listener(acceptor, None)
self.assertFalse(mock_sleep.called)
mock_socket.assert_called_once_with(socket.AF_INET, socket.SOCK_STREAM)
mock_socket.return_value.assert_has_calls([
mock.call.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1),
mock.call.bind(('', 0)),
mock.call.getsockname(),
mock.call.listen(4096),
mock.call.accept(),
mock.call.accept(),
mock.call.accept(),
mock.call.accept(),
mock.call.accept(),
mock.call.accept(),
mock.call.accept(),
mock.call.accept(),
mock.call.accept(),
mock.call.accept(),
mock.call.accept(),
mock.call.close(),
])
self.assertEqual(manager.local_addr, ('127.0.0.1', 8080))
self.assertFalse(mock_TCPTendril.called)
self.assertFalse(acceptor.called)
self.assertFalse(mock_track_tendril.called)
@mock.patch.object(gevent, 'sleep', side_effect=TestException())
@mock.patch.object(socket,
'socket',
return_value=mock.Mock(
**{
'accept.side_effect': gevent.GreenletExit(),
'getsockname.return_value': ('127.0.0.1', 8080),
'close.side_effect': socket.error(),
}))
@mock.patch.object(manager.TendrilManager, '_track_tendril')
@mock.patch.object(tcp, 'TCPTendril', return_value=mock.Mock())
def test_listener_killed(self, mock_TCPTendril, mock_track_tendril,
mock_socket, mock_sleep):
acceptor = mock.Mock()
manager = tcp.TCPTendrilManager()
manager.running = True
with self.assertRaises(gevent.GreenletExit):
manager.listener(acceptor, None)
self.assertFalse(mock_sleep.called)
mock_socket.assert_called_once_with(socket.AF_INET, socket.SOCK_STREAM)
mock_socket.return_value.assert_has_calls([
mock.call.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1),
mock.call.bind(('', 0)),
mock.call.getsockname(),
mock.call.listen(1024),
mock.call.accept(),
mock.call.close(),
])
self.assertEqual(manager.local_addr, ('127.0.0.1', 8080))
self.assertFalse(mock_TCPTendril.called)
self.assertFalse(acceptor.called)
self.assertFalse(mock_track_tendril.called)
@mock.patch.object(gevent, 'sleep', side_effect=TestException())
@mock.patch.object(socket,
'socket',
return_value=mock.Mock(
**{
'accept.side_effect': TestException(),
'getsockname.return_value': ('127.0.0.1', 8080),
}))
@mock.patch.object(manager.TendrilManager, '_track_tendril')
@mock.patch.object(tcp, 'TCPTendril', return_value=mock.Mock())
def test_listener_wrapper(self, mock_TCPTendril, mock_track_tendril,
mock_socket, mock_sleep):
wrapped_sock = mock.Mock(**{
'accept.side_effect': TestException(),
})
wrapper = mock.Mock(return_value=wrapped_sock)
acceptor = mock.Mock()
manager = tcp.TCPTendrilManager()
manager.running = True
with self.assertRaises(TestException):
manager.listener(acceptor, wrapper)
self.assertFalse(mock_sleep.called)
mock_socket.assert_called_once_with(socket.AF_INET, socket.SOCK_STREAM)
mock_socket.return_value.assert_has_calls([
mock.call.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1),
mock.call.bind(('', 0)),
mock.call.getsockname(),
])
wrapper.assert_called_once_with(mock_socket.return_value)
wrapped_sock.assert_has_calls([
mock.call.listen(1024),
mock.call.accept(),
mock.call.accept(),
mock.call.accept(),
mock.call.accept(),
mock.call.accept(),
mock.call.accept(),
mock.call.accept(),
mock.call.accept(),
mock.call.accept(),
mock.call.accept(),
mock.call.accept(),
mock.call.close(),
])
self.assertEqual(manager.local_addr, ('127.0.0.1', 8080))
self.assertFalse(mock_TCPTendril.called)
self.assertFalse(acceptor.called)
self.assertFalse(mock_track_tendril.called)
@mock.patch.object(gevent, 'sleep', side_effect=TestException())
@mock.patch.object(socket,
'socket',
return_value=mock.Mock(
**{
'accept.side_effect': TestException(),
'getsockname.return_value': ('127.0.0.1', 8080),
'listen.side_effect': TestException(),
}))
@mock.patch.object(manager.TendrilManager, '_track_tendril')
@mock.patch.object(tcp, 'TCPTendril', return_value=mock.Mock())
def test_listener_nolisten(self, mock_TCPTendril, mock_track_tendril,
mock_socket, mock_sleep):
acceptor = mock.Mock()
manager = tcp.TCPTendrilManager()
manager.running = True
with self.assertRaises(TestException):
manager.listener(acceptor, None)
self.assertFalse(mock_sleep.called)
mock_socket.assert_called_once_with(socket.AF_INET, socket.SOCK_STREAM)
mock_socket.return_value.assert_has_calls([
mock.call.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1),
mock.call.bind(('', 0)),
mock.call.getsockname(),
mock.call.listen(1024),
mock.call.close(),
])
self.assertEqual(manager.local_addr, ('127.0.0.1', 8080))
self.assertFalse(mock_TCPTendril.called)
self.assertFalse(acceptor.called)
self.assertFalse(mock_track_tendril.called)
@mock.patch.object(gevent, 'sleep', side_effect=TestException())
@mock.patch.object(socket,
'socket',
return_value=mock.Mock(**{
'getsockname.return_value': ('127.0.0.1', 8080),
}))
@mock.patch.object(manager.TendrilManager, '_track_tendril')
@mock.patch.object(tcp, 'TCPTendril')
def test_listener_acceptor(self, mock_TCPTendril, mock_track_tendril,
mock_socket, mock_sleep):
clis = [mock.Mock(), mock.Mock(), mock.Mock()]
mock_socket.return_value.accept.side_effect = [
(clis[0], ('127.0.0.2', 8082)),
(clis[1], ('127.0.0.3', 8083)),
(clis[2], ('127.0.0.4', 8084)),
TestException(),
TestException(),
TestException(),
TestException(),
TestException(),
TestException(),
TestException(),
TestException(),
TestException(),
TestException(),
TestException(),
]
tendrils = [mock.Mock(), mock.Mock(), mock.Mock()]
mock_TCPTendril.side_effect = tendrils[:]
acceptor = mock.Mock(side_effect=[
mock.Mock(),
TestException(),
mock.Mock(),
])
manager = tcp.TCPTendrilManager()
manager.running = True
with self.assertRaises(TestException):
manager.listener(acceptor, None)
self.assertFalse(mock_sleep.called)
mock_socket.assert_called_once_with(socket.AF_INET, socket.SOCK_STREAM)
mock_socket.return_value.assert_has_calls([
mock.call.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1),
mock.call.bind(('', 0)),
mock.call.getsockname(),
mock.call.listen(1024),
mock.call.accept(),
mock.call.accept(),
mock.call.accept(),
mock.call.accept(),
mock.call.accept(),
mock.call.accept(),
mock.call.accept(),
mock.call.accept(),
mock.call.accept(),
mock.call.accept(),
mock.call.accept(),
mock.call.accept(),
mock.call.accept(),
mock.call.accept(),
mock.call.close(),
])
self.assertEqual(manager.local_addr, ('127.0.0.1', 8080))
mock_TCPTendril.assert_has_calls([
mock.call(manager, clis[0], ('127.0.0.2', 8082)),
mock.call(manager, clis[1], ('127.0.0.3', 8083)),
mock.call(manager, clis[2], ('127.0.0.4', 8084)),
])
acceptor.assert_has_calls([
mock.call(tendrils[0]),
mock.call(tendrils[1]),
mock.call(tendrils[2]),
])
self.assertFalse(clis[0].close.called)
clis[1].close.assert_called_once_with()
self.assertFalse(clis[2].close.called)
mock_track_tendril.assert_has_calls([
mock.call(tendrils[0]),
mock.call(tendrils[2]),
])
tendrils[0]._start.assert_called_once_with()
self.assertFalse(tendrils[1]._start.called)
tendrils[2]._start.assert_called_once_with()
@mock.patch.object(gevent, 'sleep', side_effect=TestException())
@mock.patch.object(socket,
'socket',
return_value=mock.Mock(**{
'getsockname.return_value': ('127.0.0.1', 8080),
}))
@mock.patch.object(manager.TendrilManager, '_track_tendril')
@mock.patch.object(tcp, 'TCPTendril')
def test_listener_acceptor_reject_connection(self, mock_TCPTendril,
mock_track_tendril,
mock_socket, mock_sleep):
clis = [mock.Mock(), mock.Mock(), mock.Mock()]
mock_socket.return_value.accept.side_effect = [
(clis[0], ('127.0.0.2', 8082)),
(clis[1], ('127.0.0.3', 8083)),
(clis[2], ('127.0.0.4', 8084)),
TestException(),
TestException(),
TestException(),
TestException(),
TestException(),
TestException(),
TestException(),
TestException(),
TestException(),
TestException(),
TestException(),
]
tendrils = [mock.Mock(), mock.Mock(), mock.Mock()]
mock_TCPTendril.side_effect = tendrils[:]
acceptor = mock.Mock(side_effect=application.RejectConnection())
manager = tcp.TCPTendrilManager()
manager.running = True
with self.assertRaises(TestException):
manager.listener(acceptor, None)
self.assertFalse(mock_sleep.called)
mock_socket.assert_called_once_with(socket.AF_INET, socket.SOCK_STREAM)
mock_socket.return_value.assert_has_calls([
mock.call.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1),
mock.call.bind(('', 0)),
mock.call.getsockname(),
mock.call.listen(1024),
mock.call.accept(),
mock.call.accept(),
mock.call.accept(),
mock.call.accept(),
mock.call.accept(),
mock.call.accept(),
mock.call.accept(),
mock.call.accept(),
mock.call.accept(),
mock.call.accept(),
mock.call.accept(),
mock.call.accept(),
mock.call.accept(),
mock.call.accept(),
mock.call.close(),
])
self.assertEqual(manager.local_addr, ('127.0.0.1', 8080))
mock_TCPTendril.assert_has_calls([
mock.call(manager, clis[0], ('127.0.0.2', 8082)),
mock.call(manager, clis[1], ('127.0.0.3', 8083)),
mock.call(manager, clis[2], ('127.0.0.4', 8084)),
])
acceptor.assert_has_calls([
mock.call(tendrils[0]),
mock.call(tendrils[1]),
mock.call(tendrils[2]),
])
clis[0].close.assert_called_once_with()
clis[1].close.assert_called_once_with()
clis[2].close.assert_called_once_with()
self.assertFalse(mock_track_tendril.called)
self.assertFalse(tendrils[0]._start.called)
self.assertFalse(tendrils[1]._start.called)
self.assertFalse(tendrils[2]._start.called)
@mock.patch.object(gevent, 'sleep', side_effect=TestException())
@mock.patch.object(socket,
'socket',
return_value=mock.Mock(
**{
'getsockname.return_value': ('127.0.0.1', 8080),
'close.side_effect': socket.error(),
}))
@mock.patch.object(manager.TendrilManager, '_track_tendril')
@mock.patch.object(tcp, 'TCPTendril')
def test_listener_acceptor_badclose(self, mock_TCPTendril,
mock_track_tendril, mock_socket,
mock_sleep):
clis = [
mock.Mock(**{
'close.side_effect': socket.error(),
}),
mock.Mock(**{
'close.side_effect': socket.error(),
}),
mock.Mock(**{
'close.side_effect': socket.error(),
})
]
mock_socket.return_value.accept.side_effect = [
(clis[0], ('127.0.0.2', 8082)),
(clis[1], ('127.0.0.3', 8083)),
(clis[2], ('127.0.0.4', 8084)),
TestException(),
TestException(),
TestException(),
TestException(),
TestException(),
TestException(),
TestException(),
TestException(),
TestException(),
TestException(),
TestException(),
]
tendrils = [mock.Mock(), mock.Mock(), mock.Mock()]
mock_TCPTendril.side_effect = tendrils[:]
acceptor = mock.Mock(side_effect=[
mock.Mock(),
TestException(),
mock.Mock(),
])
manager = tcp.TCPTendrilManager()
manager.running = True
with self.assertRaises(TestException):
manager.listener(acceptor, None)
self.assertFalse(mock_sleep.called)
mock_socket.assert_called_once_with(socket.AF_INET, socket.SOCK_STREAM)
mock_socket.return_value.assert_has_calls([
mock.call.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1),
mock.call.bind(('', 0)),
mock.call.getsockname(),
mock.call.listen(1024),
mock.call.accept(),
mock.call.accept(),
mock.call.accept(),
mock.call.accept(),
mock.call.accept(),
mock.call.accept(),
mock.call.accept(),
mock.call.accept(),
mock.call.accept(),
mock.call.accept(),
mock.call.accept(),
mock.call.accept(),
mock.call.accept(),
mock.call.accept(),
mock.call.close(),
])
self.assertEqual(manager.local_addr, ('127.0.0.1', 8080))
mock_TCPTendril.assert_has_calls([
mock.call(manager, clis[0], ('127.0.0.2', 8082)),
mock.call(manager, clis[1], ('127.0.0.3', 8083)),
mock.call(manager, clis[2], ('127.0.0.4', 8084)),
])
acceptor.assert_has_calls([
mock.call(tendrils[0]),
mock.call(tendrils[1]),
mock.call(tendrils[2]),
])
self.assertFalse(clis[0].close.called)
clis[1].close.assert_called_once_with()
self.assertFalse(clis[2].close.called)
mock_track_tendril.assert_has_calls([
mock.call(tendrils[0]),
mock.call(tendrils[2]),
])
tendrils[0]._start.assert_called_once_with()
self.assertFalse(tendrils[1]._start.called)
tendrils[2]._start.assert_called_once_with()
@mock.patch.object(gevent, 'sleep', side_effect=TestException())
@mock.patch.object(socket,
'socket',
return_value=mock.Mock(
**{
'accept.side_effect': TestException(),
'getsockname.return_value': ('127.0.0.1', 8080),
'listen.side_effect': TestException(),
'close.side_effect': socket.error(),
}))
@mock.patch.object(manager.TendrilManager, '_track_tendril')
@mock.patch.object(tcp, 'TCPTendril', return_value=mock.Mock())
def test_listener_nolisten_noclose(self, mock_TCPTendril,
mock_track_tendril, mock_socket,
mock_sleep):
acceptor = mock.Mock()
manager = tcp.TCPTendrilManager()
manager.running = True
with self.assertRaises(TestException):
manager.listener(acceptor, None)
self.assertFalse(mock_sleep.called)
mock_socket.assert_called_once_with(socket.AF_INET, socket.SOCK_STREAM)
mock_socket.return_value.assert_has_calls([
mock.call.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1),
mock.call.bind(('', 0)),
mock.call.getsockname(),
mock.call.listen(1024),
mock.call.close(),
])
self.assertEqual(manager.local_addr, ('127.0.0.1', 8080))
self.assertFalse(mock_TCPTendril.called)
self.assertFalse(acceptor.called)
self.assertFalse(mock_track_tendril.called)
@mock.patch.object(gevent, 'sleep', side_effect=TestException())
@mock.patch.object(socket,
'socket',
return_value=mock.Mock(**{
'getsockname.return_value': ('127.0.0.1', 8080),
}))
@mock.patch.object(manager.TendrilManager, '_track_tendril')
@mock.patch.object(tcp, 'TCPTendril')
def test_listener_err_thresh(self, mock_TCPTendril, mock_track_tendril,
mock_socket, mock_sleep):
clis = [mock.Mock(), mock.Mock(), mock.Mock()]
mock_socket.return_value.accept.side_effect = [
TestException(),
TestException(),
TestException(),
TestException(),
TestException(),
TestException(),
TestException(),
TestException(),
TestException(),
(clis[0], ('127.0.0.2', 8082)),
(clis[1], ('127.0.0.3', 8083)),
(clis[2], ('127.0.0.4', 8084)),
TestException(),
TestException(),
TestException(),
]
tendrils = [mock.Mock(), mock.Mock(), mock.Mock()]
mock_TCPTendril.side_effect = tendrils[:]
acceptor = mock.Mock(side_effect=[
mock.Mock(),
TestException(),
mock.Mock(),
])
manager = tcp.TCPTendrilManager()
manager.running = True
with self.assertRaises(TestException):
manager.listener(acceptor, None)
self.assertFalse(mock_sleep.called)
mock_socket.assert_called_once_with(socket.AF_INET, socket.SOCK_STREAM)
mock_socket.return_value.assert_has_calls([
mock.call.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1),
mock.call.bind(('', 0)),
mock.call.getsockname(),
mock.call.listen(1024),
mock.call.accept(),
mock.call.accept(),
mock.call.accept(),
mock.call.accept(),
mock.call.accept(),
mock.call.accept(),
mock.call.accept(),
mock.call.accept(),
mock.call.accept(),
mock.call.accept(),
mock.call.accept(),
mock.call.accept(),
mock.call.accept(),
mock.call.accept(),
mock.call.accept(),
mock.call.close(),
])
self.assertEqual(manager.local_addr, ('127.0.0.1', 8080))
mock_TCPTendril.assert_has_calls([
mock.call(manager, clis[0], ('127.0.0.2', 8082)),
mock.call(manager, clis[1], ('127.0.0.3', 8083)),
mock.call(manager, clis[2], ('127.0.0.4', 8084)),
])
acceptor.assert_has_calls([
mock.call(tendrils[0]),
mock.call(tendrils[1]),
mock.call(tendrils[2]),
])
self.assertFalse(clis[0].close.called)
clis[1].close.assert_called_once_with()
self.assertFalse(clis[2].close.called)
mock_track_tendril.assert_has_calls([
mock.call(tendrils[0]),
mock.call(tendrils[2]),
])
tendrils[0]._start.assert_called_once_with()
self.assertFalse(tendrils[1]._start.called)
tendrils[2]._start.assert_called_once_with()
def _exited_with(code):
if code == -signal.SIGINT:
return gevent.GreenletExit('Exited by SIGINT')
return ProcessExit(code)
