def _init_sio_client(self):
try:
self.client = SocketIO(
self.host,
self.port,
resource='sio',
transports=['websocket'],
params={
'userid': self.player_id,
'roomid': self.room_id
},
wait_for_connection=False # Disabling auto-reconnection.
)
'''
The NodeJs server MUST be using `"socket.io": "1.7.2"` or an error complaining
"
... socketIO_client.__init__, in _get_engineIO_session ...
... transport.recv_packet() ...
"
would be thrown here.
'''
self.client.on('connect', self.on_connect)
self.client.on('disconnect', self.on_disconnect)
self.client.on('message', self.on_message)
except ConnectionError:
print(
"SimplestConnectionEstablishmentPlayer instance for player.id == %s to room_id == %s. The sio-server is down, try again later."
% (self.player_id, self.room_id))
raise GreenletExit()
except KeyboardInterrupt:
raise GreenletExit()
def save_search_result(p, queue, retry=0):
proxy = Proxy.get_random()['address']
url = SEARCH_URL.format(SEARCH_TEXT, p)
try:
r = fetch(url, proxy=proxy)
except (Timeout, ConnectionError):
sleep(0.1)
retry += 1
if retry > 5:
queue.put(url)
raise GreenletExit()
try:
p = Proxy.objects.get(address=proxy)
if p:
p.delete()
except DoesNotExist:
pass
return save_search_result(url, queue, retry)
soup = BeautifulSoup(r.text, 'lxml')
results = soup.find(class_='results')
if results is None:
# 此代理已经被封, 换其他的代理
sleep(0.1)
retry += 1
if retry > 5:
queue.put(url)
raise GreenletExit()
return save_search_result(url, queue, retry)
articles = results.find_all('div', lambda x: 'wx-rb' in x)
for article in articles:
save_article(article)
def run(self, *args, **kwargs):
self.args = args
self.kwargs = kwargs
try:
if hasattr(self, "on_start"):
self.on_start()
except InterruptTaskSet as e:
if e.reschedule:
six.reraise(RescheduleTaskImmediately,
RescheduleTaskImmediately(e.reschedule),
sys.exc_info()[2])
else:
six.reraise(RescheduleTask, RescheduleTask(e.reschedule),
sys.exc_info()[2])
while (True):
try:
if not self._task_queue:
self.schedule_task(self.get_next_task())
try:
if self.locust._state == LOCUST_STATE_STOPPING:
raise GreenletExit()
self.execute_next_task()
if self.locust._state == LOCUST_STATE_STOPPING:
raise GreenletExit()
except RescheduleTaskImmediately:
if self.locust._state == LOCUST_STATE_STOPPING:
raise GreenletExit()
pass
except RescheduleTask:
self.wait()
else:
self.wait()
except InterruptTaskSet as e:
if e.reschedule:
six.reraise(RescheduleTaskImmediately,
RescheduleTaskImmediately(e.reschedule),
sys.exc_info()[2])
else:
six.reraise(RescheduleTask, RescheduleTask(e.reschedule),
sys.exc_info()[2])
except StopLocust:
raise
except GreenletExit:
raise
except Exception as e:
events.locust_error.fire(locust_instance=self,
exception=e,
tb=sys.exc_info()[2])
if self.locust._catch_exceptions:
sys.stderr.write("\n" + traceback.format_exc())
self.wait()
else:
raise
def run(self, *args, **kwargs):
self.args = args
self.kwargs = kwargs
try:
if hasattr(self, "on_start"):
self.on_start()
except InterruptTaskSet as e:
if e.reschedule:
raise RescheduleTaskImmediately(e.reschedule).with_traceback(
sys.exc_info()[2])
else:
raise RescheduleTask(e.reschedule).with_traceback(
sys.exc_info()[2])
while (True):
try:
if not self._task_queue:
self.schedule_task(self.get_next_task())
try:
if self.locust._state == LOCUST_STATE_STOPPING:
raise GreenletExit()
self.execute_next_task()
if self.locust._state == LOCUST_STATE_STOPPING:
raise GreenletExit()
except RescheduleTaskImmediately:
if self.locust._state == LOCUST_STATE_STOPPING:
raise GreenletExit()
pass
except RescheduleTask:
self.wait()
else:
self.wait()
except InterruptTaskSet as e:
if e.reschedule:
raise RescheduleTaskImmediately(e.reschedule) from e
else:
raise RescheduleTask(e.reschedule) from e
except StopLocust:
raise
except GreenletExit:
raise
except Exception as e:
self.locust.environment.events.locust_error.fire(
locust_instance=self, exception=e, tb=sys.exc_info()[2])
if self.locust._catch_exceptions:
logger.error("%s\n%s", e, traceback.format_exc())
self.wait()
else:
raise
def on_disconnect(self):
'''
A subtlety to be concerned here.
'''
if (True == self.client._should_stop_waiting()):
# If the execution reaches here by actively calling `self.client.disconnect()`, then one finds "True == self.client._should_stop_waiting() == self.client._wants_to_close".
print(
'[ACTIVELY DISCONNECTED] SimplestConnectionEstablishmentPlayer for player.id == %s to room_id == %s.'
% (self.player_id, self.room_id))
raise StopLocust() # This is within the "main locust".
else:
# If the execution reaches here passively within `self.client.wait()`, then one finds "False == self.client._should_stop_waiting() == self.client._wants_to_close", and should help terminate the loop in the spawned `self.client.wait()`. See https://github.com/invisibleroads/socketIO-client/blob/master/socketIO_client/__init__.py for details (socketIO_client v0.7.2).
print(
'[PASSIVELY DISCONNECTED] SimplestConnectionEstablishmentPlayer for player.id == %s to room_id == %s.'
% (self.player_id, self.room_id))
self.client._close()
gevent.getcurrent().spawning_greenlet().kill()
'''
Killing the current `gevent.Greenlet` instance silently.
Quoted from http://www.gevent.org/api/gevent.greenlet.html#gevent.GreenletExit for "gevent v1.3.7.dev0".
"
A special exception that kills the greenlet silently.
When a greenlet raises GreenletExit or a subclass, the traceback is not printed and the greenlet is considered successful. The exception instance is available under value property as if it was returned by the greenlet, not raised.
"
'''
raise GreenletExit(
) # This is within the "spawned greenlet from main locust".
def stop_greenlet_from_hub(result: AsyncResult) -> None:
""" Stop the greenlet if the nursery is stopped. """
try:
result.get()
except BaseException as e:
greenlet.throw(e)
else:
greenlet.throw(GreenletExit())
def save_search_result(page, queue, retry=0):
proxy = Proxy.get_random()['address']
url = SEARCH_URL.format(SEARCH_TEXT, page)
try:
r = fetch(url, proxy=proxy)
except (Timeout, ConnectionError, IOError):
sleep(0.1)
retry += 1
if retry > 5:
put_new_page(page, queue)
raise GreenletExit()
try:
p = Proxy.objects.get(address=proxy)
if p:
p.delete()
except DoesNotExist:
pass
return save_search_result(page, queue, retry)
soup = BeautifulSoup(r.text, 'lxml')
results = soup.find(class_='results')
if results is None:
# 此代理已经被封, 换其他的代理
sleep(0.1)
retry += 1
if retry > 5:
put_new_page(page, queue)
print 'retry too much!'
raise GreenletExit()
return save_search_result(page, queue, retry)
articles = results.find_all('div', lambda x: 'wx-rb' in x)
for article in articles:
save_article(article)
page_container = soup.find(id='pagebar_container')
if page_container and u'下一页' in page_container.text:
last_page = int(page_container.find_all('a')[-2].text)
current_page = int(page_container.find('span').text)
for page in range(current_page + 1, last_page + 1):
put_new_page(page, queue)
def _get_connection_pool(account_id, pool_size, pool_map, readonly):
with _lock_map[account_id]:
try:
pool = pool_map.get(account_id)
return pool if pool else \
pool_map.setdefault(
account_id,
CrispinConnectionPool(account_id,
num_connections=pool_size,
readonly=readonly))
except AuthError:
logger.error('Auth error for account {}'.format(account_id))
raise GreenletExit()
def on_error(self, subtask: Greenlet) -> None:
""" Default callback for substasks link_exception
Default callback re-raises the exception inside _run() """
log.error(
"Runnable subtask died!",
this=self,
running=bool(self),
subtask=subtask,
exc=subtask.exception,
)
if not self.greenlet:
return
exception = subtask.exception or GreenletExit()
self.greenlet.kill(exception)
def _run(self):
try:
if self.is_terminated() or self.has_error():
if self._lock._is_owned():
self._lock.release()
self.trace_locks("run - unlock")
return
self.status = StatusValues.ACTIVE
self.trace_funcs("Started")
if self.component.ports[IN_NULL].is_connected():
self._null_input = self.component.ports[IN_NULL]
# block here until null input receives a packet
self._null_input.receive_once()
if self.component.ports[OUT_NULL].is_connected():
self._null_output = self.component.ports[OUT_NULL]
self_started = self.self_starting
while (self_started or not self.is_all_drained()
or self._null_input is not None
or (self.is_all_drained() and self.must_run)
or self.component.stack_size() > 0):
self._null_input = None
self.has_run = True
# FIXME: added has_error to allow this loop to exit if another
# thread calls parent.signal_error() to set our status to ERROR
if self.is_terminated() or self.has_error():
break
for inp in self.component.inports:
if inp.is_initialized() and not inp.is_null():
inp.open()
self.trace_funcs(colored("Activated", attrs=['bold']))
self.component.execute()
self.trace_funcs(colored("Deactivated", attrs=['bold']))
if self.component._packet_count != 0 and not self.ignore_packet_count_error:
self.trace_funcs("deactivated holding {} packets".format(
self.component._packet_count))
self.error("{} packets not disposed of during component "
"deactivation".format(
self.component._packet_count))
# FIXME: what is the significance of closing and reopening the InitializationConnections?
# - is_all_drained only checks Connections.
# - tests succeed if we simply hard-wire InitializationConnection to always open
# - it ensures that it yields a new result when component is re-activated
for inp in self.component.inports:
if inp.is_initialized() and not inp.is_null():
inp.close()
# if (not icp.is_closed()):
# raise FlowError("Component deactivated with IIP port not closed: " + self.get_name())
#
if self_started:
break
if self.is_all_drained() and self.component.stack_size() == 0:
break # while
if self._null_output is not None:
# p = create("")
# self._null_output.send(p)
self._null_output.close()
self.close_ports()
if self.component.stack_size() != 0:
self.error("Compodenent terminated with stack not empty")
self.parent_network.indicate_terminated(self)
except ComponentError as e:
# FIXME:
if e.get_value() > 0:
self.trace_funcs("Component exception: " + e.get_value())
if e.get_value() > 999:
self.logger.error("terminated with exception code " +
e.get_value())
if self.parent_network is not None:
# record the error and terminate siblings
self.parent_network.signal_error(e)
self.close_ports()
raise GreenletExit()
except Exception as err:
# don't tell the parent if we are already in the ERROR or TERMINATE state
# because then the parent told us to terminate
if self.is_terminated() or self.has_error():
# if we are in the TERMINATED or ERROR state we terminated
# intentionally
return
import traceback
traceback.print_exc()
self.status = StatusValues.ERROR
if self.parent_network is not None:
# record the error and terminate siblings
self.parent_network.signal_error(err)
self.close_ports()
def run():
raise GreenletExit()