def test_async_multi_publish_consume():
with conn.channel() as channel:
# first message
message_body = 'test_async_multi_publish_consume message 1'
channel.basic_publish(
exchange='unit_test_room',
routing_key='user1',
body=message_body
)
recv_queue = Queue()
rchannel = conn.allocate_channel()
rchannel.basic_consume(queue='listener1', callback=recv_queue.put)
resp = recv_queue.get()
eq_(resp.body, message_body)
resp.ack()
assert recv_queue.empty()
with conn.channel() as channel:
# second message
message_body = 'test_async_multi_publish_consume message 2'
channel.basic_publish(
exchange='unit_test_room',
routing_key='user1',
body=message_body
)
resp = recv_queue.get()
eq_(resp.body, message_body)
resp.ack()
def joinall(greenlets, timeout=None, raise_error=False, count=None):
from gevent.queue import Queue
queue = Queue()
put = queue.put
if count is None:
count = len(greenlets)
timeout = Timeout.start_new(timeout)
try:
try:
for greenlet in greenlets:
greenlet.rawlink(put)
if raise_error:
for _ in xrange(count):
greenlet = queue.get()
if not greenlet.successful():
raise greenlet.exception
else:
for _ in xrange(count):
queue.get()
except:
if sys.exc_info()[1] is not timeout:
raise
finally:
for greenlet in greenlets:
greenlet.unlink(put)
finally:
timeout.cancel()
def process_execution(self, temp_vector, qc_params, bad_times):
interval_key = uuid4().hex
data_product_id = self.make_large_dataset(temp_vector)
async_queue = Queue()
def cb(event, *args, **kwargs):
if '_'.join(event.qc_parameter.split('_')[1:]) not in qc_params:
# I don't care about
return
times = event.temporal_values
self.assertEquals(len(times), bad_times)
async_queue.put(1)
es = EventSubscriber(event_type=OT.ParameterQCEvent, origin=data_product_id, callback=cb, auto_delete=True)
es.start()
self.addCleanup(es.stop)
config = DotDict()
config.process.interval_key = interval_key
config.process.qc_params = qc_params
self.sync_launch(config)
# So now the process is started, time to throw an event at it
ep = EventPublisher(event_type='TimerEvent')
ep.publish_event(origin=interval_key)
try:
async_queue.get(timeout=120)
except Empty:
raise AssertionError('QC was not flagged in time')
def test_last_update_cache(self):
handle = self.start_worker()
queue = Queue()
o_process = handle.process
def new_process(msg):
o_process(msg)
queue.put(True)
handle.process = new_process
definition = SBE37_CDM_stream_definition()
publisher = Publisher()
stream_def_id = self.pubsub_cli.create_stream_definition(container=definition)
stream_id = self.pubsub_cli.create_stream(stream_definition_id=stream_def_id)
time = float(0.0)
for granule in self.make_points(definition=definition, stream_id=stream_id, N=10):
publisher.publish(granule, to_name=(self.XP, stream_id+'.data'))
# Determinism sucks
try:
queue.get(timeout=5)
except Empty:
self.assertTrue(False, 'Process never received the message.')
doc = self.db.read(stream_id)
ntime = doc.variables['time'].value
self.assertTrue(ntime >= time, 'The documents did not sequentially get updated correctly.')
time = ntime
def process_execution(self, temp_vector, qc_params, bad_times):
interval_key = uuid4().hex
data_product_id = self.make_large_dataset(temp_vector)
async_queue = Queue()
def cb(event, *args, **kwargs):
times = event.temporal_values
self.assertEquals(len(times), bad_times)
async_queue.put(1)
es = EventSubscriber(event_type=OT.ParameterQCEvent, origin=data_product_id, callback=cb, auto_delete=True)
es.start()
self.addCleanup(es.stop)
config = DotDict()
config.process.interval_key = interval_key
config.process.qc_params = qc_params
self.process_dispatcher.schedule_process(self.process_definition_id, process_id=self.process_id, configuration=config) # The process is now up and running maybe?
self.sync_launch(config)
# So now the process is started, time to throw an event at it
ep = EventPublisher(event_type='TimerEvent')
ep.publish_event(origin=interval_key)
try:
for i in xrange(24):
async_queue.get(timeout=10)
except Empty:
raise AssertionError('QC was not flagged in time: %d', i)
def test_qc_interval_integration(self):
# 1 need to make a dataset that only has one discrete qc violation
# 2 Launch the process
# 3 Setup the scheduler to run it say three times
# 4 Get the Events and verify the data
#--------------------------------------------------------------------------------
# Make a dataset that has only one discrete qc violation
#--------------------------------------------------------------------------------
dp_id, dataset_id, stream_def_id = self.make_data_product()
ph = ParameterHelper(self.dataset_management, self.addCleanup)
monitor = DatasetMonitor(dataset_id)
self.addCleanup(monitor.stop)
for rdt in self.populate_vectors(stream_def_id, 1, lambda x : [41] + [39] * (x-1)):
ph.publish_rdt_to_data_product(dp_id, rdt)
self.assertTrue(monitor.event.wait(10)) # Give it 10 seconds to populate
#--------------------------------------------------------------------------------
# Launch the process
#--------------------------------------------------------------------------------
interval_key = uuid4().hex
config = DotDict()
config.process.interval_key = interval_key
config.process.qc_params = ['glblrng_qc'] # The others are tested in other tests for completeness
self.sync_launch(config)
async_queue = Queue()
def callback(event, *args, **kwargs):
times = event.temporal_values
self.assertEquals(len(times), 1)
async_queue.put(1)
es = EventSubscriber(event_type=OT.ParameterQCEvent, origin=dp_id, callback=callback, auto_delete=True)
es.start()
self.addCleanup(es.stop)
#--------------------------------------------------------------------------------
# Setup the scheduler
#--------------------------------------------------------------------------------
timer_id = self.scheduler_service.create_interval_timer(start_time=time.time(),
end_time=time.time()+13,
interval=5,
event_origin=interval_key)
#--------------------------------------------------------------------------------
# Get the events and verify them
#--------------------------------------------------------------------------------
try:
for i in xrange(2):
async_queue.get(timeout=10)
except Empty:
raise AssertionError('QC Events not raised')
class TestBase(unittest.TestCase):
class MockConfig(object):
def __init__(self):
self.snmp_host = "127.0.0.1"
self.snmp_port = 1337
def setUp(self):
self.log_queue = Queue()
dom = etree.parse('templates/default.xml')
mibs = dom.xpath('//conpot_template/snmp/mibs/*')
#only enable snmp server if we have configuration items
if not mibs:
raise Exception("No configuration for SNMP server")
else:
self.snmp_server = snmp_command_responder.CommandResponder(self.log_queue, server_config=self.MockConfig())
for mib in mibs:
mib_name = mib.attrib['name']
for symbol in mib:
symbol_name = symbol.attrib['name']
value = symbol.xpath('./value/text()')[0]
self.snmp_server.register(mib_name, symbol_name, value)
self.snmp_server.snmpEngine.transportDispatcher.start()
def tearDown(self):
self.snmp_server.snmpEngine.transportDispatcher.stop()
def mock_callback(self, sendRequestHandle, errorIndication, errorStatus, errorIndex, varBindTable, cbCtx):
self.result = None
if errorIndication:
self.result = errorIndication
elif errorStatus:
self.result = errorStatus.prettyPrint()
else:
for oid, val in varBindTable:
self.result = val.prettyPrint()
def test_snmp_get(self):
client = snmp_client.SNMPClient(client_config=self.MockConfig())
OID = ((1, 3, 6, 1, 2, 1, 1, 1, 0), None)
client.get_command(OID, callback=self.mock_callback)
self.assertEqual("Siemens, SIMATIC, S7-200", self.result)
log_item = self.log_queue.get(True, 2)
self.assertIsInstance(log_item['timestamp'], datetime)
self.assertEqual('127.0.0.1', log_item['remote'][0])
self.assertEquals('snmp', log_item['data_type'])
def test_snmp_set(self):
client = snmp_client.SNMPClient(client_config=self.MockConfig())
OID = ((1, 3, 6, 1, 2, 1, 1, 6, 0), rfc1902.OctetString('test comment'))
client.set_command(OID, callback=self.mock_callback)
# FIXME: no log entry for set commands
client.get_command(OID, callback=self.mock_callback)
self.assertEqual("test comment", self.result)
get_log_item = self.log_queue.get(True, 5)
self.assertIsInstance(get_log_item['timestamp'], datetime)
self.assertEqual('127.0.0.1', get_log_item['remote'][0])
self.assertEquals('snmp', get_log_item['data_type'])
class MemorySession(Session):
"""
In memory session with a outgoing gevent Queue as the message
store.
"""
def __init__(self, server, session_id=None):
super(MemorySession, self).__init__(server, session_id=session_id)
self.session_id = session_id or str(uuid.uuid4())[:8]
self.server = server
self.queue = Queue()
self.hits = 0
self.heartbeats = 0
self.connected = False
def add_message(self, msg):
self.queue.put_nowait(msg)
def get_messages(self, **kwargs):
timeout = kwargs.get('timeout', None)
self.incr_hits()
if self.queue.empty():
try:
return self.queue.get(**kwargs)
except Empty:
return []
else:
accum = []
try:
while not self.queue.empty():
if timeout:
accum.append(self.queue.get(timeout=timeout))
else:
accum.append(self.queue.get_nowait())
finally:
return accum
def interrupt(self):
"""
A kill event trigged through a client accessible endpoint
Internal expires will not have is_interupted() == True
"""
self.interrupted = True
self.kill()
def kill(self):
self.connected = False
# Expire only once
if not self.expired:
self.expired = True
self.timeout.set()
def test_link_to_queue(self):
p = gevent.spawn(lambda: 100)
q = Queue()
p.link(q.put)
self.assertEqual(q.get().get(), 100)
for i in range(3):
p.link(q.put)
self.assertEqual(q.get().get(), 100)
class UA(object):
def __init__(self, app, stack):
self.app, self.options, self._stack = app, app.options, stack
self._closeQ = self._ua = self._gen = None
self.scheme = self._stack.transport.secure and "sips" or "sip"
self.localParty = rfc2396.Address("%s:%s@%s" % (self.scheme, self.options.user, self.options.domain))
self.proxy = rfc2396.URI("%s:%s" % (self.scheme, self.options.proxy)) if self.options.proxy else None
def _waitOnClose(self): # Wait on close event to be signaled by another task
if self._closeQ is None:
self._closeQ = Queue()
self._closeQ.get()
self._closeQ = None
else:
raise ValueError("some other task is already waiting on close")
def _signalClose(self): # Signal the close event on this object.
if self._closeQ is not None:
self._closeQ.put(None)
def _createClient(self, localParty, remoteParty, remoteTarget):
ua = self._ua = rfc3261.UserAgent(self._stack)
ua.app, ua.localParty, ua.remoteParty, ua.remoteTarget = (
self,
localParty.dup(),
remoteParty.dup(),
remoteTarget.dup(),
)
def _scheduleRefresh(
self, response, handler
): # Schedule handler to be invoked before response.Expires or self._interval
interval = int(response.Expires.value) if response.Expires else self._interval
interval = max(interval - random.randint(5000, 15000) / 1000.0, 5)
if interval > 0:
logger.debug("scheduling refresh after %r", interval)
self._gen = gevent.spawn_later(interval, handler)
def _scheduleRetry(self, handler): # Schedule handler to be invoked after retry_interval.
logger.debug("scheduling retry after %r", self.options.retry_interval)
self._gen = gevent.spawn_later(self.options.retry_interval, handler)
def _closeGen(self):
if self._gen is not None:
self._gen.kill()
self._gen = None
def _closeUA(self):
if self._ua is not None:
self._ua.app = None
self._ua = None
self.app = None # remove reference
def dialogCreated(self, ua, dialog): # Invoked by SIP stack to inform that UserAgent is converted to Dialog.
if self._ua is not None:
self._ua.app = None
self._ua, dialog.app = dialog, self
def test_link_to_channel(self):
p1 = gevent.spawn(lambda: 101)
p2 = gevent.spawn(lambda: 102)
p3 = gevent.spawn(lambda: 103)
q = Queue(0)
p1.link(q.put)
p2.link(q.put)
p3.link(q.put)
results = [q.get().get(), q.get().get(), q.get().get()]
assert sorted(results) == [101, 102, 103], results
def bad():
q = Queue()
print('switching out, then throwing in')
try:
q.get(block=True, timeout=0.1)
except Empty:
print("Got Empty")
print('switching out')
gevent.sleep(0.1)
print('switched in')
class Rig(object):
def __init__(self, device):
self.txlines = Queue()
self.rxlines = Queue()
self.tty = serial.Serial(device, 9600, timeout=1)
gevent.spawn(self.receiver)
gevent.spawn(self.sender)
self.vfos = (VFO(), VFO())
self.con_vfo = None
self.tx_vfo = None
def receiver(self):
rxbuf = []
while True:
char = self.tty.read(1)
if char == "":
continue
elif char == "\r":
log.debug("read %s" % repr("".join(rxbuf)))
self.rxlines.put("".join(rxbuf))
rxbuf = []
elif char != "":
rxbuf.append(char)
def sendline(self, line):
self.txlines.put(line)
def sender(self):
while True:
line = self.txlines.get()
log.debug("write %s" % repr(line))
self.tty.write(line + "\r")
def print_lines(self):
while True:
print "AI %s" % repr(self.rxlines.get())
def close(self):
self.tty.close()
def __enter__(self, *args, **kwargs):
pass
def __exit__(self, *args, **kwargs):
self.close()
@property
def freq(self):
self.tty.write("BUF 0\n")
return self.tty.readline()
@freq.setter
def freq(self, hz):
pass
def waitall(events):
# QQQ add timeout?
from gevent.queue import Queue
queue = Queue()
put = queue.put
try:
for event in events:
event.rawlink(put)
for _ in xrange(len(events)):
queue.get()
finally:
for event in events:
event.unlink(put)
def init_tasks(self):
filenames = list()
for task_str in self.tasks_str:
if task_str.isnumeric():
info = self.client.get_info(task_str.encode('utf-8'))
obj = FileInfo.create_from_download_info(info, self.filename)
if obj.filename not in filenames:
filenames.append(obj.filename)
else:
if task_str not in filenames:
filenames.append(task_str)
file_infos = list(map(FileInfo.get, filenames))
""":type: List[FileInfo]"""
DBSession().expunge_all()
gevent.spawn(self.start_server, file_infos)
inq = Queue()
filename_outq_map = {}
""":type: Dict[str, Queue]"""
filename_info_map = {}
""":type: Dict[str, FileInfo]"""
for filename in filenames:
filename_outq_map[filename] = Queue()
for info in file_infos:
assert isinstance(info, FileInfo)
assert info.requests, "%s.request is empty" % info.filename
filename_info_map[info.filename] = info
undownload_filenames = list(filenames)
for _ in range(min(self.max_concurrent, len(undownload_filenames))):
inq.put((Classification.FILE_FINISHED, ))
downloading_files = 0
while undownload_filenames:
data = inq.get()
assert data[0] == Classification.FILE_FINISHED
filename = undownload_filenames.pop(0)
info = filename_info_map[filename]
""":type: FileInfo"""
if info.is_finished():
logger.info("%s is finished", filename)
continue
g = gevent.pool.Group()
obj = FileDownloader(filename_info_map[filename], filename_outq_map[filename], inq, g)
g.spawn(obj.start)
downloading_files += 1
while downloading_files:
data = inq.get()
assert data[0] == Classification.FILE_FINISHED
downloading_files -= 1
self.exit_event.set()
class IRP(object):
'''I/O Request Packet'''
complete_tag = object()
def __init__(self):
from gevent.queue import Queue
self.queue = Queue()
def complete(self):
# from gevent_extension import the_hub
self.queue.put(self.complete_tag)
# the_hub.interrupt(self.queue.put, self.complete_tag)
def wait(self):
self.queue.get()
class Channel(gevent.Greenlet):
"""Baseline for inter-app communication"""
def __init__(self, remote, t, c=None):
super(Channel, self).__init__()
self.remote = remote
self.transmit = remote.send
self.c = c
self.t = t
self.inq = Queue()
self.wait_for_roundtrip = False
if self.c is None:
self.wait_for_roundtrip = True
self.c = os.urandom(16).encode('hex')
def _run(self):
self.running = True
if self.wait_for_roundtrip:
data, body = self.inq.get()
self._recv_first(data, body)
#explicit, but not required
self.wait_for_roundtrip = False
gevent.sleep(0)
while self.running:
data, body = self.inq.get()
self._recv(data, body)
def _recv_first(self, data, body):
log.debug("Channel %s recv:\n%s" % (self.c, data))
self.handle_unknown(data, body)
def _recv(self, data, body):
log.debug("Channel %s recv:\n%s" % (self.c, data))
self.handle_unknown(data, body)
def send(self, data, body=''):
data['c'] = self.c
self.transmit(data, body)
def handle_unknown(self, data, body):
if 'err' in data:
log.debug('Remote error: %s' % data['err'])
return
if 'end' in data:
return
err = '%s currently unimplemented' % self.t
resp = {'end': True, 'err': err}
#log.debug('To %s: %s' % (remote.id.hash_name, err))
self.send(resp)
class Actor(gevent.Greenlet):
def __init__(self):
self.running = False
self.inbox = Queue()
gevent.Greenlet.__init__(self)
def received(self, message):
"""
Define your subclass to handle incoming messages here...
"""
raise NotImplementedError()
def cast(self, message):
""" Send a message to the actor.
If the actor is busy, the message will be enqueued for later
consumption. There is no return value.
>>> a = Actor()
>>> a.received = lambda msg: msg
>>> a.cast("hello")
"""
self.inbox.put_nowait(message)
def _run(self):
self.running = True
while self.running:
message = self.inbox.get()
self.received(message)
class Spider:
def __init__(self, url='', depth=1, threads=4):
self.url = url
self.depth = depth
self.threads = threads
self.tasks = Queue()
self.bucket = []
def run(self):
self.tasks.put(Task(self.url, self.depth))
threds = [
gevent.spawn(self.worker)
for i in range(self.threads)
]
gevent.joinall(threds)
def worker(self, worker_id=''):
while not self.tasks.empty():
task = self.tasks.get()
if task.url in self.bucket:
# here have a bug
continue
self.bucket.append(task.url)
task.run()
for t in task.subtasks:
self.tasks.put_nowait(t)
def parallel_loop(func=None, elem=None, *args, **kwargs):
import gevent
from gevent.queue import Queue
ret = []
api.bui.cli.logger.debug('Using gevent')
if not callable(func):
api.abort(500, 'The provided \'func\' is not callable!')
if not elem:
return []
output = Queue()
processes = [
gevent.spawn(
func,
e,
output,
*args,
**kwargs
) for e in elem
]
# wait for process termination
gevent.joinall(processes)
for p in processes:
tmp = output.get()
if isinstance(tmp, basestring):
api.abort(500, tmp)
elif tmp:
ret.append(tmp)
return ret
class C2DMService(object):
def __init__(self, source, email, password):
self.source = source
self.email = email
self.password = password
self._send_queue = Queue()
self._send_queue_cleared = Event()
self.log = logging.getLogger('pulsus.service.c2dm')
def _send_loop(self):
self._send_greenlet = gevent.getcurrent()
try:
self.log.info("C2DM service started")
while True:
notification = self._send_queue.get()
try:
self._do_push(notification)
except Exception, e:
self.log.exception("Error while pushing")
self._send_queue.put(notification)
gevent.sleep(5.0)
finally:
if self._send_queue.qsize() < 1 and \
not self._send_queue_cleared.is_set():
self._send_queue_cleared.set()
class BlackBerryPushService(object):
def __init__(self, app_id, password, push_url):
self.app_id = app_id
self.password = password
self.push_url = push_url
self._send_queue = Queue()
self._send_queue_cleared = Event()
self.log = logging.getLogger('pulsus.service.bbp')
def _send_loop(self):
self._send_greenlet = gevent.getcurrent()
try:
self.log.info("BlackBerry Push service started")
while True:
notification = self._send_queue.get()
try:
self._do_push(notification)
except Exception, e:
print e
self._send_queue.put(notification)
gevent.sleep(5.0)
finally:
if self._send_queue.qsize() < 1 and \
not self._send_queue_cleared.is_set():
self._send_queue_cleared.set()
class WebSocketClient(ThreadedClient):
def __init__(self, url, protocols=None, version='8'):
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0)
ThreadedClient.__init__(self, url, protocols=protocols, version=version, sock=sock)
self._lock = Semaphore()
self._th = Greenlet(self._receive)
self._messages = Queue()
self.extensions = []
def opened(self, protocols, extensions):
self.protocols = protocols
self.extensions = extensions
def received_message(self, m):
self._messages.put(copy.deepcopy(m))
def write_to_connection(self, bytes):
if not self.client_terminated:
return self.sock.sendall(bytes)
def closed(self, code, reason=None):
self._messages.put(StreamClosed(code, reason))
def receive(self, msg_obj=False):
msg = self._messages.get()
if isinstance(msg, StreamClosed):
return None
if msg_obj:
return msg
else:
return msg.data
class NotifyingQueue(Event):
""" A queue that follows the wait protocol. """
def __init__(self):
super(NotifyingQueue, self).__init__()
self._queue = Queue()
def put(self, item):
""" Add new item to the queue. """
self._queue.put(item)
self.set()
def empty(self):
return self._queue.empty()
def get(self, block=True, timeout=None):
""" Removes and returns an item from the queue. """
value = self._queue.get(block, timeout)
if self._queue.empty():
self.clear()
return value
def stop(self):
""" Request a stop event. """
self.set()
class WebSocketHandler(ThreadedHandler):
"""WebSocket API for handlers
This provides a socket-like interface similar to the browser
WebSocket API for managing a WebSocket connection.
"""
def __init__(self, sock, protocols, extensions, environ):
ThreadedHandler.__init__(self, sock, protocols, extensions)
self.environ = environ
self._messages = Queue()
self._lock = Lock()
self._th = gevent.spawn(self._receive)
def closed(self, code, reason=None):
self._messages.put(StreamClosed(code, reason))
def received_message(self, m):
self._messages.put(copy.deepcopy(m))
def receive(self, msg_obj=False):
msg = self._messages.get()
if isinstance(msg, StreamClosed):
# Maybe we'll do something better
return None
if msg_obj:
return msg
else:
return msg.data
class CrawlBase(object):
spider_count = 0
def __init__(self):
self.group = Group()
self.queue = Queue()
def read_seed(self, file='seeds.txt'):
with open(file) as f:
for line in f:
if len(line) > 0 and line != "\n":
yield line.strip()
else:
return
def dispatch(self):
for url in self.read_seed():
g = gevent.spawn(Spider, self, url)
self.group.add(g)
self.group.join()
def harvest(self):
try:
while True:
content = self.queue.get(timeout=2)
print(content)
except Empty:
pass
class CHubCallbackQueueBase(CHubCallbackBasicBase):
def __init__(self,sHubId):
CHubCallbackBasicBase.__init__(self,sHubId)
self.__queue4Return = Queue() #当前应答队列
def PutCmdStrToReturnQueue(self, lsCmdStr):
self.__queue4Return.put(lsCmdStr)
def GetCmdStrFmReturnQueue(self):
return self.__queue4Return.get() # (sClientIPPort,dwCmdId,CmdOStr)
def HandleRequestCmd(self, sClientIPPort, dwCmdId, CmdIStr):
# 处理客户端请求命令
bDone = CHubCallbackBasicBase.HandleRequestCmd(self, sClientIPPort, dwCmdId, CmdIStr)
if not bDone and CmdIStr[0].startswith(CMD0_ECHO_CMD):
CmdOStr = ['OK','CHubCallbackQueueBase']
CmdOStr.extend(CmdIStr)
dwCmdId = GetCmdReplyFmRequest(dwCmdId)
self.PutCmdStrToReturnQueue([sClientIPPort,dwCmdId,CmdOStr])
bDone = True
return bDone
def DoHandleCheckAllLinkReply(self):
# 处理检查所有链接的应答返还消息（包括通知消息）等
# 该函数在该类中实现后，一般情况下子类无需再继承。
while not self.bQuitLoopFlag:
return self.GetCmdStrFmReturnQueue()
class FlowTests(TestCase):
def create(self, conf={}, events=[]):
self.input = Queue()
self.output = Queue()
context = DummyContext()
with context:
self.i = self.create_stage(**conf)
self.input = self.i.setup(self.output)
self.assertEquals(1, len(context.stages))
self.i.start()
for ev in events:
self.input.put(ev)
return self.i
def wait(self, timeout=1.0, events=1):
with gevent.Timeout(timeout):
# wait for input to be consumed and output to be produced
while self.input.qsize():
gevent.sleep(0.0)
while self.output.qsize() < events:
gevent.sleep(0.0)
self.i.stop()
if events:
return [self.output.get() for n in xrange(events)]
class LocalControlProgramDescriptor(object):
def __init__(self, hc, agentUuid, program_id):
self.log = logging.getLogger('LocalControlProgramDescriptor_{}'.format(program_id))
self.agentUuid = agentUuid
self.id = program_id
self.hc = hc
self.queue = Queue()
def _receive_msg(self, msg):
self.queue.put(msg)
def recv(self, block=True, timeout=None):
try:
self.log.debug("Waiting for msg in blocking call")
msg = self.queue.get(block=block, timeout=timeout)
return msg
except gevent.timeout.Timeout as e:
return None
except gevent.queue.Empty as e:
return None
def send(self, msg):
return self.hc.send_msg_to_local_control_program(self.id, msg, self.agentUuid)
def close(self):
return self.hc.stop_local_control_program(self.id, self.agentUuid)
class Spider:
def __init__(self, url='', depth=1):
self.tasks = Queue()
self.tasks.put(url)
self.init_url = url or ''
self.depth = depth or ''
def run(self):
threds = [
gevent.spawn(self.work),
gevent.spawn(self.work),
gevent.spawn(self.work),
gevent.spawn(self.work)
]
gevent.joinall(threds)
def work(self):
while not self.tasks.empty():
page = self.tasks.get()
p = Page(page, '')
p.do_request()
p.parse_content()
hrefs = p.hrefs
for href in hrefs:
self.tasks.put_nowait(href)
def lookup(self, target_key):
target_id = keccak256(target_key)
closest = []
while not closest:
closest = self.closest(target_id, BUCKET_SIZE)
if not closest:
# add seed nodes
for bn in self.server.boot_nodes:
self.add_node(bn)
asked = [self.self_node.node_id]
pending_queries = 0
reply_queue = Queue()
while True:
for n in closest:
if pending_queries >= KAD_ALPHA:
break
if n.node_id not in asked:
asked.append(n.node_id)
pending_queries += 1
gevent.spawn(self.find_neighbours, n, target_key,
reply_queue)
if pending_queries == 0:
break
ns = reply_queue.get()
pending_queries -= 1
if ns:
for node in ns:
farther = find_farther_to_target_than(
closest, target_id, node)
if farther:
closest.remove(farther)
if len(closest) < BUCKET_SIZE:
closest.append(node)
class OutputService(BaseService):
"""
This is a service which one expects output
to be tracked. The mechanism is via the use
of an output_queue of type gevent.Queue.
As opposed to a scheduled
service which is almost a fire and forget.
Note that a fire and forget is great where
it can still log output to an external
system for deferred parsing.
"""
def __init__(self, name, size=None, items=None):
BaseService.__init__(self, name)
self._queue = Queue(size, items)
def put(self, item, block=True, timeout=None):
return self._queue.put(item, block, timeout)
def get(self, block=True, timeout=None):
"""
Gets from the backing Queue.
If calling get on empty Queue, it will block
until Queue actually has contents to return.
If that is undesired set block=False to prevent
greenlet blocking.
:param block:
:param timeout:
:return:
"""
return self._queue.get(block, timeout)
def size(self):
return self.queue_ref().qsize()
def queue_ref(self):
"""
Direct access if necessary. Warning this will allow
direct access for modification. Tend not to use it.
:return:
"""
return self._queue
class GeventActor(Greenlet):
def __init__(self):
super(GeventActor, self).__init__()
self._inbox = Queue()
self._running = False
def tell(self, message):
self._inbox.put(message)
def on_receive(self, message):
raise NotImplementedError()
def _run(self):
self._running = True
while self._running:
message = self._inbox.get()
if not isinstance(message, Dokka.Messages.Message):
continue
if isinstance(message, Dokka.Messages.StopActor):
self._running = False
self.on_receive(message)
class NotifyingQueue(Event):
""" A queue that follows the wait protocol. """
def __init__(self):
super(NotifyingQueue, self).__init__()
self._queue = Queue()
def put(self, item):
""" Add new item to the queue. """
self._queue.put(item)
self.set()
def get(self, block=True, timeout=None):
""" Removes and returns an item from the queue. """
value = self._queue.get(block, timeout)
if self._queue.empty():
self.clear()
return value
def stop(self):
""" Request a stop event. """
self.set()
class BufferedOpusEncoderPlayable(BasePlayable, AbstractOpus, OpusEncoder):
def __init__(self, source, *args, **kwargs):
self.source = source
self.frames = Queue(kwargs.pop('queue_size', 4096))
super(BufferedOpusEncoderPlayable, self).__init__(*args, **kwargs)
gevent.spawn(self._encoder_loop)
def _encoder_loop(self):
while self.source:
raw = self.source.read(self.frame_size)
if len(raw) < self.frame_size:
break
self.frames.put(self.encode(raw, self.samples_per_frame))
gevent.idle()
self.source = None
def next_frame(self):
if not self.source:
return None
return self.frames.get()
def get_from_pool(pool_id):
# check and initialize
if (boto_sessions[0] == None and config.aws_config):
boto_sessions[0] = boto3.session.Session(**config.aws_config)
conn = None
conn_pool = conn_pools.get(pool_id, None)
if (conn_pool == None):
# if no conn_pool for given pool_id create and store in conn_pools
conn_pool = Queue()
conn_pools[pool_id] = conn_pool
try:
conn = conn_pool.get(block=False)
# print "reusing from pool"
except Empty:
#if queue is empty create a connection for use.
if (pool_id in _pool_item_generators):
conn = _pool_item_generators[pool_id]()
return conn
class NotifyingQueue(Event, Generic[T]):
"""This is not the same as a JoinableQueue. Here, instead of waiting for
all the work to be processed, the wait is for work to be available.
"""
def __init__(self, maxsize: int = None, items: Iterable[T] = ()) -> None:
super().__init__()
self.queue = Queue(maxsize, items)
if items:
self.set()
def put(self, item: T) -> None:
""" Add new item to the queue. """
self.queue.put(item)
self.set()
def get(self, block: bool = True, timeout: float = None) -> T:
""" Removes and returns an item from the queue. """
value = self.queue.get(block, timeout)
if self.queue.empty():
self.clear()
return value
def peek(self, block: bool = True, timeout: float = None) -> T:
return self.queue.peek(block, timeout)
def __len__(self) -> int:
return len(self.queue)
def copy(self) -> List[T]:
""" Copies the current queue items. """
copy = self.queue.copy()
result = list()
while not copy.empty():
result.append(copy.get_nowait())
return result
def __repr__(self) -> str:
return f"NotifyingQueue(id={id(self)}, num_items={len(self.queue)})"
class FirstReply:
def __init__(self):
self.size = 0
self.index = 0
self.q = Queue()
def push(self, *args, **kwargs):
self.size += 1
callback = args[0]
args = args[1:]
if callback:
def handle():
if args and not kwargs:
self.q.put(callback(*args))
elif not args and kwargs:
self.q.put(callback())
else:
self.q.put(callback(*args, **kwargs))
spawn(handle)
return self.q
def __len__(self):
return self.size
def __iter__(self):
return self
def next(self):
v = None
if self.index < self.size:
v = self.q.get()
self.index += 1
else:
raise StopIteration
return v
__add__ = push
append = push
class GreenletExecutor(AbstractExecutor):
"""
GreenletExecutor is an AbstractExecutor subclass that uses a pool of
greenlets to execute calls asynchronously.
NOTE: Use this executor for I/O-bound tasks. Since all greenlets are
multiplexed on a single pthread, do NOT use this for compute-bound
callables. Try using the GIPCExecutor instead.
"""
def __init__(self, num_greenlets=50, **kwargs):
super(GreenletExecutor, self).__init__(**kwargs)
self.pool = Pool(size=num_greenlets)
self.task_queue = Queue()
self.num_ready = 0
def _shutdown(self):
for _ in xrange(len(self.pool)):
self.task_queue.put(None)
if self.force_kill_on_shutdown:
self.pool.kill()
else:
self.pool.join()
def _worker_loop(self):
try:
self.num_ready += 1
while True:
self.num_ready -= 1
task = self.task_queue.get()
if task is None:
return
task.execute()
self.num_ready += 1
except:
pass
def _submit(self, task):
self.task_queue.put(task)
if not self.num_ready and self.pool.free_count():
self.pool.spawn(self._worker_loop)
class GeventQueue(object):
def __init__(self, csvdicts, goal_path, gevent_num=100):
# self.function = function
self.csvdicts = csvdicts
self.goal_path = goal_path
self.gevent_num = gevent_num
self.tasks = Queue()
def worker(self, i):
print('worker %d start' % (i))
while not self.tasks.empty():
try:
# print('worker %d start'%(i))
task = self.tasks.get()
# write current task into worker's running log
f = open(os.path.join('logfile', '%s_running.log' % (i)), 'wb')
f.write(task)
f.close()
# run the task
Reorganizer(task, self.goal_path).start()
# write finished work
root, base = os.path.split(task)
fs = open(os.path.join(root, 'success'), 'ab')
fs.write(base)
fs.write('\n')
fs.close()
gevent.sleep(0)
except Exception, e:
print(e)
# write error work into error log
print('error:%d; task:%s' % (i, task))
fl = open(os.path.join('logfile', 'error.log'), 'ab')
fl.write(task)
fl.write('\n')
fl.close()
continue
class GeventQueue(object):
def __init__(self, old_path, new_path, gevent_num=100):
self.old_path = old_path
self.new_path = new_path
self.gevent_num = gevent_num
self.tasks = Queue()
def worker(self, i):
print('worker %d start' % (i))
while not self.tasks.empty():
try:
# print('worker %d start'%(i))
task = self.tasks.get()
# write current task into worker's running log
f = open(
os.path.join('logfile_extractor', '%s_running.log' % (i)),
'wb')
f.write(task.encode('utf-8'))
f.close()
# run the task
Extractor(task, self.old_path, self.new_path).start()
# write finished work
fs = open(os.path.join('extracted'), 'ab')
fs.write(task.encode('utf-8'))
fs.write('\n')
fs.close()
gevent.sleep(0)
except Exception, e:
print(e)
# write error work into error log
print('error:%d; task:%s' % (i, task))
fl = open(os.path.join('logfile', 'error.log'), 'ab')
fl.write(task.encode('utf-8'))
fl.write('\n')
fl.close()
continue
class MyCrawler(object):
def __init__(self):
self.pool = Pool(10)
self.tasks = Queue()
def init_tasks(self):
resp = requests.get('http://www.ttmeiju.com/summary.html')
assert resp.status_code==200
soup = BeautifulSoup(resp.text)
tb = soup.find('table')
def mya(tag):
return tag.name=='a' and tag.parent.name=='td'
meiju={}
for atag in tb.find_all(mya)[3:]:
meiju[atag.string] = atag['href']
self.tasks.put_nowait({'key':atag.string, 'val': atag['href']})
print (meiju)
def do_jobs(self):
self.init_tasks()
while not self.tasks.empty():
task = self.tasks.get()
self.before(task)
self.pool.spawn(self.work, task).rawlink(self.after)
# 等待任务完成
print('~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~')
self.pool.join()
def work(self, task):
print ('子类实现')
def before(self, task):
print('')
def after(self, asynret):
print('')
def messages(self):
"""Listen to the messages WebSocket stream
Retrieve the WebSocket stream and send to it each messages displayed by
the display method. They are JSON encoded as a dict { style: message }.
"""
profile = request.params.get("profile", self.actionsmap.default_authentication)
authenticator = self.actionsmap.get_authenticator(profile)
s_id = authenticator.get_session_cookie()["id"]
try:
queue = self.log_queues[s_id]
except KeyError:
# Create a new queue for the session
queue = Queue()
self.log_queues[s_id] = queue
wsock = request.environ.get("wsgi.websocket")
if not wsock:
raise HTTPResponse(m18n.g("websocket_request_expected"), 500)
while True:
item = queue.get()
try:
# Retrieve the message
style, message = item
except TypeError:
if item == StopIteration:
# Delete the current queue and break
del self.log_queues[s_id]
break
logger.exception("invalid item in the messages queue: %r", item)
else:
try:
# Send the message
wsock.send(json_encode({style: message}))
except WebSocketError:
break
sleep(0)
class MemorySession(session.Session):
"""
In memory session with a ``gevent.pool.Queue`` as the message store.
"""
def __init__(self, *args, **kwargs):
super(MemorySession, self).__init__(*args, **kwargs)
self._queue = Queue()
def add_message(self, frame, data):
log.info('session closed: %s', self.id)
self._queue.put_nowait((frame, data))
#waiter = self._waiter
#if waiter is not None:
# self._waiter = None
# if not waiter.cancelled():
# waiter.set_result(True)
self._tick()
def get_messages(self, timeout=None):
self._tick()
messages = []
# get all messages immediately pending in the queue
while not self._queue.empty():
try:
msg = self._queue.get_nowait()
except Empty:
break
messages.append(msg)
if not messages:
try:
messages.append(self._queue.get(timeout=timeout))
except Empty:
pass
return messages
def _concurrent_execute(self, context, start_req, parser, pool, pool_size):
queue = Queue() # 任务队列
# 将初始化请求加入任务队列
for r in start_req:
queue.put_nowait(r)
if pool is None:
pool = GeventPool(pool_size)
greenlets = []
while True:
try:
req = self._check_req(queue.get(timeout=1))
if req.parser is None:
req.parser = parser
greenlets.append(pool.spawn(req, context, queue))
except Empty:
break
return [greenlet.get() for greenlet in greenlets]
class Actor:
def __init__(self):
self.inbox = Queue()
self.greenlet = spawn(self._listen)
def _listen(self):
try:
while True:
message = self.inbox.get()
self._receive(message)
sleep(0)
except ActorStop:
pass
def _receive(self, message):
raise NotImplemented()
def stop(self):
killall([self.greenlet], exception=ActorStop)
def post(self, message, block=True, timeout=None):
self.inbox.put(message, block, timeout)
def get_from_pool(pool_id='dynamodb'):
conn = None
conn_pool = conn_pools.get(pool_id, None)
if (conn_pool == None):
#if no conn_pool for given pool_id create and store in co]nn_pools
conn_pool = Queue()
conn_pools[pool_id] = conn_pool
try:
conn = conn_pool.get(block=False)
# print "reusing from pool"
except Empty:
#if queue is empty create a connection for use.
if (pool_id.startswith("dynamodb")):
#table_name = pool_id.split("_")[1]
#create new session
if (IS_LOCAL_ENDPOINT[0]):
conn = boto3.resource(
'dynamodb',
endpoint_url='http://{addr}:{port}'.format(
addr=IS_LOCAL_ENDPOINT[0][0],
port=IS_LOCAL_ENDPOINT[0][1]),
aws_access_key_id=" ",
aws_secret_access_key=" ",
region_name="ap-south-1")
else:
conn = boto_session.resource('dynamodb')
elif (pool_id.startswith("s3")):
conn = boto_session.resource('s3')
# print "new connection"
elif (pool_id.startswith("sqs")):
conn = boto_session.client('sqs')
elif (pool_id.startswith("ses")):
conn = boto_session.client('ses', region_name="eu-west-1")
return conn
class Stream(Expression):
def __init__(self, io: bool):
self.io = io
self.typ = self.Type.STREAM
self.q = Queue()
self.closed = False
self.value = None
def is_in(self):
return self.io
def read(self):
if self.closed:
return None
e = self.q.get()
if e == EOS:
self.closed = True
return None
return e
def write(self, e):
return self.q.put(e)
def send_proc():
q = Queue()
if ptoken:
if ptoken in tokenSubscriptions:
tokenSubscriptions[ptoken].append(q)
else:
tokenSubscriptions[ptoken] = [q]
if not hashstr in subscriptions:
subscriptions[hashstr] = [q]
else:
subscriptions[hashstr].append(q)
try:
while True:
yield "data: %s\n\n" % q.get()
# TODO: this may actually not work, need an explicit cleanup here
finally:
# print "Finally! for ptoken", ptoken, "hash:", hashstr
if hashstr in subscriptions:
try:
subscriptions[hashstr].remove(q)
except ValueError:
pass
if len(subscriptions[hashstr]) == 0:
# print "Subs zero", ptoken, "hash:", hashstr
try:
del subscriptions[hashstr]
except KeyError:
pass
if ptoken and ptoken in tokenSubscriptions:
try:
tokenSubscriptions[ptoken].remove(q)
except ValueError:
pass
if len(tokenSubscriptions[ptoken]) == 0:
# print "Tokens zero!", ptoken, "hash:", hashstr
try:
del tokenSubscriptions[ptoken]
except KeyError:
pass
class NotifyingQueue(Event):
def __init__(self):
super(NotifyingQueue, self).__init__()
self._queue = Queue()
def put(self, item):
""" Add new item to the queue. """
self._queue.put(item)
self.set()
def get(self, block=True, timeout=None):
""" Removes and returns an item from the queue. """
value = self._queue.get(block, timeout)
if self._queue.empty():
self.clear()
return value
def peek(self, block=True, timeout=None):
return self._queue.peek(block, timeout)
def __len__(self):
return len(self._queue)
class Buffer(Greenlet):
latency = 0.01
def __init__(self, server, num_frames=1000):
self.server = server
self.num_frames = num_frames
self.frames = Queue(100)
Greenlet.__init__(self)
def _run(self):
for i in range(self.num_frames):
# request data
# print('sleep')
gevent.sleep(self.latency)
# print('get')
f = self.server.get_frame(i)
# print('pre put')
self.frames.put(f) # blocks if full
# print('post put')
def get_frame(self):
return self.frames.get() # blocks if empty
def messages(self):
"""Listen to the messages WebSocket stream
Retrieve the WebSocket stream and send to it each messages displayed by
the core.MoulinetteSignals.display signal. They are JSON encoded as a
dict { style: message }.
"""
s_id = request.get_cookie('session.id')
try:
queue = self.log_queues[s_id]
except KeyError:
# Create a new queue for the session
queue = Queue()
self.log_queues[s_id] = queue
wsock = request.environ.get('wsgi.websocket')
if not wsock:
raise HTTPErrorResponse(m18n.g('websocket_request_expected'))
while True:
item = queue.get()
try:
# Retrieve the message
style, message = item
except TypeError:
if item == StopIteration:
# Delete the current queue and break
del self.log_queues[s_id]
break
logger.exception("invalid item in the messages queue: %r",
item)
else:
try:
# Send the message
wsock.send(json_encode({style: message}))
except WebSocketError:
break
sleep(0)
def command(self, method: str, **params) -> dict:
self._seq_no += 1
seq_no = self._seq_no
data = dict(id=seq_no, method=method, params=params)
q = Queue()
self._response_queues[seq_no] = q
try:
self._send_command(json.dumps(data))
return q.get(timeout=self.TIMEOUT)
except Empty as e:
if self.is_connected():
self.logger.exception('method={} params={}\n{}'.format(
method, params, str(e)))
raise
return dict(error='response timeout by disconnection')
except Exception as e:
self.logger.exception('method={} params={}\n{}'.format(
method, params, str(e)))
raise
finally:
del self._response_queues[seq_no]
class _Channel(object):
def __init__(self, from_fsm, to_fsm, tracer, queue=None):
if queue is None:
self.queue = Queue()
else:
self.queue = queue
self.from_fsm = from_fsm
self.to_fsm = to_fsm
self.tracer = tracer
def put(self, item):
self.tracer.send_trace_message(
messages.ChannelTrace(
self.tracer.trace_order_seq(),
self.from_fsm.fsm_id if self.from_fsm else None,
self.to_fsm.fsm_id if self.to_fsm else None,
item.__class__.__name__))
self.queue.put(item)
def get(self, block=True, timeout=None):
return self.queue.get(block, timeout)
receive = get
class Actor(gevent.Greenlet):
def __init__(self):
self.inbox = Queue()
gevent.Greenlet.__init__(self)
def on_message(self, message, answer_box=None):
""" Define in your subclass. """
raise NotImplemented()
def send_to(self, dest, message):
mail = Mail(self.inbox, message)
if isinstance(dest, Actor):
dest = dest.inbox
dest.put(mail)
def _run(self):
self.running = True
while self.running:
message = self.inbox.get()
if isinstance(message, Mail):
self.on_message(message.body, answer_box=message.answer_box)
else:
self.on_message(message)
def __bfs(self):
q = Queue()
# 消费者
def dispatch(step=0, args=tuple()):
job = self.jobs[step]
job_name = '%s%s' % (job.__name__, args)
logger.info('Dispatching jobs of %s.', job_name)
rv = job(*args)
if rv:
step += 1
for next_args in rv:
q.put((step, next_args))
logger.info('Jobs of %s have been dispatched.', job_name)
# 生产者
q.put((0, tuple()))
while True:
while not q.empty():
self.pool.spawn(dispatch, *q.get())
self.pool.join()
if q.empty():
break
class Importer(gevent.Greenlet):
def __init__(self):
self.inbox = Queue()
self._es = Elasticsearch({
"host": config['es_host'],
"port": config['es_port']
})
Greenlet.__init__(self)
def _process(self, message):
if message == "new_content":
content_importer = ContentLocalImporter()
content_importer.download_files()
else:
helpers.bulk(self._es, message)
def _run(self):
self.running = True
while self.running:
message = self.inbox.get()
self._process(message)
class GCMService(BaseService):
def __init__(self, api_key):
self.feedback_queue = Queue()
self.workers = [
GCMServiceWorker(i, api_key, self.feedback_queue)
for i in range(WORKER_COUNT)
]
self.next_worker = 0
def get_feedback(self, block=True, timeout=None):
return self.feedback_queue.get(block=block, timeout=timeout)
def queue_notification(self, notification):
self.workers[self.next_worker].queue_notification(notification)
self.next_worker = (self.next_worker + 1) % WORKER_COUNT
def start(self):
for w in self.workers:
w.start()
def stop(self, timeout=10.0):
for w in self.workers:
w.stop()
def test_concurrent_set_total_deposit(
token_network_proxy: TokenNetwork) -> None:
CHANNEL_COUNT = 3
DEPOSIT_COUNT = 5
channels = Queue()
def open_channel() -> None:
partner = factories.make_address()
settle_timeout = 500
given_block_identifier = BLOCK_ID_LATEST
channel_identifier, _, block_hash = token_network_proxy.new_netting_channel(
partner, settle_timeout, given_block_identifier)
channels.put((channel_identifier, block_hash, partner))
channel_grenlets = {
gevent.spawn(open_channel)
for _ in range(CHANNEL_COUNT)
}
deposit_greenlets = set()
for _ in range(CHANNEL_COUNT):
channel_identifier, block_hash, partner = channels.get()
for i in range(DEPOSIT_COUNT):
given_block_identifier = block_hash
total_deposit = i + 1
g = gevent.spawn(
token_network_proxy.approve_and_set_total_deposit,
given_block_identifier,
channel_identifier,
total_deposit,
partner,
)
deposit_greenlets.add(g)
all_greenlets: Set[Greenlet] = channel_grenlets.union(deposit_greenlets)
gevent.joinall(set(all_greenlets), raise_error=True)
class GeventActor(gevent.Greenlet):
def __init__(self, interval, name=''):
super(GeventActor, self).__init__()
self.inbox = G_Queue() # AsyncResult
self._interval = interval
self._timeout = None
self.name = name
def _work(self):
"""
Define in your subclass.
"""
raise NotImplemented()
def _run(self):
while True:
try:
msg = self.inbox.get(block=True, timeout=self._timeout)
if msg == 'stop':
self._timeout = None
elif msg == 'start':
self._timeout = self._interval
elif msg == 'exit':
break
except G_Empty:
self._work()
gevent.sleep(0)
def start(self):
self.inbox.put('start')
return super(GeventActor, self).start()
def stop(self):
self.inbox.put('stop')
def shutdown(self):
self.inbox.put('exit')
