class DownloadByGevent(GetTitleUrls): def __init__(self, download_urls): super(DownloadByGevent, self).__init__() self.workQueue = Queue(1000) self.download_urls = download_urls def crawler(self, index): Process_id = 'Process-' + str(index) while not self.workQueue.empty(): url = self.workQueue.get(timeout=2) try: r = self.get_soup(url, timeout=20) my_write(str(' '.join([str(x) for x in [Process_id, self.workQueue.qsize(), r.status_code, url]]))) except Exception as e: # print(Process_id, self.workQueue.qsize, url, 'Error: ', e) my_write(str(' '.join([str(x) for x in [Process_id, self.workQueue.qsize(), r.status_code, url]]))) def boss(self): for url in self.download_urls: self.workQueue.put_nowait(url) def start(self): start_time = time.time() gevent.spawn(self.boss).join() jobs = [] for i in range(10): jobs.append(gevent.spawn(self.crawler, i)) gevent.joinall(jobs) print('end') print("爬虫时间为%s"%time.time() - start_time)
def test_add_to_retry_queue(self): retry_items_queue = Queue() worker = ResourceItemWorker( config_dict=self.worker_config, retry_resource_items_queue=retry_items_queue) retry_item = { 'id': uuid.uuid4().hex, 'dateModified': datetime.datetime.utcnow().isoformat(), } self.assertEqual(retry_items_queue.qsize(), 0) # Add to retry_resource_items_queue worker.add_to_retry_queue(retry_item) sleep(worker.config['retry_default_timeout'] * 2) self.assertEqual(retry_items_queue.qsize(), 1) retry_item_from_queue = retry_items_queue.get() self.assertEqual(retry_item_from_queue['retries_count'], 1) self.assertEqual(retry_item_from_queue['timeout'], worker.config['retry_default_timeout'] * 2) # Add to retry_resource_items_queue with status_code '429' worker.add_to_retry_queue(retry_item, status_code=429) retry_item_from_queue = retry_items_queue.get() self.assertEqual(retry_item_from_queue['retries_count'], 1) self.assertEqual(retry_item_from_queue['timeout'], worker.config['retry_default_timeout'] * 2) # Drop from retry_resource_items_queue retry_item['retries_count'] = 3 worker.add_to_retry_queue(retry_item) self.assertEqual(retry_items_queue.qsize(), 0) del worker
def test_concurrent_rw(self): written_data = Queue() def _writer(_buffer): while True: data = b"".join([ ascii_letters[m].encode() for m in [randrange(0, 8) for _ in range(8)] ]) _buffer.write(data) written_data.put(data) sleep(0.2) writer = spawn(_writer, self.buffer) writer.start() sleep(0.5) data = self.buffer.read() _data = b"" while written_data.qsize() != 0: _data += written_data.get() self.assertEqual(data, _data) sleep(0.5) data = self.buffer.read() _data = b"" while written_data.qsize() != 0: _data += written_data.get() self.assertEqual(data, _data) writer.kill() writer.get()
class FilterTests(TestCase): def create_stage(self, **conf): return self.cls(**conf) def create(self, conf={}, events=[]): self.input = Queue() self.output = Queue() with DummyContext(): self.i = self.create_stage(**conf) self.input = self.i.setup(self.output) self.i.start() for ev in events: self.input.put(ev) return self.i def wait(self, timeout=1.0, events=1): with gevent.Timeout(timeout): # wait for input to be consumed and output to be produced while self.input.qsize(): gevent.sleep(0.0) while self.output.qsize() < events: gevent.sleep(0.0) if events: return [self.output.get() for n in xrange(events)] def tearDown(self): self.i.stop()
class FilterTests(TestCase): def create_stage(self, **conf): return self.cls(**conf) def create(self, conf={}, events=[]): self.input = Queue() self.output = Queue() with DummyContext(): self.i = self.create_stage(**conf) self.input = self.i.setup(self.output) self.i.start() for ev in events: self.input.put(ev) return self.i def wait(self, timeout=1.0, events=1): with gevent.Timeout(timeout): # wait for input to be consumed and output to be produced while self.input.qsize(): gevent.sleep(0.0) while self.output.qsize() < events: gevent.sleep(0.0) if events: return [self.output.get() for n in xrange(events)] def tearDown(self): self.i.stop()
class FlowTests(TestCase): def create(self, conf={}, events=[]): self.input = Queue() self.output = Queue() context = DummyContext() with context: self.i = self.create_stage(**conf) self.input = self.i.setup(self.output) self.assertEquals(1, len(context.stages)) self.i.start() for ev in events: self.input.put(ev) return self.i def wait(self, timeout=1.0, events=1): with gevent.Timeout(timeout): # wait for input to be consumed and output to be produced while self.input.qsize(): gevent.sleep(0.0) while self.output.qsize() < events: gevent.sleep(0.0) self.i.stop() if events: return [self.output.get() for n in xrange(events)]
class FlowTests(TestCase): def create(self, conf={}, events=[]): self.input = Queue() self.output = Queue() context = DummyContext() with context: self.i = self.create_stage(**conf) self.input = self.i.setup(self.output) self.assertEquals(1, len(context.stages)) self.i.start() for ev in events: self.input.put(ev) return self.i def wait(self, timeout=1.0, events=1): with gevent.Timeout(timeout): # wait for input to be consumed and output to be produced while self.input.qsize(): gevent.sleep(0.0) while self.output.qsize() < events: gevent.sleep(0.0) self.i.stop() if events: return [self.output.get() for n in xrange(events)]
class ConnectionQueue(object): """ Holds connections to resources. Each time it's called a connection is fetched from its underlying queue assuming any connection is still available. """ def __init__(self, pool_size, queue_build_cap, conn_name, conn_type, address, add_client_func): self.queue = Queue(pool_size) self.queue_build_cap = queue_build_cap self.conn_name = conn_name self.conn_type = conn_type self.address = address self.add_client_func = add_client_func self.logger = logging.getLogger(self.__class__.__name__) def __call__(self): return _Connection(self.queue, self.conn_name) def put_client(self, client): self.queue.put(client) self.logger.info('Added `%s` client to %s (%s)', self.conn_name, self.address, self.conn_type) def build_queue(self): """ Spawns greenlets to populate the queue and waits up to self.queue_build_cap seconds until the queue is full. If it never is, raises an exception stating so. """ for x in range(self.queue.maxsize): gevent.spawn(self.add_client_func) start = datetime.utcnow() build_until = start + timedelta(seconds=self.queue_build_cap) while not self.queue.full(): gevent.sleep(0.5) now = datetime.utcnow() if now >= build_until: self.logger.error( 'Built %s/%s %s clients to `%s` within %s seconds, giving up', self.queue.qsize(), self.queue.maxsize, self.conn_type, self.address, self.queue_build_cap) return self.logger.info( '%d/%d %s clients connected to `%s` (%s) after %s (cap: %ss)', self.queue.qsize(), self.queue.maxsize, self.conn_type, self.address, self.conn_name, now - start, self.queue_build_cap) self.logger.info('Obtained %d %s clients to `%s` for `%s`', self.queue.maxsize, self.conn_type, self.address, self.conn_name)
def main(psize, filename=None): if filename: urls = Queue() results = Queue() pool = Pool(int(psize)) reader = gevent.spawn(readfile, filename, urls) request = gevent.spawn(work_input_file, urls, results, reader) pool.add(reader) pool.add(request) pool.join() pool.free_count() print results.qsize(), 3333333333333333333 print urls.qsize(), 3333333333333333333 return results
class ASyncHandler(Group): def __init__(self, master, count=100, waitout=0.1): super(ASyncHandler, self).__init__( master, log=master.log, count=count, timeout=1) self.master = master self.queue = Queue() self.waitout = waitout self._size = 0 self._last = 0 def size(self): if time.time() - self._last < 1: return self._size self._last = time.time() self._size = self.queue.qsize() return self._size def do(self, func, *args, **kwargs): if self.size() < 500: self.queue.put((func, args, kwargs)) else: func(*args, **kwargs) def quick(self, limit=50): count = 0 while count < limit: try: func, args, kwargs = self.queue.get_nowait() func(*args, **kwargs) count += 1 except Empty: break def handle(self): count = 0 while True: try: func, args, kwargs = self.queue.get() func(*args, **kwargs) count += 1 except Empty: break self.wait(self.waitout) def on_quit(self): if not self.queue.empty(): self.log.info('%s - has %d tasks not done. now doing ...' % (self.name, self.queue.qsize()))
def test_put_failure(self): queue = Queue(1) # unbounded queue.put(object()) assert_that(queue.qsize(), is_(1)) transaction.begin() put_nowait(queue, self) # still size 1 assert_that(queue.qsize(), is_(1)) with self.assertRaises(Full) as cm: transaction.commit() assert_that(cm.exception, is_(Full)) assert_that(queue.get(block=False), is_(object))
def main(psize, filename=None): psize = int(psize) if filename: urls = Queue() results = Queue() reader = gevent.spawn(readfile, filename, urls) jobs = [gevent.spawn(work_input_file, urls, results, reader) for i in xrange(psize)] print jobs, 11111111111 gevent.joinall(jobs) print results.qsize(), 3333333333333333333 print urls.qsize(), 3333333333333333333 return results else: pool = Pool(psize) urls = ['http://www.baidu.com'] * 100 return pool.map(work, urls)
class Scheduler(object): """ Scheduler """ def __init__(self, spider): self.request_filter = RequestFilter() self.queue = Queue() self.settings = spider.settings self.timeout = self.settings.get('TIMEOUT', 5) self.download_delay = self.settings.get('DOWNLOAD_DELAY', 0) self.logger = spider.logger def enqueue_request(self, request): """put request """ if self.request_filter.request_seen(request): self.logger.debug("ignore %s", request.url) return self.queue.put(request) def next_request(self): """next request """ gevent.sleep(self.download_delay) return self.queue.get(timeout=self.timeout * 3) def __len__(self): return self.queue.qsize()
def batch_request(url, reqs, config={}, workers_num=1): if "interval_time" not in config: config["interval_time"] = None if "headers" not in config: config["headers"] = {} elif "content-type" in config["headers"]: if config["headers"]["content-type"] == "application/json": reqs = [json.dumps(req).strip() for req in reqs] if "method" not in config: config["method"] = "POST" if "expected" not in config: config["expected"] = "True" reqs_queue = Queue() for req in reqs: reqs_queue.put_nowait(req) count = reqs_queue.qsize() workers = [gevent.spawn(request_worker, url, reqs_queue, config, worker=worker) for worker in range(workers_num)] start = time() # send requests at the same time gevent.joinall(workers) # Response.elapsed for a single request, offered by requests print "total requests number: %d" % (count) print "total elapsed time: %s\n" % (time()-start)
class SyncBaseDeamon(object): def __init__(self, interval=5): self.ip_addr = socket.gethostbyname(socket.gethostname()) self.pid = os.getpid() self.channel = Queue(1) self.sync_interval = int(interval) self.exit_flag = Event() self.lock_path = "" self.logger = logging.getLogger() def exit(self): if not self.exit_flag.isSet(): self.exit_flag.set() self.channel.put('exit') else: self.logger.warn("Waiting exit signal been resolved") def terminate(self, code): zk.close() sys.exit(code) def run(self): greenlets = [] try: zk.create( self.lock_path, "%s:%s" % (self.ip_addr, self.pid), ephemeral=True, makepath=True ) except NodeExistsError: ret, _ = zk.get(self.lock_path) ip, pid = ret.split(':') self.logger.error("Servers syncing process is running on %s, pid: %s" % (ip, pid)) self.terminate(EXIT_CODES['leave_alone']) gevent.signal(signal.SIGTERM, self.exit) gevent.signal(signal.SIGINT, self.exit) gevent.signal(signal.SIGHUP, self.exit) greenlets.append(gevent.spawn(self.sync)) gevent.spawn(self.ticker) gevent.joinall(greenlets) self.logger.warn("syncing deamon exited") self.terminate(EXIT_CODES['normal_exit']) def ticker(self): while True: try: if self.channel.qsize() == 0: self.channel.put("sync", block=False) except Full: self.logger.error("something wrong occur in syncing thread") pass gevent.sleep(self.sync_interval) def sync(self, test=False): pass
class BlackBerryPushService(object): def __init__(self, app_id, password, push_url): self.app_id = app_id self.password = password self.push_url = push_url self._send_queue = Queue() self._send_queue_cleared = Event() self.log = logging.getLogger('pulsus.service.bbp') def _send_loop(self): self._send_greenlet = gevent.getcurrent() try: self.log.info("BlackBerry Push service started") while True: notification = self._send_queue.get() try: self._do_push(notification) except Exception, e: print e self._send_queue.put(notification) gevent.sleep(5.0) finally: if self._send_queue.qsize() < 1 and \ not self._send_queue_cleared.is_set(): self._send_queue_cleared.set()
class ConnectionPool: def __init__(self, db_config, time_to_sleep=30, test_run=False): self.username = db_config.get('user') self.password = db_config.get('password') self.host = db_config.get('host') self.port = int(db_config.get('port')) self.max_pool_size = 20 self.test_run = test_run self.pool = None self.time_to_sleep = time_to_sleep self._initialize_pool() def get_initialized_connection_pool(self): return self.pool def _initialize_pool(self): self.pool = Queue(maxsize=self.max_pool_size) current_pool_size = self.pool.qsize() if current_pool_size < self.max_pool_size: # this is a redundant check, can be removed for _ in xrange(0, self.max_pool_size - current_pool_size): try: conn = db.connect(host=self.host, user=self.username, passwd=self.password, port=self.port) self.pool.put_nowait(conn) except db.OperationalError, e: LOGGER.error( "Cannot initialize connection pool - retrying in {} seconds" .format(self.time_to_sleep)) LOGGER.exception(e) break self._check_for_connection_loss()
class C2DMService(object): def __init__(self, source, email, password): self.source = source self.email = email self.password = password self._send_queue = Queue() self._send_queue_cleared = Event() self.log = logging.getLogger('pulsus.service.c2dm') def _send_loop(self): self._send_greenlet = gevent.getcurrent() try: self.log.info("C2DM service started") while True: notification = self._send_queue.get() try: self._do_push(notification) except Exception, e: self.log.exception("Error while pushing") self._send_queue.put(notification) gevent.sleep(5.0) finally: if self._send_queue.qsize() < 1 and \ not self._send_queue_cleared.is_set(): self._send_queue_cleared.set()
class NotifyingQueue(Event): def __init__(self): super(NotifyingQueue, self).__init__() self._queue = Queue() def put(self, item): """ Add new item to the queue. """ self._queue.put(item) self.set() def get(self, block=True, timeout=None): """ Removes and returns an item from the queue. """ value = self._queue.get(block, timeout) if self._queue.empty(): self.clear() return value def peek(self, block=True, timeout=None): return self._queue.peek(block, timeout) def __len__(self): return len(self._queue) def copy(self): """ Copies the current queue items. """ return [self.peek(i) for i in range(self._queue.qsize())]
class Transport(object): def __init__(self, tid, name): self.__tid = tid self.__name = name self.q_maxsize = 100 self.__task_queue = Queue(maxsize=self.q_maxsize) def add_task(self, task): if self.__task_queue.qsize() < self.q_maxsize: self.__task_queue.put_nowait(task) return True else: # queue is full logger.debug('The task queue is full') return False def get(self, timeout=None): task = None try: task = self.__task_queue.get(block=True, timeout=timeout) except Exception as e: if not self.__task_queue.empty(): logger.error('Transport Thread still in processing data') finally: return task @property def name(self): return str(self.__name)
def test_put_failure( self ): queue = Queue(1) # unbounded queue.put( object() ) assert_that( queue.qsize(), is_( 1 ) ) transaction.begin() put_nowait( queue, self ) # still size 1 assert_that( queue.qsize(), is_( 1 ) ) with self.assertRaises( Full ) as cm: transaction.commit() assert_that( cm.exception, is_( Full ) ) assert_that( queue.get(block=False), is_( object ) )
class ConnectionPool: def __init__(self, db_config, time_to_sleep=30, test_run=False): self.username = db_config.get('user') self.password = db_config.get('password') self.host = db_config.get('host') self.port = int(db_config.get('port')) self.max_pool_size = 20 self.test_run = test_run self.pool = None self.time_to_sleep = time_to_sleep self._initialize_pool() def get_initialized_connection_pool(self): return self.pool def _initialize_pool(self): self.pool = Queue(maxsize=self.max_pool_size) current_pool_size = self.pool.qsize() if current_pool_size < self.max_pool_size: # this is a redundant check, can be removed for _ in xrange(0, self.max_pool_size - current_pool_size): try: conn = db.connect(host=self.host, user=self.username, passwd=self.password, port=self.port) self.pool.put_nowait(conn) except db.OperationalError, e: LOGGER.error("Cannot initialize connection pool - retrying in {} seconds".format(self.time_to_sleep)) LOGGER.exception(e) break self._check_for_connection_loss()
class BlackBerryPushService(object): def __init__(self, app_id, password, push_url): self.app_id = app_id self.password = password self.push_url = push_url self._send_queue = Queue() self._send_queue_cleared = Event() self.log = logging.getLogger('pulsus.service.bbp') def _send_loop(self): self._send_greenlet = gevent.getcurrent() try: self.log.info("BlackBerry Push service started") while True: notification = self._send_queue.get() try: self._do_push(notification) except Exception, e: print e self._send_queue.put(notification) gevent.sleep(5.0) finally: if self._send_queue.qsize() < 1 and \ not self._send_queue_cleared.is_set(): self._send_queue_cleared.set()
class C2DMService(object): def __init__(self, source, email, password): self.source = source self.email = email self.password = password self._send_queue = Queue() self._send_queue_cleared = Event() self.log = logging.getLogger('pulsus.service.c2dm') def _send_loop(self): self._send_greenlet = gevent.getcurrent() try: self.log.info("C2DM service started") while True: notification = self._send_queue.get() try: self._do_push(notification) except Exception, e: self.log.exception("Error while pushing") self._send_queue.put(notification) gevent.sleep(5.0) finally: if self._send_queue.qsize() < 1 and \ not self._send_queue_cleared.is_set(): self._send_queue_cleared.set()
class Scheduler(object): """ Scheduler """ def __init__(self): self.request_filter = RequestFilter() self.queue = Queue() def enqueue_request(self, request): """put request """ if not request.dont_filter \ and self.request_filter.request_seen(request): logger.warn("ignore %s", request.url) return self.queue.put(request) def next_request(self): """next request """ if self.queue.empty(): return None return self.queue.get() def __len__(self): return self.queue.qsize()
class DBConnectionPool(object): """ The Connection Pool "Classic" pool of connections with connection lifecycle management """ def __init__(self, dsn, db_module='psycopg2', pool_size=10, conn_lifetime=600, do_log=False): """ :param string dsn: DSN for the default `class:DBConnectionPool` :param string db_module: name of the DB-API module to use :param int pool_size: Poolsize of the first/default `class:DBConnectionPool` :param int conn_lifetime: Number of seconds after which a connection will be recycled when :meth:`.put` back :param bool do_log: Log to the console or not """ if do_log: import logging logging.basicConfig(level=logging.INFO, format="%(asctime)s %(message)s") self.logger = logging.getLogger() self.do_log = do_log self.dsn = dsn self.db_module = db_module self.pool_size = pool_size self.CONN_RECYCLE_AFTER = conn_lifetime if conn_lifetime is not None else 0 self.pool = Queue(self.pool_size) __import__(db_module) self.connection_jobs = map( lambda x: gevent.spawn(self.create_connection), xrange(self.pool_size)) try: gevent.joinall(self.connection_jobs, timeout=10) assert self.pool_size == self.pool.qsize() if self.do_log: self.logger.info("$ poolsize: %i" % self.pool.qsize()) self.ready = True except AssertionError, e: raise DBPoolConnectionException( "Could not get %s connections for the pool as requested. %s" % (self.pool_size, e.message)) except Exception, e: raise e
class ConnectionQueue(object): """ Holds connections to resources. Each time it's called a connection is fetched from its underlying queue assuming any connection is still available. """ def __init__(self, pool_size, queue_build_cap, conn_name, conn_type, address, add_client_func): self.queue = Queue(pool_size) self.queue_build_cap = queue_build_cap self.conn_name = conn_name self.conn_type = conn_type self.address = address self.add_client_func = add_client_func self.logger = logging.getLogger(self.__class__.__name__) def __call__(self): return _Connection(self.queue, self.conn_name) def put_client(self, client): self.queue.put(client) self.logger.info('Added `%s` client to %s (%s)', self.conn_name, self.address, self.conn_type) def build_queue(self): """ Spawns greenlets to populate the queue and waits up to self.queue_build_cap seconds until the queue is full. If it never is, raises an exception stating so. """ for x in range(self.queue.maxsize): gevent.spawn(self.add_client_func) start = datetime.utcnow() build_until = start + timedelta(seconds=self.queue_build_cap) while not self.queue.full(): gevent.sleep(0.5) now = datetime.utcnow() if now >= build_until: self.logger.error('Built %s/%s %s clients to `%s` within %s seconds, giving up', self.queue.qsize(), self.queue.maxsize, self.conn_type, self.address, self.queue_build_cap) return self.logger.info('%d/%d %s clients connected to `%s` (%s) after %s (cap: %ss)', self.queue.qsize(), self.queue.maxsize, self.conn_type, self.address, self.conn_name, now - start, self.queue_build_cap) self.logger.info('Obtained %d %s clients to `%s` for `%s`', self.queue.maxsize, self.conn_type, self.address, self.conn_name)
def salt_top_run(request): center_server = request.POST.get('center_server') run_target = request.POST.get('run_target') state = request.POST.get('state') for server in center_server.split('|'): if not check_center_server_up(CENTER_SERVER[server][0],CENTER_SERVER[server][1]): return HttpResponse(json.dumps({'code':1,'msg':u'无法连接到%s' % server, 'cmd_results':''}),content_type="application/json") run_target_dict = {} target = [] cmd_results = '' time_now = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') for i in zip(center_server.split('|'),run_target.split('|'),state.split('|')): if not run_target_dict.has_key(i[0]): run_target_dict[i[0]] = [] run_target_dict[i[0]].append((i[1],i[2])) for i in run_target_dict.keys(): master_dir = commands.getoutput('''ssh %s "grep -A2 '^file_roots' /etc/salt/master |grep 'base:' -A1|grep '-'|cut -d'-' -f2"''' % CENTER_SERVER[i][0]) content = 'base:\n' for j in run_target_dict[i]: for n in j[0].split(','): if not n in target: target.append(n) cmd = u'state模块 < %s >' % j[1] log.objects.create(source_ip=n,username=request.user.username,command=cmd,time=time_now) else: return HttpResponse(json.dumps({'code':1,'msg':u'目标主机不能重复','cmd_results':cmd_results}),content_type="application/json") content += " '%s':\n" % n for m in j[1].split(','): content += ' - %s\n' % m content += 'EOF' os.system('''ssh %s "cat > %s/top.sls << EOF\n%s"''' % (CENTER_SERVER[i][0],master_dir,content)) try: def gevent_run_all(CENTER_SERVER,client_send_data,p,q): for i in run_target_dict.keys(): for j in run_target_dict[i]: p.spawn(gevent_run,CENTER_SERVER,client_send_data,i,j[0],q) def gevent_run(CENTER_SERVER,client_send_data,i,j,q): cmd_result = client_send_data(json.dumps({'salt':1,'act':'state.highstate','hosts':j,'argv':''}),CENTER_SERVER[i][0],CENTER_SERVER[i][1]) cmd_result = convert_str_to_html(cmd_result) q.put(cmd_result) p = Pool() q = Queue() p.spawn(gevent_run_all,CENTER_SERVER,client_send_data,p,q) p.join() for i in range(q.qsize()): cmd_result = q.get() if not cmd_results: cmd_results = cmd_result else: cmd_results = cmd_results + '<br><br><br><br>' + cmd_result return HttpResponse(json.dumps({'code':0,'msg':u'模块执行完成','cmd_results':cmd_results}),content_type="application/json") except Exception,e: return HttpResponse(json.dumps({'code':1,'msg':u'模块执行失败'}),content_type="application/json")
class QueueStatReporter(StatReporter): def _start_queue(self): self.queue = Queue() def _flush_queue(self, exit_exc=None, caller=None): sleep(1) self.logger.info("Flushing a queue of size {}" .format(self.queue.qsize())) self.queue.put(StopIteration) for item in self.queue: self._run_queue_item(item) self.logger.info("Queue flush complete, Exit.") @loop(setup='_start_queue', fin='_flush_queue') def _queue_proc(self): item = self.queue.get() if self._run_queue_item(item) == "retry": # Put it at the back of the queue for retry self.queue.put(item) sleep(1) def _run_queue_item(self, item): name, args, kwargs = item if __debug__: self.logger.debug("Queue running {} with args '{}' kwargs '{}'" .format(name, args, kwargs)) try: func = getattr(self, name, None) if func is None: raise NotImplementedError( "Item {} has been enqueued that has no valid function!" .format(name)) func(*args, **kwargs) except self.queue_exceptions as e: self.logger.error("Unable to process queue item, retrying! " "{} Name: {}; Args: {}; Kwargs: {};" .format(e, name, args, kwargs)) return "retry" except Exception: # Log any unexpected problem, but don't retry because we might # end up endlessly retrying with same failure self.logger.error("Unkown error, queue data discarded!" "Name: {}; Args: {}; Kwargs: {};" .format(name, args, kwargs), exc_info=True) def log_one_minute(self, *args, **kwargs): self.queue.put(("_queue_log_one_minute", args, kwargs)) def add_block(self, *args, **kwargs): self.queue.put(("_queue_add_block", args, kwargs)) def _queue_add_block(self, address, height, total_subsidy, fees, hex_bits, hex_hash, currency, algo, merged=False, worker=None, **kwargs): raise NotImplementedError def _queue_log_one_minute(self, address, worker, algo, stamp, typ, amount): raise NotImplementedError
class Grep(object): """ 用于启动并管理grep的子进程 """ def __init__(self, log_path, content, timeout): self._log_path = log_path # 文件路径 self._time_out = timeout # 子进程timeout self._content = content # 需要grep的内容 self._process = None # 用于引用创建的子进程 self._datas = Queue(maxsize=10) # grep出来的数据将会放到这里来等待获取 self._over = False # 用于标记子进程是否已经执行完毕了 self._is_time_out = False # 用于标记子进程是否执行超时 gevent.spawn(self._start) # 在一个单独的协程中启动一个子进程 def _start(self): """ 这个方法需要在一个协程中单独的执行,用于启动一个子进程,然后执行grep方法,并通过过标准输出来获取数据 通过一个超时了控制子进程的运行时间,防止grep执行过久,占用太多系统资源,在超时之后要保证启动的grep进程被关闭 """ try: with gevent.timeout.Timeout(self._time_out): p = subprocess.Popen(["grep", self._content, self._log_path], stdout=subprocess.PIPE) self._process = p while 1: line = p.stdout.readline() if not line: # 这个一般是子进程退出了 break self._datas.put(line) except: """ 如果发发生了异常,那么判断异常的类型,如果是超时异常,那么需要设置超时标志位 """ ex = sys.exc_info()[1] if isinstance(ex, gevent.Timeout): logging.error(u"grep执行超时 %s:%s", self._log_path, self._content) self._is_time_out = True finally: if self._process and self._process.returncode is None: self._process.kill() self._over = True def get_data(self): """ 获取已经grep出来的数据,如果没有数据,那么返回None,同时需要返回启动的子进程是否已经结束,是否是超时 """ if self._datas.qsize() > 0: out = self._datas self._datas = Queue(maxsize=10) out_data = [] while out.qsize() > 0: out_data.append(out.get()) return out_data, self._over, self._is_time_out else: return None, self._over, self._is_time_out
class EventPersister(StandaloneProcess): def on_init(self): # Time in between event persists self.persist_interval = 1.0 # Holds received events FIFO self.event_queue = Queue() # Temporarily holds list of events to persist while datastore operation not yet completed self.events_to_persist = None # bookkeeping for timeout greenlet self._persist_greenlet = None self._terminate_persist = Event() # when set, exits the timeout greenlet # The event subscriber self.event_sub = None def on_start(self): # Persister thread self._persist_greenlet = spawn(self._trigger_func, self.persist_interval) log.debug('Publisher Greenlet started in "%s"' % self.__class__.__name__) # Event subscription self.event_sub = EventSubscriber(pattern=EventSubscriber.ALL_EVENTS, callback=self._on_event) self.event_sub.start() def on_quit(self): # Stop event subscriber self.event_sub.stop() # tell the trigger greenlet we're done self._terminate_persist.set() # wait on the greenlet to finish cleanly self._persist_greenlet.join(timeout=10) def _on_event(self, event, *args, **kwargs): self.event_queue.put(event) def _trigger_func(self, persist_interval): log.debug('Starting event persister thread with persist_interval=%s', persist_interval) # Event.wait returns False on timeout (and True when set in on_quit), so we use this to both exit cleanly and do our timeout in a loop while not self._terminate_persist.wait(timeout=persist_interval): try: self.events_to_persist = [self.event_queue.get() for x in xrange(self.event_queue.qsize())] self._persist_events(self.events_to_persist) self.events_to_persist = None except Exception as ex: log.exception("Failed to persist received events") return False def _persist_events(self, event_list): if event_list: bootstrap.container_instance.event_repository.put_events(event_list)
def parse_results_queue(results_queue: Queue): results = defaultdict(lambda: defaultdict(list)) while results_queue.qsize(): result = results_queue.get() for param_name, value in result.items(): url, reasons, type, response = value['url'], value['reasons'], value['type'], value['response'] results[url][type].append({'param': param_name, 'reasons': reasons, 'response': response}) return results
class NotificationService(object): def __init__(self, sandbox = True, **kwargs): if "certfile" not in kwargs: raise ValueError, u"Must specify a PEM bundle." self._sslargs = kwargs self._push_connection = None self._feedback_connection = None self._sandbox = sandbox self._send_queue = Queue() self._error_queue = Queue() self._feedback_queue = Queue() self._send_greenlet = None self._error_greenlet = None self._feedback_greenlet = None self._send_queue_cleared = Event() def _check_send_connection(self): if self._push_connection is None: s = ssl.wrap_socket(socket(AF_INET, SOCK_STREAM, 0), ssl_version=ssl.PROTOCOL_SSLv3, **self._sslargs) addr = ["gateway.push.apple.com", 2195] if self._sandbox: addr[0] = "gateway.sandbox.push.apple.com" s.connect_ex(tuple(addr)) self._push_connection = s self._error_greenlet = gevent.spawn(self._error_loop) def _check_feedback_connection(self): if self._feedback_connection is None: s = ssl.wrap_socket(socket(AF_INET, SOCK_STREAM, 0), ssl_version = ssl.PROTOCOL_SSLv3, **self._sslargs) addr = ["feedback.push.apple.com", 2196] if self._sandbox: addr[0] = "feedback.sandbox.push.apple.com" s.connect_ex(tuple(addr)) self._feedback_connection = s def _send_loop(self): self._send_greenlet = gevent.getcurrent() try: while True: msg = self._send_queue.get() self._check_send_connection() try: self._push_connection.send(str(msg)) except Exception, e: self._send_queue.put(msg) self._push_connection.close() self._push_connection = None gevent.sleep(5.0) finally: if self._send_queue.qsize() < 1 and \ not self._send_queue_cleared.is_set(): self._send_queue_cleared.set()
class Grep(object): """ 用于启动并管理grep的子进程 """ def __init__(self, log_path, content, timeout): self._log_path = log_path # 文件路径 self._time_out = timeout # 子进程timeout self._content = content # 需要grep的内容 self._process = None # 用于引用创建的子进程 self._datas = Queue(maxsize=10) # grep出来的数据将会放到这里来等待获取 self._over = False # 用于标记子进程是否已经执行完毕了 self._is_time_out = False # 用于标记子进程是否执行超时 gevent.spawn(self._start) # 在一个单独的协程中启动一个子进程 def _start(self): """ 这个方法需要在一个协程中单独的执行,用于启动一个子进程,然后执行grep方法,并通过过标准输出来获取数据 通过一个超时了控制子进程的运行时间,防止grep执行过久,占用太多系统资源,在超时之后要保证启动的grep进程被关闭 """ try: with gevent.timeout.Timeout(self._time_out): p = subprocess.Popen(["grep", self._content, self._log_path], stdout=subprocess.PIPE) self._process = p while 1: line = p.stdout.readline() if not line: # 这个一般是子进程退出了 break self._datas.put(line) except: """ 如果发发生了异常,那么判断异常的类型,如果是超时异常,那么需要设置超时标志位 """ ex = sys.exc_info()[1] if isinstance(ex, gevent.Timeout): logging.error(u"grep执行超时 %s:%s", self._log_path, self._content) self._is_time_out = True finally: if self._process and self._process.returncode is None: self._process.kill() self._over = True def get_data(self): """ 获取已经grep出来的数据,如果没有数据,那么返回None,同时需要返回启动的子进程是否已经结束,是否是超时 """ if self._datas.qsize() > 0: out = self._datas self._datas = Queue(maxsize=10) out_data = [] while out.qsize() > 0: out_data.append(out.get()) return out_data, self._over, self._is_time_out else: return None, self._over, self._is_time_out
class NotificationService(object): def __init__(self, sandbox = True, **kwargs): if "certfile" not in kwargs: raise ValueError, u"Must specify a PEM bundle." self._sslargs = kwargs self._push_connection = None self._feedback_connection = None self._sandbox = sandbox self._send_queue = Queue() self._error_queue = Queue() self._feedback_queue = Queue() self._send_greenlet = None self._error_greenlet = None self._feedback_greenlet = None self._send_queue_cleared = Event() def _check_send_connection(self): if self._push_connection is None: s = ssl.wrap_socket(socket(AF_INET, SOCK_STREAM, 0), ssl_version=ssl.PROTOCOL_SSLv3, **self._sslargs) addr = ["gateway.push.apple.com", 2195] if self._sandbox: addr[0] = "gateway.sandbox.push.apple.com" s.connect_ex(tuple(addr)) self._push_connection = s self._error_greenlet = gevent.spawn(self._error_loop) def _check_feedback_connection(self): if self._feedback_connection is None: s = ssl.wrap_socket(socket(AF_INET, SOCK_STREAM, 0), ssl_version = ssl.PROTOCOL_SSLv3, **self._sslargs) addr = ["feedback.push.apple.com", 2196] if self._sandbox: addr[0] = "feedback.sandbox.push.apple.com" s.connect_ex(tuple(addr)) self._feedback_connection = s def _send_loop(self): self._send_greenlet = gevent.getcurrent() try: while True: msg = self._send_queue.get() self._check_send_connection() try: self._push_connection.send(str(msg)) except Exception, e: self._send_queue.put(msg) self._push_connection.close() self._push_connection = None gevent.sleep(5.0) finally: if self._send_queue.qsize() < 1 and \ not self._send_queue_cleared.is_set(): self._send_queue_cleared.set()
class AsyncPool(object): __slots__ = ["sockets", "socket_factory", "pool_size", "log", "local", "pid" ] def __init__(self, socket_factory, pool_size): self.pid = os.getpid() self.pool_size = pool_size self.socket_factory = socket_factory self.sockets = Queue() self.log = logging.getLogger('ming.async.AsyncPool') self.local = local() def _get_sock(self): return getattr(self.local, 'sock', None) def _set_sock(self, value): self.local.sock = value sock = property(_get_sock, _set_sock) def socket(self): pid = os.getpid() if pid != self.pid: self.sock = None self.sockets = Queue() self.pid = pid if self.sock is not None: self.log.debug('Return existing socket to greenlet %s', gevent.getcurrent() ) return self.sock gl = gevent.getcurrent() try: self.sock = self.sockets.get_nowait() self.log.debug('Checkout socket %s to greenlet %s', self.sock, gl ) except Empty: self.sock = self.socket_factory() self.log.debug('Create socket in greenlet %s', gl) self.sock.last_greenlet = gl return self.sock def return_socket(self): if self.sock is None: self.log.debug('No socket to return from greenlet %s', gevent.getcurrent() ) return if self.sockets.qsize() < self.pool_size: gl = gevent.getcurrent() self.log.debug('Checkin socket %s from greenlet %s', self.sock, gl) self.sockets.put(self.sock) self.sock = None else: self.log.debug('Close socket in greenlet %s', gevent.getcurrent() ) self.sock.close() self.sock = None self.local.sock = None
def test_put_succeeds(self): queue = Queue() # unbounded transaction.begin() put_nowait(queue, self) # still empty assert_that(queue.qsize(), is_(0)) transaction.commit() assert_that(queue.get(block=False), is_(self))
def test_put_succeeds(self): queue = Queue() # unbounded transaction.begin() put_nowait( queue, self ) # still empty assert_that( queue.qsize(), is_( 0 ) ) transaction.commit() assert_that( queue.get(block=False), is_( self ) )
def test_clear_api_client_queue(self): queue = Queue() client_dict = {'id': uuid.uuid4().hex, 'client': None} client_dict2 = {'id': uuid.uuid4().hex, 'client': None} clients_info = { client_dict['id']: { 'destroy': False }, client_dict2['id']: { 'destroy': True } } queue.put(client_dict) queue.put(client_dict2) self.assertEqual(queue.qsize(), 2) clear_api_client_queue(queue, clients_info) self.assertEqual(queue.qsize(), 1) client_dict_from_queue = queue.get() self.assertEqual(client_dict, client_dict_from_queue)
def save_and_convert(self, items_queue, requests_queue): # can not use ItemQueue cause ItemQueue was singleton tmp_items_queue = Queue() # each round just parse 5 items while not items_queue.empty(): if tmp_items_queue.qsize() < 5: tmp_items_queue.put(items_queue.get(block=True, timeout=3)) else: break # save and convert async self._save_and_convert_async(items_queue=tmp_items_queue, requests_queue=requests_queue)
def test_put_transaction_abort(self): queue = Queue() transaction.begin() put_nowait(queue, 'aborted') transaction.abort() transaction.begin() put_nowait(queue, 'committed') transaction.commit() assert_that(queue.qsize(), is_(1)) assert_that(queue.get(block=False), is_('committed'))
def test_put_transaction_abort(self): queue = Queue() transaction.begin() put_nowait( queue, 'aborted' ) transaction.abort() transaction.begin() put_nowait( queue, 'committed' ) transaction.commit() assert_that( queue.qsize(), is_( 1 ) ) assert_that( queue.get( block=False ), is_( 'committed' ) )
class DBConnectionPool( object ): """ The Connection Pool "Classic" pool of connections with connection lifecycle management """ def __init__( self, dsn, db_module = 'psycopg2', pool_size = 10, conn_lifetime = 600, do_log = False ): """ :param string dsn: DSN for the default `class:DBConnectionPool` :param string db_module: name of the DB-API module to use :param int pool_size: Poolsize of the first/default `class:DBConnectionPool` :param int conn_lifetime: Number of seconds after which a connection will be recycled when :meth:`.put` back :param bool do_log: Log to the console or not """ if do_log: import logging logging.basicConfig( level = logging.INFO, format = "%(asctime)s %(message)s" ) self.logger = logging.getLogger() self.do_log = do_log self.dsn = dsn self.db_module = db_module self.pool_size = pool_size self.CONN_RECYCLE_AFTER = conn_lifetime if conn_lifetime is not None else 0 self.pool = Queue( self.pool_size ) __import__( db_module ) self.connection_jobs = map( lambda x: gevent.spawn( self.create_connection ), xrange( self.pool_size ) ) try: gevent.joinall( self.connection_jobs, timeout = 10 ) assert self.pool_size == self.pool.qsize() if self.do_log: self.logger.info( "$ poolsize: %i" % self.pool.qsize() ) self.ready = True except AssertionError, e: raise DBPoolConnectionException( "Could not get %s connections for the pool as requested. %s" % ( self.pool_size, e.message ) ) except Exception, e: raise e
class ReusableCursor(Greenlet): def __init__(self, pool, key, sql, values): super(ReusableCursor, self).__init__(self.work) self.pool = pool self.key = self._formatted_info = key self.sql = sql self.values = values self.offset = 0 self.queue = Queue() self._count = AsyncResult() self.last_access = time.time() self.idle = False self.listeners = [] self.window = config.xgetint('web', 'query-reusable-window', 30) @property def count(self): return self._count.get() def work(self): try: with self.pool.connection() as conn: cur = conn.cursor('_cur') cur.execute(self.sql, self.values) logging.debug(cur.query) cur_tmp = conn.cursor() cur_tmp.execute('MOVE ALL FROM _cur') self._count.set(int(cur_tmp.statusmessage.split()[-1])) cur_tmp.close() cur.scroll(0, 'absolute') while True: if not self.queue.qsize(): self.idle = True for l in self.listeners: spawn(l.onIdle, self) result, limit, offset = self.queue.get(timeout=self.window) self.idle = False if limit is None: raise Killed(result) if self.offset != offset: cur.scroll(offset, 'absolute') data = cur.fetchmany(limit) self.offset = offset + limit result.set(data) self.last_access = time.time() except Empty: pass except Killed, k: k.result.set() finally:
class Producer(object): def __init__(self, queue_size=2048): self.Q = Queue(maxsize=queue_size) def make(self): pass def run(self): items = self.make() if not items: return for item in items: self.Q.put(item) def sync(self, qsize=0, interval=5): while self.Q.qsize() > qsize: time.sleep(interval)
class ConnectionPool(object): def __init__(self, connection_cls, maxsize=100, **kwargs): if not isinstance(maxsize, integer_types): raise TypeError('Expected integer, got %r' % (maxsize, )) self._connection_cls = connection_cls self._maxsize = maxsize self._pool = Queue() self._size = 0 self._conn_params = kwargs def get(self): if self._size >= self._maxsize or self._pool.qsize(): return self._pool.get() else: self._size += 1 try: return self._connection_cls(**self._conn_params) except: self._size -= 1 raise def put(self, item): self._pool.put(item) def closeall(self): while not self._pool.empty(): conn = self._pool.get_nowait() try: conn.close() except Exception: pass @contextlib.contextmanager def connection(self): conn = self.get() try: yield conn except: if conn.closed: conn = None self.closeall() raise finally: if conn is not None and not conn.closed: self.put(conn)
class InputTests(TestCase): def create(self, conf): self.output = Queue() with DummyContext(): self.i = i = self.cls(**conf) i.setup(self.output) i.start() return i def waitForQueue(self, timeout=1.0, events=1): with gevent.Timeout(timeout): while self.output.qsize() < events: gevent.sleep(0.0) self.i.stop() if events: return [self.output.get() for n in xrange(events)]
class InputTests(TestCase): def create(self, conf): self.output = Queue() with DummyContext(): self.i = i = self.cls(**conf) i.setup(self.output) i.start() return i def waitForQueue(self, timeout=1.0, events=1): with gevent.Timeout(timeout): while self.output.qsize() < events: gevent.sleep(0.0) self.i.stop() if events: return [self.output.get() for n in xrange(events)]
class LimitedIMapUnordered(pool.IMapUnordered): def __init__(self, func, iterable, max_queue, spawn=None): pool.IMapUnordered.__init__(self, func, iterable, spawn) self.queue = Queue(max_queue) self.max_queue = max_queue def _run(self): try: func = self.func for item in self.iterable: while self.queue.qsize() + self.count + 2 == self.max_queue: gevent.sleep(0.1) self.count += 1 self.spawn(func, item).rawlink(self._on_result) finally: self.__dict__.pop('spawn', None) self.__dict__.pop('func', None) self.__dict__.pop('iterable', None)
def test_put_transaction_savepoint(self): queue = Queue() transaction.begin() put_nowait( queue, 'presavepoint' ) # we can get a non-optimistic savepoint savepoint = transaction.savepoint(optimistic=False) assert_that( savepoint._savepoints, has_length( 1 ) ) repr(savepoint._savepoints) # cover put_nowait( queue, 'aftersavepoint' ) # If we rollback the savepoint now, what we just # did will be lost, but the original work # will still happen savepoint.rollback() transaction.commit() assert_that( queue.qsize(), is_( 1 ) ) assert_that( queue.get( block=False ), is_( 'presavepoint' ) )
def test_put_transaction_savepoint(self): queue = Queue() transaction.begin() put_nowait(queue, 'presavepoint') # we can get a non-optimistic savepoint savepoint = transaction.savepoint(optimistic=False) assert_that(savepoint._savepoints, has_length(1)) repr(savepoint._savepoints) # cover put_nowait(queue, 'aftersavepoint') # If we rollback the savepoint now, what we just # did will be lost, but the original work # will still happen savepoint.rollback() transaction.commit() assert_that(queue.qsize(), is_(1)) assert_that(queue.get(block=False), is_('presavepoint'))
class LimitedIMapUnordered(pool.IMapUnordered): def __init__(self, func, iterable, max_queue, spawn=None): pool.IMapUnordered.__init__(self, func, iterable, spawn) self.queue = Queue(max_queue) self.max_queue = max_queue def _run(self): try: func = self.func for item in self.iterable: while self.queue.qsize() + self.count + 2 == self.max_queue: gevent.sleep(0.1) self.count += 1 self.spawn(func, item).rawlink(self._on_result) finally: self.__dict__.pop('spawn', None) self.__dict__.pop('func', None) self.__dict__.pop('iterable', None)
class ZMQSummarizedTestResult(ZMQTestResult): def __init__(self, args): super(ZMQSummarizedTestResult, self).__init__(args) self.interval = 1. self._data = Queue() gevent.spawn_later(self.interval, self._dump_data) def push(self, data_type, **data): self._data.put_nowait((data_type, data)) def close(self): while not self._data.empty(): self._dump_data(loop=False) self.context.destroy() def _dump_data(self, loop=True): if self._data.empty() and loop: gevent.spawn_later(self.interval, self._dump_data) return data = { 'data_type': 'batch', 'agent_id': self.agent_id, 'hostname': get_hostname(), 'run_id': self.run_id, 'counts': defaultdict(list) } # grabbing what we have for _ in range(self._data.qsize()): data_type, message = self._data.get() data['counts'][data_type].append(message) while True: try: self._push.send(self.encoder.encode(data), zmq.NOBLOCK) break except zmq.ZMQError as e: if e.errno in (errno.EAGAIN, errno.EWOULDBLOCK): continue else: raise if loop: gevent.spawn_later(self.interval, self._dump_data)
def parse_and_convert(self, pages_queue, items_queue): """ parse page and create Item and send to items queue :param pages_queue: pages queue use to get Page :param items_queue: items queue use to receive Item :return: None """ tmp_pages_queue ,save_data_urls_list = Queue(), [] # each round just parse 5 pages while not pages_queue.empty(): if tmp_pages_queue.qsize() < 5: tmp_pages_queue.put(pages_queue.get(block=True, timeout=3)) else: break # TODO: i should find out should parse page using gevent # extract self._extract_async(pages_queue=tmp_pages_queue, save_data_urls_list=save_data_urls_list) # convert self._convert_to_item(save_data_urls_list=save_data_urls_list, items_queue=items_queue)
class ZMQSummarizedTestResult(ZMQTestResult): def __init__(self, args): super(ZMQSummarizedTestResult, self).__init__(args) self.interval = 1. self._data = Queue() gevent.spawn_later(self.interval, self._dump_data) def push(self, data_type, **data): self._data.put_nowait((data_type, data)) def close(self): while not self._data.empty(): self._dump_data(loop=False) self.context.destroy() def _dump_data(self, loop=True): if self._data.empty() and loop: gevent.spawn_later(self.interval, self._dump_data) return data = {'data_type': 'batch', 'agent_id': self.agent_id, 'hostname': get_hostname(), 'run_id': self.run_id, 'counts': defaultdict(list)} # grabbing what we have for _ in range(self._data.qsize()): data_type, message = self._data.get() data['counts'][data_type].append(message) while True: try: self._push.send(self.encoder.encode(data), zmq.NOBLOCK) break except zmq.ZMQError as e: if e.errno in (errno.EAGAIN, errno.EWOULDBLOCK): continue else: raise if loop: gevent.spawn_later(self.interval, self._dump_data)
def setup_bucket_sizes(self): """ Устанавливает размер порций для всех запросов """ args_queue = Queue() # Запускаем на один и тот же запрос разные работы for info in self.info_list: for finder in self.finders: if not finder.is_info_searchable(info): continue args_queue.put((finder.determine_bucket_size, info)) # Запускаем воркеры workers = [ SetBucketWorker(args_queue, self.logger) for _ in range(self.threads) ] greenlets = [gevent.spawn(worker.run) for worker in workers] # Ждем заверщения работы while any([worker.is_running() for worker in workers]) or args_queue.qsize(): gevent.sleep(0) # Выключаем воркеры for worker in workers: worker.finish() # Ждем выключения gevent.joinall(greenlets) # Устанавливаем размеры порций for info in self.info_list: for finder in self.finders: if not finder.is_info_searchable(info): continue finder.set_bucket_size(info) self.logger.debug( f'{finder.__class__.__name__}: {info.origin_url} - размер порции {finder.get_bucket_size(info)}' )
def download_and_convert(self, requests_queue, pages_queue): """ Download page using requests or selenium downloader :param requests_queue: requests queue :param pages_queue: pages queue :return: None """ tmp_requests_queue, tmp_selenium_queue, url_page_list = Queue(), Queue(), list() while not requests_queue.empty(): request = requests_queue.get(block=True, timeout=3) if "SELENIUM" not in request.method: tmp_requests_queue.put(request) else: tmp_selenium_queue.put(request) if tmp_selenium_queue.qsize() == 5 or tmp_requests_queue.qsize() == 5: # TODO: 5 is the max tmp queue size break # do requests download self._async_requests_download(tmp_requests_queue=tmp_requests_queue, url_page_list=url_page_list) # do selenium download # TODO: in windows seem like can run this, try on linux # self._linear_selenium_download(tmp_selenium_queue=tmp_selenium_queue, url_page_list=url_page_list) self._convert_and_send_to_page_queue(url_page_list=url_page_list, pages_queue=pages_queue)
class MemoryScheduler(Scheduler): def __init__(self, scheduler_name): self._scheduler_key = 'Scheduler:{scheduler_name}'.format( scheduler_name=scheduler_name) self._queue = Queue() @classmethod def from_spider(cls, spider): scheduler_name = spider.settings.get('SPIDER_NAME') return cls(scheduler_name) def __len__(self): return self._queue.qsize() def push(self, r): self._queue.put_nowait(r) def pull(self, count=DEFAULT_SCHEDULER_PULL_COUNT, timeout=DEFAULT_SCHEDULER_PULL_TIMEOUT): try: return self._queue.get(timeout=timeout) except Empty: raise SchedulerEmpty
class ChainService(WiredService): """ Manages the chain and requests to it. """ # required by BaseService name = 'chain' default_config = dict(eth=dict(privkey_hex='')) # required by WiredService wire_protocol = eth_protocol.ETHProtocol # create for each peer # initialized after configure: chain = None genesis = None synchronizer = None config = None block_queue_size = 1024 transaction_queue_size = 1024 def __init__(self, app): self.config = app.config self.db = app.services.db assert self.db is not None super(ChainService, self).__init__(app) log.info('initializing chain') self.chain = Chain(self.db, new_head_cb=self._on_new_head) self.synchronizer = Synchronizer(self, force_sync=None) self.chain.coinbase = privtoaddr(self.config['eth']['privkey_hex'].decode('hex')) self.block_queue = Queue(maxsize=self.block_queue_size) self.transaction_queue = Queue(maxsize=self.transaction_queue_size) self.add_blocks_lock = False self.broadcast_filter = DuplicatesFilter() def _on_new_head(self, block): pass def add_block(self, t_block, proto): "adds a block to the block_queue and spawns _add_block if not running" self.block_queue.put((t_block, proto)) # blocks if full if not self.add_blocks_lock: self.add_blocks_lock = True gevent.spawn(self._add_blocks) def _add_blocks(self): log.debug('add_blocks', qsize=self.block_queue.qsize()) try: while not self.block_queue.empty(): t_block, proto = self.block_queue.get() if t_block.header.hash in self.chain: log.warn('known block', block=t_block) continue if t_block.header.prevhash not in self.chain: log.warn('missing parent', block=t_block) continue if not t_block.header.check_pow(): log.warn('invalid pow', block=t_block) # FIXME ban node continue try: # deserialize st = time.time() block = t_block.to_block(db=self.chain.db) elapsed = time.time() - st log.debug('deserialized', elapsed='%.2fs' % elapsed, gas_used=block.gas_used, gpsec=int(block.gas_used / elapsed)) except processblock.InvalidTransaction as e: log.warn('invalid transaction', block=t_block, error=e) # FIXME ban node continue if self.chain.add_block(block): log.debug('added', block=block) gevent.sleep(0.001) finally: self.add_blocks_lock = False def broadcast_newblock(self, block, chain_difficulty, origin=None): assert isinstance(block, eth_protocol.TransientBlock) if self.broadcast_filter.known(block.header.hash): log.debug('already broadcasted block') else: log.debug('broadcasting newblock', origin=origin) bcast = self.app.services.peermanager.broadcast bcast(eth_protocol.ETHProtocol, 'newblock', args=(block, chain_difficulty), num_peers=None, exclude_protos=[origin]) # wire protocol receivers ########### def on_wire_protocol_start(self, proto): log.debug('on_wire_protocol_start', proto=proto) assert isinstance(proto, self.wire_protocol) # register callbacks proto.receive_status_callbacks.append(self.on_receive_status) proto.receive_transactions_callbacks.append(self.on_receive_transactions) proto.receive_getblockhashes_callbacks.append(self.on_receive_getblockhashes) proto.receive_blockhashes_callbacks.append(self.on_receive_blockhashes) proto.receive_getblocks_callbacks.append(self.on_receive_getblocks) proto.receive_blocks_callbacks.append(self.on_receive_blocks) proto.receive_newblock_callbacks.append(self.on_receive_newblock) # send status head = self.chain.head proto.send_status(chain_difficulty=head.chain_difficulty(), chain_head_hash=head.hash, genesis_hash=self.chain.genesis.hash) def on_wire_protocol_stop(self, proto): assert isinstance(proto, self.wire_protocol) log.debug('on_wire_protocol_stop', proto=proto) def on_receive_status(self, proto, eth_version, network_id, chain_difficulty, chain_head_hash, genesis_hash): log.debug('status received', proto=proto, eth_version=eth_version) assert eth_version == proto.version, (eth_version, proto.version) if network_id != proto.network_id: log.warn("invalid network id", remote_id=proto.network_id, network_id=network_id) raise eth_protocol.ETHProtocolError('wrong network_id') # check genesis if genesis_hash != self.chain.genesis.hash: log.warn("invalid genesis hash", remote_id=proto, genesis=genesis_hash.encode('hex')) raise eth_protocol.ETHProtocolError('wrong genesis block') # request chain self.synchronizer.receive_status(proto, chain_head_hash, chain_difficulty) # send transactions transactions = self.chain.get_transactions() if transactions: log.debug("sending transactions", remote_id=proto) proto.send_transactions(*transactions) # transactions def on_receive_transactions(self, proto, transactions): "receives rlp.decoded serialized" log.debug('remote_transactions_received', count=len(transactions), remote_id=proto) log.debug('skipping, FIXME') return for tx in transactions: # fixme bloomfilter self.chain.add_transaction(tx) # blockhashes ########### def on_receive_getblockhashes(self, proto, child_block_hash, count): log.debug("handle_get_blockhashes", count=count, block_hash=encode_hex(child_block_hash)) max_hashes = min(count, self.wire_protocol.max_getblockhashes_count) found = [] if child_block_hash not in self.chain: log.debug("unknown block") proto.send_blockhashes(*[]) return last = child_block_hash while len(found) < max_hashes: last = rlp.decode_lazy(self.chain.db.get(last))[0][0] if last: found.append(last) else: break log.debug("sending: found block_hashes", count=len(found)) proto.send_blockhashes(*found) def on_receive_blockhashes(self, proto, blockhashes): if blockhashes: log.debug("on_receive_blockhashes", count=len(blockhashes), remote_id=proto, first=encode_hex(blockhashes[0]), last=encode_hex(blockhashes[-1])) else: log.debug("recv 0 remote block hashes, signifying genesis block") self.synchronizer.receive_blockhashes(proto, blockhashes) # blocks ################ def on_receive_getblocks(self, proto, blockhashes): log.debug("on_receive_getblocks", count=len(blockhashes)) found = [] for bh in blockhashes[:self.wire_protocol.max_getblocks_count]: try: found.append(self.chain.db.get(bh)) except KeyError: log.debug("unknown block requested", block_hash=encode_hex(bh)) if found: log.debug("found", count=len(found)) proto.send_blocks(*found) def on_receive_blocks(self, proto, transient_blocks): log.debug("recv blocks", count=len(transient_blocks), remote_id=proto, highest_number=max(x.header.number for x in transient_blocks)) if transient_blocks: self.synchronizer.receive_blocks(proto, transient_blocks) def on_receive_newblock(self, proto, block, chain_difficulty): log.debug("recv newblock", block=block, remote_id=proto) self.synchronizer.receive_newblock(proto, block, chain_difficulty)
class Emotiv(object): """ Receives, decrypts and stores packets received from Emotiv Headsets. """ def __init__(self, display_output=False, serial_number="", is_research=False): """ Sets up initial values. """ self.running = True self.packets = Queue() self.packets_received = 0 self.packets_processed = 0 self.battery = 0 self.display_output = display_output self.is_research = is_research self.sensors = { 'F3': {'value': 0, 'quality': 0}, 'FC6': {'value': 0, 'quality': 0}, 'P7': {'value': 0, 'quality': 0}, 'T8': {'value': 0, 'quality': 0}, 'F7': {'value': 0, 'quality': 0}, 'F8': {'value': 0, 'quality': 0}, 'T7': {'value': 0, 'quality': 0}, 'P8': {'value': 0, 'quality': 0}, 'AF4': {'value': 0, 'quality': 0}, 'F4': {'value': 0, 'quality': 0}, 'AF3': {'value': 0, 'quality': 0}, 'O2': {'value': 0, 'quality': 0}, 'O1': {'value': 0, 'quality': 0}, 'FC5': {'value': 0, 'quality': 0}, 'X': {'value': 0, 'quality': 0}, 'Y': {'value': 0, 'quality': 0}, 'Unknown': {'value': 0, 'quality': 0} } self.serial_number = serial_number # You will need to set this manually for OS X. self.old_model = False def setup(self): """ Runs setup function depending on platform. """ print system_platform + " detected." if system_platform == "Windows": self.setup_windows() elif system_platform == "Linux": self.setup_posix() elif system_platform == "Darwin": self.setup_darwin() def setup_windows(self): """ Setup for headset on the Windows platform. """ devices = [] try: for device in hid.find_all_hid_devices(): if device.vendor_id != 0x21A1 and device.vendor_id != 0xED02: continue if device.product_name == 'Brain Waves': devices.append(device) device.open() self.serial_number = device.serial_number device.set_raw_data_handler(self.handler) elif device.product_name == 'EPOC BCI': devices.append(device) device.open() self.serial_number = device.serial_number device.set_raw_data_handler(self.handler) elif device.product_name == '00000000000': devices.append(device) device.open() self.serial_number = device.serial_number device.set_raw_data_handler(self.handler) elif device.product_name == 'Emotiv RAW DATA': devices.append(device) device.open() self.serial_number = device.serial_number device.set_raw_data_handler(self.handler) crypto = gevent.spawn(self.setup_crypto, self.serial_number) console_updater = gevent.spawn(self.update_console) while self.running: try: gevent.sleep(0) except KeyboardInterrupt: self.running = False finally: for device in devices: device.close() gevent.kill(crypto, KeyboardInterrupt) gevent.kill(console_updater, KeyboardInterrupt) def handler(self, data): """ Receives packets from headset for Windows. Sends them to a Queue to be processed by the crypto greenlet. """ assert data[0] == 0 tasks.put_nowait(''.join(map(chr, data[1:]))) self.packets_received += 1 return True def setup_posix(self): """ Setup for headset on the Linux platform. Receives packets from headset and sends them to a Queue to be processed by the crypto greenlet. """ _os_decryption = False if os.path.exists('/dev/eeg/raw'): # The decryption is handled by the Linux epoc daemon. We don't need to handle it. _os_decryption = True hidraw = open("/dev/eeg/raw") else: serial, hidraw_filename = get_linux_setup() self.serial_number = serial if os.path.exists("/dev/" + hidraw_filename): hidraw = open("/dev/" + hidraw_filename) else: hidraw = open("/dev/hidraw4") crypto = gevent.spawn(self.setup_crypto, self.serial_number) console_updater = gevent.spawn(self.update_console) while self.running: try: data = hidraw.read(32) if data != "": if _os_decryption: self.packets.put_nowait(EmotivPacket(data)) else: #Queue it! self.packets_received += 1 tasks.put_nowait(data) gevent.sleep(0) else: # No new data from the device; yield # We cannot sleep(0) here because that would go 100% CPU if both queues are empty gevent.sleep(DEVICE_POLL_INTERVAL) except KeyboardInterrupt: self.running = False hidraw.close() if not _os_decryption: gevent.kill(crypto, KeyboardInterrupt) gevent.kill(console_updater, KeyboardInterrupt) def setup_darwin(self): """ Setup for headset on the OS X platform. Receives packets from headset and sends them to a Queue to be processed by the crypto greenlet. """ # Set this to True if the OS is performing the encryption of the packets _os_decryption = False # Change these values to the hex equivalent from the output of hid_enumerate. If they are incorrect. # Current values = VendorID: 8609 ProductID: 1 #hidraw = hid.device(0x31a1, 0x2001) hidraw = hid.device(0x1234,0xed02) hidraw.open(0x1234, 0xed02) self.serial_number = 'SN20120229000290' if not hidraw: hidraw = hid.device(0x21a1, 0x1234) if not hidraw: hidraw = hid.device(0xed02, 0x1234) if not hidraw: print "Device not found. Uncomment the code in setup_darwin and modify hid.device(vendor_id, product_id)" raise ValueError if self.serial_number == "": print "Serial number needs to be specified manually in __init__()." raise ValueError print "Serial number:" + self.serial_number crypto = gevent.spawn(self.setup_crypto, self.serial_number) console_updater = gevent.spawn(self.update_console) zero = 0 while self.running: try: # Doesn't seem to matter how big we make the buffer 32 returned every time, 33 for other platforms data = hidraw.read(34,10) #data = [48]*32 if len(data) == 32: # Most of the time the 0 is truncated? That's ok we'll add it... data = [zero] + data if data != "": if _os_decryption: self.packets.put_nowait(EmotivPacket(data)) else: #Queue it! print ('Queuing package:'+len(data)) tasks.put_nowait(''.join(map(chr, data[1:]))) self.packets_received += 1 print ('Waiting...') gevent.sleep(0.01) else: # No new data from the device; yield # We cannot sleep(0) here because that would go 100% CPU if both queues are empty. gevent.sleep(DEVICE_POLL_INTERVAL) except KeyboardInterrupt: self.running = False hidraw.close() gevent.kill(crypto, KeyboardInterrupt) gevent.kill(console_updater, KeyboardInterrupt) def setup_crypto(self, sn): """ Performs decryption of packets received. Stores decrypted packets in a Queue for use. """ if is_old_model(sn): self.old_model = True print self.old_model k = ['\0'] * 16 k[0] = sn[-1] k[1] = '\0' k[2] = sn[-2] if self.is_research: k[3] = 'H' k[4] = sn[-1] k[5] = '\0' k[6] = sn[-2] k[7] = 'T' k[8] = sn[-3] k[9] = '\x10' k[10] = sn[-4] k[11] = 'B' else: k[3] = 'T' k[4] = sn[-3] k[5] = '\x10' k[6] = sn[-4] k[7] = 'B' k[8] = sn[-1] k[9] = '\0' k[10] = sn[-2] k[11] = 'H' k[12] = sn[-3] k[13] = '\0' k[14] = sn[-4] k[15] = 'P' key = ''.join(k) iv = Random.new().read(AES.block_size) cipher = AES.new(key, AES.MODE_ECB, iv) print ("Encryption...") for i in k: print "0x%.02x " % (ord(i)) while self.running: print ("Check encryption queue...") while not tasks.empty(): task = tasks.get() try: print ("Adding packets to queue...:"+str(self.packets.qsize())) data = cipher.decrypt(task[:16]) + cipher.decrypt(task[16:]) print ("Data received:"+str(len(data))) self.packets.put_nowait(EmotivPacket(data, self.sensors, self.old_model)) self.packets_processed += 1 print ("Packets added..") except Exception as ex: print ("Exception raied..:"+str(ex)) pass gevent.sleep(0.01) gevent.sleep(0.01) def dequeue(self): """ Returns an EmotivPacket popped off the Queue. """ try: print ("Returning something...:"+str(self.packets.empty())) return self.packets.get() except Exception, e: print e