def __init__( self ): self._canRead = Event() self._canWrite = Event() self._mutex = BoundedSemaphore( value = 1 ) self._readers = 0 self._isWriting = False self._canRead.set() self._canWrite.set()
def __init__(self, account_id, num_connections, readonly): log.info('Creating Crispin connection pool for account {} with {} ' 'connections'.format(account_id, num_connections)) self.account_id = account_id self.readonly = readonly self._queue = Queue(num_connections, items=num_connections * [None]) self._sem = BoundedSemaphore(num_connections) self._set_account_info()
def test_process_request(self): self.spider.settings.set("PROXY_LIST", ['124.88.67.54:80']) request = Request('http://httpbin.org/get') pm = ProxyMiddleware(self.spider.settings, self.spider.logger) dh = DownloadHandler(self.spider, None, BoundedSemaphore(1)) pm.process_request(request) response = dh.fetch(request) assert response.body
def __setitem__(self, key, factory): if not callable(factory): raise ValueError('value %s should be callable' % factory) lock = self.data[key] = BoundedSemaphore() lock.acquire() self.data[key] = factory() assert not isinstance(self.data[key], BoundedSemaphore) lock.release()
def __init__(self, media_environment): self.active_scripts = gevent.pool.Group() self.sleeping_scripts = [] self.project = None self.media_environment = media_environment # Get the script_lock before adding or removing scripts self.script_lock = BoundedSemaphore(1) self.clients = []
def configure_publish_lock(max_connections=0): global _publish_lock if _publish_lock is not None: raise RuntimeError("socket_lock already configured!") if max_connections < 1: _publish_lock = DummySemaphore() else: _publish_lock = BoundedSemaphore(max_connections)
def __init__(self, agent_addr, parallel_task_limit=None): self.agent_addr = agent_addr self.tasks = set() self.ptask_semaphore = None self.capacity = parallel_task_limit or self.cpu_count() self.ptask_semaphore = BoundedSemaphore(self.capacity) self.sync_tag = 0 self.sync_flag = False
def test_imap_message_deduplication(db, generic_account, inbox_folder, generic_trash_folder, mock_imapclient): uid = 22 uid_values = uid_data.example() mock_imapclient.list_folders = lambda: [(( '\\All', '\\HasNoChildren', ), '/', u'/Inbox'), (( '\\Trash', '\\HasNoChildren', ), '/', u'/Trash')] mock_imapclient.idle = lambda: None mock_imapclient.add_folder_data(inbox_folder.name, {uid: uid_values}) mock_imapclient.add_folder_data(generic_trash_folder.name, {uid: uid_values}) folder_sync_engine = FolderSyncEngine(generic_account.id, generic_account.namespace.id, inbox_folder.name, generic_account.email_address, 'custom', BoundedSemaphore(1), mock.Mock()) folder_sync_engine.initial_sync() trash_folder_sync_engine = FolderSyncEngine(generic_account.id, generic_account.namespace.id, generic_trash_folder.name, generic_account.email_address, 'custom', BoundedSemaphore(1), mock.Mock()) trash_folder_sync_engine.initial_sync() # Check that we have two uids, but just one message. assert [(uid, )] == db.session.query( ImapUid.msg_uid).filter(ImapUid.folder_id == inbox_folder.id).all() assert [(uid, )] == db.session.query(ImapUid.msg_uid).filter( ImapUid.folder_id == generic_trash_folder.id).all() # used to uniquely ID messages body_sha = sha256(uid_values['BODY[]']).hexdigest() assert db.session.query(Message).filter( Message.namespace_id == generic_account.namespace.id, Message.data_sha256 == body_sha).count() == 1
def __init__(self, config_file): u"""Cluster 抽象""" self.config_file = config_file self.config = Config(config_file) self.context = Context(self.config.parse_refer(), self.config.parse_registry()) self.clients = {} self.sem = BoundedSemaphore(1)
def __init__(self, *args, **kwargs): BaseProcessor.__init__(self, *args, **kwargs) self.build_args() # 缓存的城市信息 from gevent.lock import BoundedSemaphore self._city_cache = {} self._city_cache_lock = BoundedSemaphore(1)
def test_process_request(self): request = Request('http://httpbin.org/user-agent') self.assertIs(request.headers.get("User-Agent"), None) uam = UserAgentMiddleware(self.spider.settings, self.spider.logger) dh = DownloadHandler(self.spider, None, BoundedSemaphore(1)) uam.process_request(request) response = dh.fetch(request) self.assertEqual( json.loads(response.body)['user-agent'], request.headers['User-Agent'])
def _get_call_semaphore(self, call): if call.name.startswith("debug."): # XXX A bit of a hack return DummySemaphore() if self._call_semaphore is None: if self.max_concurrent_calls is None: semaphore = DummySemaphore() else: semaphore = BoundedSemaphore(self.max_concurrent_calls) self._call_semaphore = semaphore return self._call_semaphore
def __init__(self, account, heartbeat=1, refresh_frequency=30): self.refresh_frequency = refresh_frequency self.syncmanager_lock = BoundedSemaphore(1) self.saved_remote_folders = None self.sync_engine_class = FolderSyncEngine self.folder_monitors = Group() self.delete_handler = None BaseMailSyncMonitor.__init__(self, account, heartbeat)
def getGevent(): from gevent import monkey monkey.patch_all(socket=True, select=True) from gevent.queue import Queue #get,put from gevent.local import local try: from gevent.lock import BoundedSemaphore except: from gevent.coros import BoundedSemaphore sem = BoundedSemaphore(2) #acquire,release return local, Queue, sem #返回常量,队列,锁
def lock(self, key): """ Synchronization primitive for client functions to ensure that multiple functions are not trying to act on the same contract at the same time. """ try: sem = self._locks[key] except KeyError: sem = BoundedSemaphore() self._locks[key] = sem return sem
def __init__(self, path: str, name: str, host: str, port: int): """ Constructor """ self._name = name self.host = host self.port = port self.datadir = os.path.join(path, 'data') if not os.path.exists(self.datadir): os.mkdir(self.datadir) db_path = os.path.join(self.datadir, f'device_{self._name}.sqlite3') self.lock_db = BoundedSemaphore(1) self.db = sqlite3.connect(db_path)
def __init__(self, account_id, num_connections, readonly): log.info( "Creating Crispin connection pool", account_id=account_id, num_connections=num_connections, ) self.account_id = account_id self.readonly = readonly self._queue = Queue(num_connections, items=num_connections * [None]) self._sem = BoundedSemaphore(num_connections) self._set_account_info()
def __init__(self): BaseHardwareInterface.__init__(self) self.update_thread = None # Thread for running the main update loop self.pass_record_callback = None # Function added in server.py self.hardware_log_callback = None # Function added in server.py self.new_enter_or_exit_at_callback = None # Function added in server.py self.node_crossing_callback = None # Function added in server.py self.i2c = smbus.SMBus(1) # Start i2c bus self.semaphore = BoundedSemaphore(1) # Limits i2c to 1 read/write at a time self.i2c_timestamp = -1 # Scans all i2c_addrs to populate nodes array self.nodes = [] # Array to hold each node object i2c_addrs = [8, 10, 12, 14, 16, 18, 20, 22] # Software limited to 8 nodes for index, addr in enumerate(i2c_addrs): try: self.i2c.read_i2c_block_data(addr, READ_ADDRESS, 1) print "Node {0} found at address {1}".format(index+1, addr) gevent.sleep(I2C_CHILL_TIME) node = Node() # New node instance node.i2c_addr = addr # Set current loop i2c_addr node.index = index self.nodes.append(node) # Add new node to RHInterface except IOError as err: print "No node at address {0}".format(addr) gevent.sleep(I2C_CHILL_TIME) for node in self.nodes: node.frequency = self.get_value_16(node, READ_FREQUENCY) # read NODE_API_LEVEL and verification value: rev_val = self.get_value_16(node, READ_REVISION_CODE) if (rev_val >> 8) == 0x25: # if verify passed (fn defined) then set API level node.api_level = rev_val & 0xFF else: node.api_level = 0 # if verify failed (fn not defined) then set API level to 0 if node.api_level >= 10: node.api_valid_flag = True # set flag for newer API functions supported node.node_peak_rssi = self.get_value_16(node, READ_NODE_RSSI_PEAK) if node.api_level >= 13: node.node_nadir_rssi = self.get_value_16(node, READ_NODE_RSSI_NADIR) node.enter_at_level = self.get_value_16(node, READ_ENTER_AT_LEVEL) node.exit_at_level = self.get_value_16(node, READ_EXIT_AT_LEVEL) print "Node {0}: API_level={1}, Freq={2}, EnterAt={3}, ExitAt={4}".format(node.index+1, node.api_level, node.frequency, node.enter_at_level, node.exit_at_level) else: print "Node {0}: API_level={1}".format(node.index+1, node.api_level) if node.index == 0: if node.api_valid_flag: self.filter_ratio = self.get_value_16(node, READ_FILTER_RATIO) else: self.filter_ratio = 10 else: self.set_filter_ratio(node.index, self.filter_ratio)
def __init__(self, api: RaidenAPI, token_address: TokenAddress) -> None: assert isinstance(api, RaidenAPI) self.ready = Event() self.api = api self.token_address = token_address existing_channels = self.api.get_channel_list( api.raiden.default_registry.address, self.token_address ) open_channels = [ channel_state for channel_state in existing_channels if channel.get_status(channel_state) == ChannelState.STATE_OPENED ] if len(open_channels) == 0: token_proxy = self.api.raiden.proxy_manager.token(self.token_address) if not token_proxy.balance_of(self.api.raiden.address) > 0: raise ValueError( f"Not enough funds for echo node " f"{to_checksum_address(self.api.raiden.address)} for token " f"{to_checksum_address(self.token_address)}" ) # Using the balance of the node as funds funds = TokenAmount(token_proxy.balance_of(self.api.raiden.address)) self.api.token_network_connect( registry_address=self.api.raiden.default_registry.address, token_address=self.token_address, funds=funds, initial_channel_target=10, joinable_funds_target=0.5, ) self.num_seen_events = 0 self.received_transfers: Queue[EventPaymentReceivedSuccess] = Queue() # This is used to signal REMOVE_CALLBACK and stop echo_workers self.stop_signal: Optional[bool] = None self.greenlets: Set[Greenlet] = set() self.lock = BoundedSemaphore() self.seen_transfers: Deque[EventPaymentReceivedSuccess] = deque(list(), TRANSFER_MEMORY) self.num_handled_transfers = 0 self.lottery_pool = Queue() # register ourselves with the raiden alarm task self.api.raiden.alarm.register_callback(self.echo_node_alarm_callback) self.echo_worker_greenlet = gevent.spawn(self.echo_worker) log.info("Echo node started")
def __init__(self, cfg_file): if os.path.isfile(cfg_file): self._cfg_file = cfg_file self._init_cfg() else: raise FileNotFoundError('ERROR: can not find {}'.format(cfg_file)) self._setting = dict(self.config.items('setting')) if 'sem' in self._setting and self._setting['sem'].isdigit(): self._sem = BoundedSemaphore(int(self._setting['sem'])) self._pool = Pool() self._events = dict() self._node_events = {'begin': list(), 'final': list()}
def test_process_request_interval(self): self.spider.settings.set("PROXY_LIST", ['218.76.106.78:3128']) request = Request('http://httpbin.org/get') pm = ProxyMiddleware(self.spider.settings, self.spider.logger) dh = DownloadHandler(self.spider, None, BoundedSemaphore(1)) pm.process_request(request) time1 = time.time() dh.fetch(request) request = Request('http://httpbin.org/get') pm.process_request(request) self.assertGreater(time.time() - time1, 3)
def _buildSocket( self ): if self.s is not None: self.s.close( linger = 0 ) self.s = None self.s = self.ctx.socket( self._socketType ) self.s.set( zmq.LINGER, 0 ) if self._isBind: self.s.bind( self._url ) else: self.s.connect( self._url ) if self._isTransactionSocket: self._lock = BoundedSemaphore( 1 )
def __init__(self, discServer, zk_srv_ip='127.0.0.1', zk_srv_port='2181', reset_config=False): self._reset_config = reset_config self._service_id_to_type = {} self._ds = discServer self._zk_sem = BoundedSemaphore(1) self._election = None self._restarting = False zk_endpts = [] for ip in zk_srv_ip.split(','): zk_endpts.append('%s:%s' % (ip, zk_srv_port)) # logging logger = logging.getLogger('discovery-service') logger.setLevel(logging.WARNING) handler = logging.handlers.RotatingFileHandler( '/var/log/contrail/discovery_zk.log', maxBytes=1024 * 1024, backupCount=10) log_format = logging.Formatter('%(asctime)s [%(name)s]: %(message)s', datefmt='%m/%d/%Y %I:%M:%S %p') handler.setFormatter(log_format) logger.addHandler(handler) self._zk = kazoo.client.KazooClient( hosts=','.join(zk_endpts), handler=kazoo.handlers.gevent.SequentialGeventHandler(), logger=logger) self._logger = logger # connect self.connect() if reset_config: self.delete_node("/services", recursive=True) self.delete_node("/clients", recursive=True) self.delete_node("/election", recursive=True) # create default paths self.create_node("/services") self.create_node("/clients") self.create_node("/election") self._debug = { 'subscription_expires': 0, 'oos_delete': 0, 'db_excepts': 0, }
def test_controller(test_urls, writer_buff): tasks_gevent = [] error_cnt = mp.Manager().Value('i', 0) sem = BoundedSemaphore(1) l = test_urls.qsize() for i in range(l): tasks_gevent.append( gevent.spawn(test_scan, url=test_urls.get(), writer_buff=writer_buff, error_cnt=error_cnt, sem=sem)) gevent.joinall(tasks_gevent)
def test_gmail_message_deduplication(db, default_account, all_mail_folder, trash_folder, mock_imapclient): uid = 22 uid_values = uid_data.example() mock_imapclient.list_folders = lambda: [(( '\\All', '\\HasNoChildren', ), '/', u'[Gmail]/All Mail'), (( '\\Trash', '\\HasNoChildren', ), '/', u'[Gmail]/Trash')] mock_imapclient.idle = lambda: None mock_imapclient.add_folder_data(all_mail_folder.name, {uid: uid_values}) mock_imapclient.add_folder_data(trash_folder.name, {uid: uid_values}) all_folder_sync_engine = GmailFolderSyncEngine( default_account.id, default_account.namespace.id, all_mail_folder.name, all_mail_folder.id, default_account.email_address, 'gmail', BoundedSemaphore(1)) all_folder_sync_engine.initial_sync() trash_folder_sync_engine = GmailFolderSyncEngine( default_account.id, default_account.namespace.id, trash_folder.name, trash_folder.id, default_account.email_address, 'gmail', BoundedSemaphore(1)) trash_folder_sync_engine.initial_sync() # Check that we have two uids, but just one message. assert [(uid, )] == db.session.query( ImapUid.msg_uid).filter(ImapUid.folder_id == all_mail_folder.id).all() assert [(uid, )] == db.session.query( ImapUid.msg_uid).filter(ImapUid.folder_id == trash_folder.id).all() assert db.session.query(Message).filter( Message.namespace_id == default_account.namespace.id, Message.g_msgid == uid_values['X-GM-MSGID']).count() == 1
def __init__(self, log, mempool, params): self.log = log self.mempool = mempool self.params = params self.utxo_changes = 0 self.cache = Cache() self.cache.clear() ## level DB # pg_block: block data to insert into PG database # pg_tx: transaction data to insert into PG database # tx:* transaction outputs # misc:* state # height:* list of blocks at height h # blkmeta:* block metadata # blocks:* block seek point in stream datadir = '/data/explorer/blocks/' self.db = self.cache.db self.blk_write = io.BufferedWriter(io.FileIO(datadir + '/blocks.dat','ab')) self.blk_read = io.BufferedReader(io.FileIO(datadir + '/blocks.dat','rb')) if self.db.get(b'misc:height') is None: self.log.info('INITIALIZING EMPTY BLOCKCHAIN DATABASE') with self.db.write_batch(transaction=True) as batch: batch.put(b'misc:height', struct.pack('i', -1)) batch.put(b'misc:msg_start', self.params.NETMAGIC) batch.put(b'misc:tophash', ser_uint256(0)) batch.put(b'misc:total_work', b'0x0') start = self.db.get(b'misc:msg_start') if start != self.params.NETMAGIC: self.log.error("Database magic number mismatch. Data corruption or incorrect network?") raise RuntimeError self.block_lock = BoundedSemaphore() self.address_changes = {} self.address_change_count = 0 self.transaction_change_count = 0 self.utxo_cache = {} self.tx_lock = False self.initial_sync = True self.wallet_group = WalletGrouper('/data/explorer/wallets') self.checktransactions(True) self.checkaddresses(True) self.checkblocks(0, True) self.checkutxos(True) self.orphans = {} self.orphan_deps = {} if Block.select().count(None) == 0: self.log.info('Initialising genesis block') self.putblock(self.params.GENESIS_BLOCK)
def __init__(self, server_node, iface_cls, config): self._section_name = utils.get_module(__name__) self._logger = logging.getLogger(__name__) self._host = server_node.split(":")[0] self._port = int(server_node.split(":")[1]) self._iface_cls = iface_cls self._get_conn_timeout = config.getint( self._section_name, "pool_timeout", default=settings.DEFAULT_POOL_TIMEOUT) self._socket_timeout = config.getint( self._section_name, "request_timeout", default=settings.DEFAULT_REQUEST_TIMEOUT) * 1000 self._size = config.getint(self._section_name, "pool_size", default=settings.DEFAULT_POOL_SIZE) self._c_module_serialize = config.getboolean( self._section_name, "c_module_serialize", default=settings.USE_C_MODULE_SERIALIZE) self._closed = False if ASYNC_TAG: from gevent.lock import BoundedSemaphore from gevent import queue as Queue self._semaphore = BoundedSemaphore(self._size) self._connection_queue = Queue.LifoQueue(self._size) self._QueueEmpty = Queue.Empty else: from threading import BoundedSemaphore import Queue self._semaphore = BoundedSemaphore(self._size) self._connection_queue = Queue.LifoQueue(self._size) self._QueueEmpty = Queue.Empty
def __init__(self, f, block=True): if isinstance(f, io.IOBase): self.filename = f.name self.handle = f if not f.closed else open(f, "w") # noqa: SIM115 else: self.filename = f mkdirp(os.path.dirname(f)) self.handle = open(f, "w") # noqa: SIM115 if block: self.lock_op = fcntl.LOCK_EX else: self.lock_op = fcntl.LOCK_EX | fcntl.LOCK_NB self.block = block self.gevent_lock = BoundedSemaphore(1)
def __init__(self, process_identifier, process_number, poll_interval=SYNC_POLL_INTERVAL): self.host = platform.node() self.process_number = process_number self.process_identifier = process_identifier self.monitor_cls_for = { mod.PROVIDER: getattr(mod, mod.SYNC_MONITOR_CLS) for mod in module_registry.values() if hasattr(mod, 'SYNC_MONITOR_CLS') } for p_name, p in providers.iteritems(): if p_name not in self.monitor_cls_for: self.monitor_cls_for[p_name] = self.monitor_cls_for["generic"] self.log = get_logger() self.log.bind(process_number=process_number) self.log.info('starting mail sync process', supported_providers=module_registry.keys()) self.syncing_accounts = set() self.email_sync_monitors = {} self.contact_sync_monitors = {} self.event_sync_monitors = {} # Randomize the poll_interval so we maintain at least a little fairness # when using a timeout while blocking on the redis queues. min_poll_interval = 5 self.poll_interval = int((random.random() * (poll_interval - min_poll_interval)) + min_poll_interval) self.semaphore = BoundedSemaphore(1) self.zone = config.get('ZONE') # Note that we don't partition by zone for the private queues. # There's not really a reason to since there's one queue per machine # anyways. Also, if you really want to send an Account to a mailsync # machine in another zone you can do so. self.private_queue = EventQueue( SYNC_EVENT_QUEUE_NAME.format(self.process_identifier)) self.queue_group = EventQueueGroup([ shared_sync_event_queue_for_zone(self.zone), self.private_queue, ]) self.stealing_enabled = config.get('SYNC_STEAL_ACCOUNTS', True) self._pending_avgs_provider = None self.last_unloaded_account = time.time()
def __init__(self, tasks, result, pool=None): self._tasks = tasks self._result = result self.timeout = 5 self.headers = { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.321.132 Safari/537.36', 'Accept': '*/*', 'Pragma': 'no-cache', 'Cache-control': 'no-cache', 'Referer': 'https://www.google.com/' } self.pool = pool self.lock = BoundedSemaphore() self.prepare() # must run after init