class Task(object): task_id = 0 lock = Lock() def __init__(self, runner, *task_args, **task_kw): assert callable(runner) self.runner = runner self.task_args = task_args self.task_kw = task_kw self.name = self.__generate_name() def __generate_name(self): from gevent.threading import Lock Task.lock.acquire() Task.task_id += 1 _task_id = Task.task_id Task.lock.release() return "task_{}".format(_task_id) def execute(self): self.runner(*self.task_args, **self.task_kw) def __str__(self): return u"task({}) runner:'{}', args:'{}', kw:'{}'"\ .format(self.name, self.runner.__name__, self.task_args, self.task_kw)
def __init__(self): """ Const """ # Gevent self._gevent_pool_max = 1024 self._gevent_locker = Lock() self._gevent_pool = dict() # urllib3 # Force underlying fifo queue to 1024 via maxsize self._u3_basic_pool = PoolManager(num_pools=1024, maxsize=1024) self._u3_proxy_pool_max = 1024 self._u3_proxy_locker = Lock() self._u3_proxy_pool = dict()
def __init__(self, ipc_path=None, testnet=False, *args, **kwargs): if ipc_path is None: self.ipc_path = get_default_ipc_path(testnet) else: self.ipc_path = ipc_path self._lock = Lock() super(IPCProvider, self).__init__(*args, **kwargs)
def __init__(self, name, bind_address, *, master_address, driver_port: int): super().__init__(name, bind_address, master_address=master_address) self.driver = None self.driver_port = driver_port self.last_handle = None self.init_driver() self.disabled_until = time.time() self._lock = Lock() self.handle_to_url = {} # type:Dict[str, str]
def test_run_coroutine_with_green_thread_with_decorator(self): reset_thread_running_flag() reset_thread_running_timer() from gevent.threading import Lock global _Thread_Lock, Sleep_Function _Thread_Lock = Lock() # Sleep_Function = gevent.sleep Sleep_Function = mr.sleep _target_with_green_threads() TestPackageInit._chk_thread_result()
def __init__(self, username, password, urls, use_lock_for_reseting_jwt=False, max_retries=5): self.username = username self.password = password self.urls = urls self.lock_for_reseting_jwt = Lock( ) if use_lock_for_reseting_jwt else None self.__init_request_session(max_retries) self.__set_token()
def __init__(self, name, bind_address, *, title_re: str, instruments: List[str] = (), capture_rects: List[Dict[str, Tuple]], filter_mode: int = 0, chrome: bool = False, master_address): super().__init__(name, bind_address, master_address=master_address) self.service = name self.title_re = title_re self.instruments = instruments or ([''] * len(capture_rects)) self.capture_rects = capture_rects self.last_handle = None self.chrome = chrome self.recognizer = Recognizer(name=name, in_width=16, in_height=16, n_units=100, n_out=128, filter_mode=filter_mode, chrome=chrome) self._lock = Lock()
def __init__( self, jsonrpc_client: JSONRPCClient, user_deposit_address: UserDepositAddress, contract_manager: ContractManager, proxy_manager: "ProxyManager", block_identifier: BlockIdentifier, ) -> None: if not is_binary_address(user_deposit_address): raise ValueError("Expected binary address format for token nework") check_address_has_code_handle_pruned_block( client=jsonrpc_client, address=Address(user_deposit_address), contract_name=CONTRACT_USER_DEPOSIT, expected_code=decode_hex( contract_manager.get_runtime_hexcode(CONTRACT_USER_DEPOSIT)), given_block_identifier=block_identifier, ) self.client = jsonrpc_client self.address = user_deposit_address self.node_address = self.client.address self.contract_manager = contract_manager self.gas_measurements = gas_measurements( self.contract_manager.contracts_version) self.proxy_manager = proxy_manager self.proxy = jsonrpc_client.new_contract_proxy( abi=self.contract_manager.get_contract_abi(CONTRACT_USER_DEPOSIT), contract_address=Address(user_deposit_address), ) # Keeps track of the current in-flight deposits, to avoid sending # unecessary transactions. self._inflight_deposits: Dict[Address, InflightDeposit] = dict() # Don't allow concurrent withdraw_plan and withdraw calls. # This simplifies the precondition checks. self._withdraw_lock = Lock()
def __init__(self): self.register_lock = Lock() self.spawn_signal = Event() self.__t_reg = {} # set the worker timeout (unit: second) self.main_timeout = 1 self.worker_timeout = 1 # record the mapping relationship between tx_id and channel_name self.tx_id_channel_map = {} # self.nodeb_api = NodeBApi() # update channel api self.deposit_action = { State.OPEN: depositin, State.SETTLED: depoistout }
def __init__(self, url=None, file_name=None, path_to_store=None, total_size=None, thread_number=None, logger=None, print_progress=False): assert url or file_name file_name = file_name if file_name else url.split('?')[0].split( '/')[-1] file_name_split = file_name.split('.') if len(file_name_split[0]) > 64: file_name_split[0] = file_name_split[0][:64] file_name = '.'.join(file_name_split) self.path_to_store = path_to_store if path_to_store else f"{os.environ['HOME']}/Downloads" self.logger = logger or logging self.print_progress = print_progress self.url = url self.total_size = total_size self.file_name_with_path = self.path_to_store + '/' + file_name self.breakpoint_file_path = self.file_name_with_path + '.tmp' self.file_seeker = None self.thread_number = thread_number self.thread_buffer_size = None self.remaining_segments = [] self.start_time = None self.segment_dispatch_task = None self.speed_calculation_task = None self.bytearray_of_threads = None self.last_progress_time = None self.last_downloaded_size = 0 self.workers = [] self.coworker_end_to_indexes_map = dict() self.to_dispatcher_queue = Queue() self.complete_notify_lock = Lock() self.download_info = self.get_download_info()
def __init__(self, conf_dict): """ Init :param conf_dict: dict :type conf_dict: dict """ # Lock for acquire/release self.pool_lock = Lock() # Store self.conf_dict = conf_dict # Max size self.max_size = self.conf_dict.get("pool_max_size", 10) # Alloc self.pool = queue.Queue(maxsize=self.max_size) # Init self.size = 0
def __init__(self, host="http://trade", config=None, size=200): self._pool_size = size self._instance_list = list() ins = nge(host=host, config=config) self._instance_list.append(ins) for _ in range(self._pool_size-1): new_instance = SwaggerClient( swagger_spec=ins.swagger_spec, also_return_response=config.get("also_return_response", True) if config else True) self._instance_list.append(new_instance) self._semaphore = BoundedSemaphore(self._pool_size) self._lock = Lock() self._idx = 0
def __init__(self, proxy: PaywalledProxy, es: ElasticsearchBackend): self.proxy = proxy self.resource_cache = {} self.cache_lock = Lock() proxy.add_paywalled_resource( ExpensiveElasticsearch, '/_search', None, '/<string:_index>/_search', '/<string:_index>/<string:_type>/_search', '/_msearch', '/<string:_index>/_msearch', '/<string:_index>/<string:_type>/_msearch', '/_mapping', '/<string:_index>/_mapping', '/<string:_index>/<string:_type>/_mapping', resource_class_kwargs=dict( resource_cache=self.resource_cache, cache_lock=self.cache_lock, es=es, ) )
class MysqlConnectionPool(DatabaseConnectionPool): """ """ LOCK = Lock() WATCHDOG = None def __init__(self, maxsize): super(MysqlConnectionPool, self).__init__(maxsize) def create_connection(self, conn_params): """ :param conn_params: :type: dict :return: """ if len(HOSTS_STATUS) == 0: with MysqlConnectionPool.LOCK: if len(HOSTS_STATUS) == 0: # populate logger.debug("Populate host list") if 'host' not in conn_params: HOSTS_STATUS['localhost'] = 0 else: for host in conn_params['host'].split(','): HOSTS_STATUS[host] = 0 # Deep copy param new_conn_params = copy.deepcopy(conn_params) db_connection = False ex = Exception('No Db Host available') while not db_connection: host = self._get_random_host() if not host: logger.error("No mysql host available... %s are down", HOSTS_STATUS.keys()) raise ex # overide host in param new_conn_params['host'] = host try: db_connection = Database.connect(**new_conn_params) except Exception as e: ex = e logger.error("de-activate %s for 1 minute ex=%s", host, e) HOSTS_STATUS[host] = time.time() + 60 return db_connection def is_usable(self, conn): try: conn.ping() except Database.Error: return False else: return True # noinspection PyMethodMayBeStatic def _get_random_host(self): """ Return a host in HOSTS_STATUS where the host is up :return: """ now = time.time() hosts_up = [ host for host, prison in HOSTS_STATUS.items() if prison < now ] try: host = random.choice(hosts_up) return host except IndexError: return False
class MysqlApi(object): """ Mysql Api """ # Lock POOL_LOCK = Lock() # Static pool instances (hash from config dict => MysqlConnectionPool) D_POOL_INSTANCES = dict() @classmethod def reset_pools(cls): """ Reset all pools """ with cls.POOL_LOCK: for s_hash, pool in cls.D_POOL_INSTANCES.items(): logger.info("Closing pool, s_hash=%s", s_hash) pool.close_all() cls.D_POOL_INSTANCES = dict() @classmethod def _get_pool_hash(cls, conf_dict): """ Get pool hash :param conf_dict: dict :type conf_dict: dict :return: str :rtype: str """ s_hash = str(hash(ujson.dumps(conf_dict, sort_keys=True))) return s_hash @classmethod def _get_pool(cls, conf_dict): """ Init static pool :param conf_dict: dict :type conf_dict: dict :return pysolmysql.Pool.mysql_pool.MysqlConnectionPool :rtype pysolmysql.Pool.mysql_pool.MysqlConnectionPool """ # Hash s_hash = cls._get_pool_hash(conf_dict) # Alloc if needed if s_hash not in cls.D_POOL_INSTANCES: with cls.POOL_LOCK: if s_hash not in cls.D_POOL_INSTANCES: cls.D_POOL_INSTANCES[s_hash] = MysqlConnectionPool( conf_dict) logger.info("Allocated pool, s_hash=%s, pool.len=%s", s_hash, len(cls.D_POOL_INSTANCES)) Meters.aii("k.db_pool.hash.cur") # Over return cls.D_POOL_INSTANCES[s_hash] @classmethod def _fix_type(cls, data): """ Fix type :param data: data """ if isinstance(data, bytearray): return data.decode("utf-8") else: return data @classmethod def exec_0(cls, conf_dict, statement): """ Execute a sql statement, returning row affected. :param conf_dict: configuration dict :type conf_dict: dict :param statement: statement to execute :type statement: str :rtype: int :return rows affected """ cnx = None rows_affected = 0 try: cnx = cls._get_pool(conf_dict).connection_acquire() with closing(cnx.cursor()) as cur: cur.execute(statement) rows_affected = cur.rowcount finally: cls._get_pool(conf_dict).connection_release(cnx) return rows_affected @classmethod def exec_n(cls, conf_dict, statement, fix_types=True): """ Execute a sql statement, returning 0..N rows :param conf_dict: configuration dict :type conf_dict: dict :param statement: statement to execute :type statement: str :param fix_types: If true, fix data type :type fix_types: bool :return list of dict. :rtype list """ cnx = None try: cnx = cls._get_pool(conf_dict).connection_acquire() with closing(cnx.cursor()) as cur: cur.execute(statement) rows = cur.fetchall() for row in rows: logger.debug("row=%s", row) for k, v in row.items(): logger.debug("k=%s, %s, %s", k, type(v), v) if fix_types: row[k] = MysqlApi._fix_type(v) return rows finally: cls._get_pool(conf_dict).connection_release(cnx) @classmethod def exec_1(cls, conf_dict, statement, fix_types=True): """ Execute a sql statement, returning 1 row. Method will fail if 1 row is not returned. :rtype: object :param conf_dict: configuration dict :type conf_dict: dict :param statement: statement to execute :type statement: str :param fix_types: If true, fix data type :type fix_types: bool :return dict :rtype dict """ cnx = None try: cnx = cls._get_pool(conf_dict).connection_acquire() with closing(cnx.cursor()) as cur: cur.execute(statement) rows = cur.fetchall() for row in rows: logger.debug("row=%s", row) for k, v in row.items(): logger.debug("k=%s, %s, %s", k, type(v), v) if fix_types: row[k] = MysqlApi._fix_type(v) if len(rows) != 1: raise Exception( "Invalid row len, expecting 1, having={0}".format( len(rows))) return rows[0] finally: cls._get_pool(conf_dict).connection_release(cnx) @classmethod def exec_01(cls, conf_dict, statement, fix_types=True): """ Execute a sql statement, returning 0 or 1 row. Method will fail if 0 or 1 row is not returned. :param conf_dict: configuration dict :type conf_dict: dict :param statement: statement to execute :type statement: str :param fix_types: If true, fix data type :type fix_types: bool :return dict, None :rtype dict, None """ cnx = None try: cnx = cls._get_pool(conf_dict).connection_acquire() with closing(cnx.cursor()) as cur: cur.execute(statement) rows = cur.fetchall() for row in rows: logger.debug("row=%s", row) for k, v in row.items(): logger.debug("k=%s, %s, %s", k, type(v), v) if fix_types: row[k] = MysqlApi._fix_type(v) if len(rows) == 0: return None elif len(rows) != 1: raise Exception( "Invalid row len, expecting 1, having={0}".format( len(rows))) else: return rows[0] finally: cls._get_pool(conf_dict).connection_release(cnx) @classmethod def multi_n(cls, conf_dict, ar_statement): """ Execute multiple sql statement, reading nothing from mysql. :type conf_dict: dict :param ar_statement: list of statements to execute (for instance, batch of insert or whatever) :type ar_statement: list """ cnx = None try: cnx = cls._get_pool(conf_dict).connection_acquire() with closing(cnx.cursor()) as cur: for s in ar_statement: cur.execute(s) finally: cls._get_pool(conf_dict).connection_release(cnx)
accs = db.session.query(Account).order_by(asc(Account.disabled)).order_by( Account.username).all() mails = { 'resident': [sget_resident_mail(), sget_resident_topic()], 'tbadge': [sget_tbadge_mail(), sget_tbadge_topic()], 'other': [sget_other_mail(), sget_other_topic()] } return render_template("settings/accounts.html", roles=roles, accounts=accs, mails=mails) alt_clean_lock: Lock = Lock() last_alt_clean: datetime = datetime.min min_time_between_checks: timedelta = timedelta(hours=12) def should_clean_alts() -> bool: return (datetime.utcnow() - last_alt_clean) > min_time_between_checks def async_clean_alt_list() -> None: """ This function asyncron and returns instantly Removes links between accounts and characters if - there is a token - but the token expired - or the owner_hash changed (this should expire the token!)
def __init__(self, parent): super(Order, self).__init__(parent=parent) self._lock = Lock()