class Task(object):
task_id = 0
lock = Lock()
def __init__(self, runner, *task_args, **task_kw):
assert callable(runner)
self.runner = runner
self.task_args = task_args
self.task_kw = task_kw
self.name = self.__generate_name()
def __generate_name(self):
from gevent.threading import Lock
Task.lock.acquire()
Task.task_id += 1
_task_id = Task.task_id
Task.lock.release()
return "task_{}".format(_task_id)
def execute(self):
self.runner(*self.task_args, **self.task_kw)
def __str__(self):
return u"task({}) runner:'{}', args:'{}', kw:'{}'"\
.format(self.name, self.runner.__name__, self.task_args, self.task_kw)
def __init__(self):
"""
Const
"""
# Gevent
self._gevent_pool_max = 1024
self._gevent_locker = Lock()
self._gevent_pool = dict()
# urllib3
# Force underlying fifo queue to 1024 via maxsize
self._u3_basic_pool = PoolManager(num_pools=1024, maxsize=1024)
self._u3_proxy_pool_max = 1024
self._u3_proxy_locker = Lock()
self._u3_proxy_pool = dict()
def __init__(self, ipc_path=None, testnet=False, *args, **kwargs):
if ipc_path is None:
self.ipc_path = get_default_ipc_path(testnet)
else:
self.ipc_path = ipc_path
self._lock = Lock()
super(IPCProvider, self).__init__(*args, **kwargs)
def __init__(self, name, bind_address, *, master_address, driver_port: int):
super().__init__(name, bind_address, master_address=master_address)
self.driver = None
self.driver_port = driver_port
self.last_handle = None
self.init_driver()
self.disabled_until = time.time()
self._lock = Lock()
self.handle_to_url = {} # type:Dict[str, str]
def test_run_coroutine_with_green_thread_with_decorator(self):
reset_thread_running_flag()
reset_thread_running_timer()
from gevent.threading import Lock
global _Thread_Lock, Sleep_Function
_Thread_Lock = Lock()
# Sleep_Function = gevent.sleep
Sleep_Function = mr.sleep
_target_with_green_threads()
TestPackageInit._chk_thread_result()
def __init__(self,
username,
password,
urls,
use_lock_for_reseting_jwt=False,
max_retries=5):
self.username = username
self.password = password
self.urls = urls
self.lock_for_reseting_jwt = Lock(
) if use_lock_for_reseting_jwt else None
self.__init_request_session(max_retries)
self.__set_token()
def __init__(self, name, bind_address, *, title_re: str, instruments: List[str] = (),
capture_rects: List[Dict[str, Tuple]], filter_mode: int = 0,
chrome: bool = False,
master_address):
super().__init__(name, bind_address, master_address=master_address)
self.service = name
self.title_re = title_re
self.instruments = instruments or ([''] * len(capture_rects))
self.capture_rects = capture_rects
self.last_handle = None
self.chrome = chrome
self.recognizer = Recognizer(name=name, in_width=16, in_height=16, n_units=100, n_out=128,
filter_mode=filter_mode,
chrome=chrome)
self._lock = Lock()
def __init__(
self,
jsonrpc_client: JSONRPCClient,
user_deposit_address: UserDepositAddress,
contract_manager: ContractManager,
proxy_manager: "ProxyManager",
block_identifier: BlockIdentifier,
) -> None:
if not is_binary_address(user_deposit_address):
raise ValueError("Expected binary address format for token nework")
check_address_has_code_handle_pruned_block(
client=jsonrpc_client,
address=Address(user_deposit_address),
contract_name=CONTRACT_USER_DEPOSIT,
expected_code=decode_hex(
contract_manager.get_runtime_hexcode(CONTRACT_USER_DEPOSIT)),
given_block_identifier=block_identifier,
)
self.client = jsonrpc_client
self.address = user_deposit_address
self.node_address = self.client.address
self.contract_manager = contract_manager
self.gas_measurements = gas_measurements(
self.contract_manager.contracts_version)
self.proxy_manager = proxy_manager
self.proxy = jsonrpc_client.new_contract_proxy(
abi=self.contract_manager.get_contract_abi(CONTRACT_USER_DEPOSIT),
contract_address=Address(user_deposit_address),
)
# Keeps track of the current in-flight deposits, to avoid sending
# unecessary transactions.
self._inflight_deposits: Dict[Address, InflightDeposit] = dict()
# Don't allow concurrent withdraw_plan and withdraw calls.
# This simplifies the precondition checks.
self._withdraw_lock = Lock()
def __init__(self):
self.register_lock = Lock()
self.spawn_signal = Event()
self.__t_reg = {}
# set the worker timeout (unit: second)
self.main_timeout = 1
self.worker_timeout = 1
# record the mapping relationship between tx_id and channel_name
self.tx_id_channel_map = {}
#
self.nodeb_api = NodeBApi()
# update channel api
self.deposit_action = {
State.OPEN: depositin,
State.SETTLED: depoistout
}
def __init__(self,
url=None,
file_name=None,
path_to_store=None,
total_size=None,
thread_number=None,
logger=None,
print_progress=False):
assert url or file_name
file_name = file_name if file_name else url.split('?')[0].split(
'/')[-1]
file_name_split = file_name.split('.')
if len(file_name_split[0]) > 64:
file_name_split[0] = file_name_split[0][:64]
file_name = '.'.join(file_name_split)
self.path_to_store = path_to_store if path_to_store else f"{os.environ['HOME']}/Downloads"
self.logger = logger or logging
self.print_progress = print_progress
self.url = url
self.total_size = total_size
self.file_name_with_path = self.path_to_store + '/' + file_name
self.breakpoint_file_path = self.file_name_with_path + '.tmp'
self.file_seeker = None
self.thread_number = thread_number
self.thread_buffer_size = None
self.remaining_segments = []
self.start_time = None
self.segment_dispatch_task = None
self.speed_calculation_task = None
self.bytearray_of_threads = None
self.last_progress_time = None
self.last_downloaded_size = 0
self.workers = []
self.coworker_end_to_indexes_map = dict()
self.to_dispatcher_queue = Queue()
self.complete_notify_lock = Lock()
self.download_info = self.get_download_info()
def __init__(self, conf_dict):
"""
Init
:param conf_dict: dict
:type conf_dict: dict
"""
# Lock for acquire/release
self.pool_lock = Lock()
# Store
self.conf_dict = conf_dict
# Max size
self.max_size = self.conf_dict.get("pool_max_size", 10)
# Alloc
self.pool = queue.Queue(maxsize=self.max_size)
# Init
self.size = 0
def __init__(self, host="http://trade", config=None, size=200):
self._pool_size = size
self._instance_list = list()
ins = nge(host=host, config=config)
self._instance_list.append(ins)
for _ in range(self._pool_size-1):
new_instance = SwaggerClient(
swagger_spec=ins.swagger_spec,
also_return_response=config.get("also_return_response",
True) if config else True)
self._instance_list.append(new_instance)
self._semaphore = BoundedSemaphore(self._pool_size)
self._lock = Lock()
self._idx = 0
def __init__(self, proxy: PaywalledProxy, es: ElasticsearchBackend):
self.proxy = proxy
self.resource_cache = {}
self.cache_lock = Lock()
proxy.add_paywalled_resource(
ExpensiveElasticsearch,
'/_search',
None,
'/<string:_index>/_search',
'/<string:_index>/<string:_type>/_search',
'/_msearch',
'/<string:_index>/_msearch',
'/<string:_index>/<string:_type>/_msearch',
'/_mapping',
'/<string:_index>/_mapping',
'/<string:_index>/<string:_type>/_mapping',
resource_class_kwargs=dict(
resource_cache=self.resource_cache,
cache_lock=self.cache_lock,
es=es,
)
)
class MysqlConnectionPool(DatabaseConnectionPool):
"""
"""
LOCK = Lock()
WATCHDOG = None
def __init__(self, maxsize):
super(MysqlConnectionPool, self).__init__(maxsize)
def create_connection(self, conn_params):
"""
:param conn_params:
:type: dict
:return:
"""
if len(HOSTS_STATUS) == 0:
with MysqlConnectionPool.LOCK:
if len(HOSTS_STATUS) == 0:
# populate
logger.debug("Populate host list")
if 'host' not in conn_params:
HOSTS_STATUS['localhost'] = 0
else:
for host in conn_params['host'].split(','):
HOSTS_STATUS[host] = 0
# Deep copy param
new_conn_params = copy.deepcopy(conn_params)
db_connection = False
ex = Exception('No Db Host available')
while not db_connection:
host = self._get_random_host()
if not host:
logger.error("No mysql host available... %s are down",
HOSTS_STATUS.keys())
raise ex
# overide host in param
new_conn_params['host'] = host
try:
db_connection = Database.connect(**new_conn_params)
except Exception as e:
ex = e
logger.error("de-activate %s for 1 minute ex=%s", host, e)
HOSTS_STATUS[host] = time.time() + 60
return db_connection
def is_usable(self, conn):
try:
conn.ping()
except Database.Error:
return False
else:
return True
# noinspection PyMethodMayBeStatic
def _get_random_host(self):
"""
Return a host in HOSTS_STATUS where the host is up
:return:
"""
now = time.time()
hosts_up = [
host for host, prison in HOSTS_STATUS.items() if prison < now
]
try:
host = random.choice(hosts_up)
return host
except IndexError:
return False
class MysqlApi(object):
"""
Mysql Api
"""
# Lock
POOL_LOCK = Lock()
# Static pool instances (hash from config dict => MysqlConnectionPool)
D_POOL_INSTANCES = dict()
@classmethod
def reset_pools(cls):
"""
Reset all pools
"""
with cls.POOL_LOCK:
for s_hash, pool in cls.D_POOL_INSTANCES.items():
logger.info("Closing pool, s_hash=%s", s_hash)
pool.close_all()
cls.D_POOL_INSTANCES = dict()
@classmethod
def _get_pool_hash(cls, conf_dict):
"""
Get pool hash
:param conf_dict: dict
:type conf_dict: dict
:return: str
:rtype: str
"""
s_hash = str(hash(ujson.dumps(conf_dict, sort_keys=True)))
return s_hash
@classmethod
def _get_pool(cls, conf_dict):
"""
Init static pool
:param conf_dict: dict
:type conf_dict: dict
:return pysolmysql.Pool.mysql_pool.MysqlConnectionPool
:rtype pysolmysql.Pool.mysql_pool.MysqlConnectionPool
"""
# Hash
s_hash = cls._get_pool_hash(conf_dict)
# Alloc if needed
if s_hash not in cls.D_POOL_INSTANCES:
with cls.POOL_LOCK:
if s_hash not in cls.D_POOL_INSTANCES:
cls.D_POOL_INSTANCES[s_hash] = MysqlConnectionPool(
conf_dict)
logger.info("Allocated pool, s_hash=%s, pool.len=%s",
s_hash, len(cls.D_POOL_INSTANCES))
Meters.aii("k.db_pool.hash.cur")
# Over
return cls.D_POOL_INSTANCES[s_hash]
@classmethod
def _fix_type(cls, data):
"""
Fix type
:param data: data
"""
if isinstance(data, bytearray):
return data.decode("utf-8")
else:
return data
@classmethod
def exec_0(cls, conf_dict, statement):
"""
Execute a sql statement, returning row affected.
:param conf_dict: configuration dict
:type conf_dict: dict
:param statement: statement to execute
:type statement: str
:rtype: int
:return rows affected
"""
cnx = None
rows_affected = 0
try:
cnx = cls._get_pool(conf_dict).connection_acquire()
with closing(cnx.cursor()) as cur:
cur.execute(statement)
rows_affected = cur.rowcount
finally:
cls._get_pool(conf_dict).connection_release(cnx)
return rows_affected
@classmethod
def exec_n(cls, conf_dict, statement, fix_types=True):
"""
Execute a sql statement, returning 0..N rows
:param conf_dict: configuration dict
:type conf_dict: dict
:param statement: statement to execute
:type statement: str
:param fix_types: If true, fix data type
:type fix_types: bool
:return list of dict.
:rtype list
"""
cnx = None
try:
cnx = cls._get_pool(conf_dict).connection_acquire()
with closing(cnx.cursor()) as cur:
cur.execute(statement)
rows = cur.fetchall()
for row in rows:
logger.debug("row=%s", row)
for k, v in row.items():
logger.debug("k=%s, %s, %s", k, type(v), v)
if fix_types:
row[k] = MysqlApi._fix_type(v)
return rows
finally:
cls._get_pool(conf_dict).connection_release(cnx)
@classmethod
def exec_1(cls, conf_dict, statement, fix_types=True):
"""
Execute a sql statement, returning 1 row.
Method will fail if 1 row is not returned.
:rtype: object
:param conf_dict: configuration dict
:type conf_dict: dict
:param statement: statement to execute
:type statement: str
:param fix_types: If true, fix data type
:type fix_types: bool
:return dict
:rtype dict
"""
cnx = None
try:
cnx = cls._get_pool(conf_dict).connection_acquire()
with closing(cnx.cursor()) as cur:
cur.execute(statement)
rows = cur.fetchall()
for row in rows:
logger.debug("row=%s", row)
for k, v in row.items():
logger.debug("k=%s, %s, %s", k, type(v), v)
if fix_types:
row[k] = MysqlApi._fix_type(v)
if len(rows) != 1:
raise Exception(
"Invalid row len, expecting 1, having={0}".format(
len(rows)))
return rows[0]
finally:
cls._get_pool(conf_dict).connection_release(cnx)
@classmethod
def exec_01(cls, conf_dict, statement, fix_types=True):
"""
Execute a sql statement, returning 0 or 1 row.
Method will fail if 0 or 1 row is not returned.
:param conf_dict: configuration dict
:type conf_dict: dict
:param statement: statement to execute
:type statement: str
:param fix_types: If true, fix data type
:type fix_types: bool
:return dict, None
:rtype dict, None
"""
cnx = None
try:
cnx = cls._get_pool(conf_dict).connection_acquire()
with closing(cnx.cursor()) as cur:
cur.execute(statement)
rows = cur.fetchall()
for row in rows:
logger.debug("row=%s", row)
for k, v in row.items():
logger.debug("k=%s, %s, %s", k, type(v), v)
if fix_types:
row[k] = MysqlApi._fix_type(v)
if len(rows) == 0:
return None
elif len(rows) != 1:
raise Exception(
"Invalid row len, expecting 1, having={0}".format(
len(rows)))
else:
return rows[0]
finally:
cls._get_pool(conf_dict).connection_release(cnx)
@classmethod
def multi_n(cls, conf_dict, ar_statement):
"""
Execute multiple sql statement, reading nothing from mysql.
:type conf_dict: dict
:param ar_statement: list of statements to execute (for instance, batch of insert or whatever)
:type ar_statement: list
"""
cnx = None
try:
cnx = cls._get_pool(conf_dict).connection_acquire()
with closing(cnx.cursor()) as cur:
for s in ar_statement:
cur.execute(s)
finally:
cls._get_pool(conf_dict).connection_release(cnx)
accs = db.session.query(Account).order_by(asc(Account.disabled)).order_by(
Account.username).all()
mails = {
'resident': [sget_resident_mail(),
sget_resident_topic()],
'tbadge': [sget_tbadge_mail(), sget_tbadge_topic()],
'other': [sget_other_mail(), sget_other_topic()]
}
return render_template("settings/accounts.html",
roles=roles,
accounts=accs,
mails=mails)
alt_clean_lock: Lock = Lock()
last_alt_clean: datetime = datetime.min
min_time_between_checks: timedelta = timedelta(hours=12)
def should_clean_alts() -> bool:
return (datetime.utcnow() - last_alt_clean) > min_time_between_checks
def async_clean_alt_list() -> None:
"""
This function asyncron and returns instantly
Removes links between accounts and characters if
- there is a token
- but the token expired
- or the owner_hash changed (this should expire the token!)
def __init__(self, parent):
super(Order, self).__init__(parent=parent)
self._lock = Lock()