class CGSBoundedSemaphore:
def __init__(self,value):
self.boundedSemaphore = BoundedSemaphore(value)
self.datasetQueueLock = RLock()
self.datasetQueue = []
def acquire(self,datasetId):
try:
self.datasetQueueLock.acquire()
self.datasetQueue.append(datasetId)
finally:
self.datasetQueueLock.release()
self.boundedSemaphore.acquire()
def release(self,datasetId):
try:
self.datasetQueueLock.acquire()
self.datasetQueue.remove(datasetId)
except:
pass
finally:
self.datasetQueueLock.release()
self.boundedSemaphore.release()
def getIndexDatasetId(self,datasetId):
try:
self.datasetQueueLock.acquire()
return self.datasetQueue.index(datasetId)
except:
return -1
finally:
self.datasetQueueLock.release()
def status(self):
return list(self.datasetQueue)
def create_document(main_url, max_connections=2, filepath=None):
"""Creates an EPUB document from a fanfic.
main_url -- user given URL which should be the first chapter
max_connections -- maximum number of simultaneous connections
default: 2. This should be chosen with care as the Terms of Service
of some of the websites states that you shouldn't cause more stress
than a normal visitor.
filepath -- optional path for the resulting Epub document
By default filename is: %author - %title.epub in the current
directory. %author and %title in the path are special, they're
changed to the story author and title respectively."""
global dl_semaphore
dl_semaphore = BoundedSemaphore(max_connections)
parse, parse_ch1 = get_parser(main_url)
html_ch1, chapter_num, story_info = get_chapter1(main_url, parse_ch1)
chapters = {}
chapters[1] = html_ch1
if story_info["cover_url"]:
cover_image_req = Request(
story_info["cover_url"], headers=story_info["cover_headers"])
cover_image = urlopen(cover_image_req).read()
else:
cover_image = open("default.jpg", "rb").read()
with concurrent.futures.ThreadPoolExecutor(max_workers=max_connections+3) \
as executor:
parse_chapters = []
download_urls = story_info["chapter_urls"]
for ch in range(2, chapter_num+1):
dl_semaphore.acquire()
parse_chapters.append(
executor.submit(get_chapter, download_urls[ch], ch, parse))
for future in concurrent.futures.as_completed(parse_chapters):
html, chapter_no = future.result()
chapters[chapter_no] = html
if not filepath:
filepath = "{} - {}.epub".format(
INVALID_CHARS.sub("-", story_info["author"]),
INVALID_CHARS.sub("-", story_info["title"]))
else:
filepath = filepath.replace(
"%author", INVALID_CHARS.sub("-", story_info["author"]))
filepath = filepath.replace(
"%title", INVALID_CHARS.sub("-", story_info["title"]))
with zipfile.ZipFile(filepath, "w") as f:
f.writestr("mimetype", MIMETYPE)
f.writestr("META-INF/container.xml", CONTAINER_XML)
f.writestr("Content/titlepage.html", TITLEPAGE_HTML)
f.writestr("Content/styles.css", STYLES_CSS)
f.writestr("Content/cover.jpg", cover_image)
f.writestr("Content/toc.ncx", toc(story_info, chapter_num))
f.writestr("Content/content.opf", contents(story_info, chapter_num))
for ch in range(1, chapter_num+1):
f.writestr("Content/Chapters/ch{}.html".format(ch), chapters[ch])
def execute_semaphored_threads():
inputs = list(range(800, 1000))
print("Calculating from {} to {}".format(inputs[0], inputs[-1]))
# only four threads at time
pool_sema = BoundedSemaphore(value=4)
threads = []
for i in inputs:
# limit threads amount
pool_sema.acquire()
t = Thread(target=execute_fib, args=(i,))
threads.append(t)
t.start()
pool_sema.release()
return threads
def __init__(self):
self.__logger = logging.getLogger("framework.facade")
self.mCallBacks = None
self.mutex = BoundedSemaphore(value=1)
self.controller = None
# Messages queue
self.__qmsg = Queue()
self.jman = None
# results storage
self.__results = Results()
# load plugins
self.__logger.info("* Loading plugins")
self.load_plugins()
# load session
self.__logger.info("* Loading session")
self.__cur_session = DataModel()
if Settings().get(Settings.SEC_GRL, Settings.AUTO_SESSION) == 'True':
try:
self.__cur_session = self.load_session()
except Exception:
pass
def __init__(self, repos):
self.ui = getglobalui()
self.repos = repos
self.config = repos.getconfig()
self.tunnel = repos.getpreauthtunnel()
self.usessl = repos.getssl()
self.username = repos.getuser()
self.password = None
self.passworderror = None
self.goodpassword = None
self.hostname = repos.gethost()
self.port = repos.getport()
if self.port == None:
self.port = 993 if self.usessl else 143
self.sslclientcert = repos.getsslclientcert()
self.sslclientkey = repos.getsslclientkey()
self.sslcacertfile = repos.getsslcacertfile()
if self.sslcacertfile is None:
self.verifycert = None # disable cert verification
self.delim = None
self.root = None
self.maxconnections = repos.getmaxconnections()
self.availableconnections = []
self.assignedconnections = []
self.lastowner = {}
self.semaphore = BoundedSemaphore(self.maxconnections)
self.connectionlock = Lock()
self.reference = repos.getreference()
self.idlefolders = repos.getidlefolders()
self.gss_step = self.GSS_STATE_STEP
self.gss_vc = None
self.gssapi = False
class VDOM_semaphore(object):
def __init__(self, counter=1):
self.__semaphore = BoundedSemaphore(counter)
def lock(self):
return self.__semaphore.acquire()
def unlock(self):
self.__semaphore.release()
def __enter__(self):
self.lock()
return self
def __exit__(self, extype, exvalue, traceback):
self.unlock()
def __init__(self, storage_driver, table_info_repo, concurrent_tasks=1000,
batch_chunk_size=25, schema_operation_timeout=300):
self._storage_driver = storage_driver
self._table_info_repo = table_info_repo
self._batch_chunk_size = batch_chunk_size
self._schema_operation_timeout = schema_operation_timeout
self.__task_executor = ThreadPoolExecutor(concurrent_tasks)
self.__task_semaphore = BoundedSemaphore(concurrent_tasks)
def __init__(self, item_number, person_capacity):
if type(item_number) is not int:
raise Exception("item_number is not an integer")
if type(person_capacity) is not int:
raise Exception("person_capacity is not an integer")
self.item_number = item_number
self.person_capacity = person_capacity
self.sema = BoundedSemaphore(value=self.person_capacity)
class IDCardFactory(Factory):
def __init__(self):
self.owner = []
self.seqNum = 1
self.semaphore = BoundedSemaphore(1)
def createProduct(self, owner):
self.semaphore.acquire()
card = IDCard(self.seqNum, owner)
self.seqNum += 1
self.semaphore.release()
return card
def registerProduct(self, product):
self.owner.append(product.getOwner())
def getOwners(self):
return self.owner
def __init__(self, subscription):
super(Latest.Sink, self).__init__(subscription)
self.gate = RLock()
self.semaphore = BoundedSemaphore(1)
self.semaphore.acquire()
self.notificationAvailable = False
self.kind = None
self.value = None
self.error = None
def __init__(self, subscription):
super(Next.Sink, self).__init__(subscription)
self.gate = RLock()
self.semaphore = BoundedSemaphore(1)
self.semaphore.acquire()
self.waiting = False
self.kind = None
self.value = None
self.error = None
class MyThread(threading.Thread): def __init__(self,site): self.site = site threading.Thread.__init__(self) self.semaphore = BoundedSemaphore(value=MAXCONN) self.t = Tweetya() def run(self): link = self.t.parse(self.site) self.semaphore.acquire() urls = self.t.getlinks() for i in link: if not (i in urls): self.t.setlink(i) short = self.t.short(i) title = self.t.gettitle(short) self.t.auth(str(title)+' '+str(short)) self.semaphore.release()
def __init__(self, dAmn, logging=True, debug=False):
super(Logger, self).__init__()
self.dAmn = dAmn
self.logging = logging
self.mute_channels = []
self.debug = debug
self.running = False
self._run = False
# Create locks
self.qlock = BoundedSemaphore()
#self.mlock = BoundedSemaphore()
self.llock = BoundedSemaphore()
# Create queues
self.wqueue = [] # Output queue
#self.mqueue = [] # Muter queue
self.lqueue = [] # Logger queue. Just in case.
# Just in case.
self.subbing = False
self.subthread = None
def __init__(self, n_process=2, n_threads=5, n_taks=10, daemon=False): self.daemon = daemon self.n_taks = n_taks self.n_threads = n_threads self.n_process = n_process self.sem_threads = BoundedSemaphore(self.n_threads) self.sem_tasks = asyncio.BoundedSemaphore(self.n_taks) self.running_process = []
def __init__(self, repos):
self.ui = getglobalui()
self.repos = repos
self.config = repos.getconfig()
self.preauth_tunnel = repos.getpreauthtunnel()
self.transport_tunnel = repos.gettransporttunnel()
if self.preauth_tunnel and self.transport_tunnel:
raise OfflineImapError('%s: ' % repos + \
'you must enable precisely one '
'type of tunnel (preauth or transport), '
'not both', OfflineImapError.ERROR.REPO)
self.tunnel = \
self.preauth_tunnel if self.preauth_tunnel \
else self.transport_tunnel
self.username = \
None if self.preauth_tunnel else repos.getuser()
self.user_identity = repos.get_remote_identity()
self.authmechs = repos.get_auth_mechanisms()
self.password = None
self.oauth2 = repos.getoauth2()
self.oauth2url = repos.getoauth2url() if self.oauth2 else None
self.oauth2clientid = repos.getoauth2clientid() if self.oauth2 else None
self.oauth2clientsecret = repos.getoauth2clientsecret() if self.oauth2 else None
self.oauth2refreshtoken = repos.getoauth2refreshtoken() if self.oauth2 else None
self.passworderror = None
self.goodpassword = None
self.usessl = repos.getssl()
self.hostname = \
None if self.preauth_tunnel else repos.gethost()
self.port = repos.getport()
if self.port == None:
self.port = 993 if self.usessl else 143
self.sslclientcert = repos.getsslclientcert()
self.sslclientkey = repos.getsslclientkey()
self.sslcacertfile = repos.getsslcacertfile()
self.sslversion = repos.getsslversion()
if self.sslcacertfile is None:
self.__verifycert = None # disable cert verification
self.delim = None
self.root = None
self.maxconnections = repos.getmaxconnections()
self.availableconnections = []
self.assignedconnections = []
self.lastowner = {}
self.semaphore = BoundedSemaphore(self.maxconnections)
self.connectionlock = Lock()
self.reference = repos.getreference()
self.idlefolders = repos.getidlefolders()
self.gss_step = self.GSS_STATE_STEP
self.gss_vc = None
self.gssapi = False
def __init__(self, device):
self.dev = device
self.sts_semaphore = BoundedSemaphore(value=1)
self.dev.set_configuration()
cfg = self.dev.get_active_configuration()
intf = cfg[(0, 0)]
self.endpoint_out = usb.util.find_descriptor(
intf, custom_match= \
lambda e: \
usb.util.endpoint_direction(e.bEndpointAddress) == \
usb.util.ENDPOINT_OUT)
self.endpoint_in = usb.util.find_descriptor(
intf, custom_match= \
lambda e: \
usb.util.endpoint_direction(e.bEndpointAddress) == \
usb.util.ENDPOINT_IN)
self.cmd = Commands()
# these events are checked by the thread assigned to this device and
# cleared after each one has been serviced
self.update_integ = Event()
self.update_integ.set()
self.update_average_scans = Event()
self.update_average_scans.clear()
self.change_units = Event()
self.change_units.set()
self.data_ready = Event()
self.data_ready.clear()
self.prev_integ = 1
self.dark_integ = 1
self.prev_temp = 0
self.dark_ref_taken = False
self.light_reference = []
self.dark_reference = []
self.x_data = []
self.y_data = []
self.calibration_scan = []
self.dark_pixels = []
self.auto_integration = True
self.avg_scans = 1
self.paired = False
try:
self.read()
except Exception:
pass
self.name = self.get_device_alias()
self.file_path = ''
self.calib_coeff = []
self.wavelength_indices = []
self.irradiance_data = []
self.irrad_unit = 0
self.get_hot_pixel_indices()
self.build_wavelength_indices()
self.get_irradiance_calibration()
# initialized to 10 ms integration period
self.set_integration_period(10000)
def handle(self):
self._verified = None
self._apiExists = None
self._success = False # whether the processing has been completed or not
self._request = ''
self._response = ''
self._apicommand = False
self._semaphore = BoundedSemaphore(value=1)
self.cur_thread = None
data = ''
try:
while not (('\n' in data) or ('\r' in data)):
data = data + self.request.recv(1024)
except: #if there is an error while reading, just exit
return
data = data.replace("\b", "")
log_trace('I', '0005', "Request received", detail=data)
self._request = data.strip()+'\n'
#server.queueManager
self.cur_thread = threading.current_thread()
log_trace('D', '0006', "Request received", detail = self.cur_thread.name, data = self._request)
if self._verified is None:
self.testRequest()
if self._verified == False:
if len(self._response) < 6:
log_trace('D', '0007', "Incorrect internal verification response", detail = self.cur_thread.name, data = self._response)
self._response = "ERR204"
self.request.sendall(self._response)
self._semaphore.acquire()
#insert itself to the Queue manager
self.server.queueManager.newRequest(self._request, self.callback)
#block till semaphore is open again
self._semaphore.acquire()
if self._success:
log_trace('D', '0014', "Success - sendall() called", response=self._response)
self.request.sendall(self._response)
elif self._apicommand:
#finally we can do API command processing here
self._response = self.processAPICommand(self._response)
self.request.sendall(self._response)
else:
if len(self._response) < 6:
log_trace('C', '0013', "Incorrect result from request processing", detail=self._response, command=self._request)
self._response = "ERR204"
#TODO call a class constructor and block till its processing is completed
self.request.sendall(self._response)
self._semaphore.release()
def __init__(self, host='chat.deviantart.com', version='dAmnClient 0.3', port=3900, debug=False):
"""Initialise up in this motherfucka"""
Thread.__init__(self)
self.server.host = host
self.server.version = version
self.server.port = port
self.sock = None
self.packet = []
self.__buffer = str()
self._lock = BoundedSemaphore()
self._debug = debug
class rpiFIFOClass(deque): """ Implements the a Deque with BoundedSemaphore. Used as a FIFO buffer for the image file names (including the full path). Stores also the name of the current sub-folder. """ def __init__(self, *args): super(rpiFIFOClass,self).__init__(*args) self.FIFOSema = BoundedSemaphore() self.crtSubDir = '/' self.camID = '' def acquireSemaphore(self): self.FIFOSema.acquire() def releaseSemaphore(self): self.FIFOSema.release() def __del__(self): # self.FIFOSema.release() self.crtSubDir = ''
class BoundedThreadPoolExecutor(object):
"""
The default python ThreadPoolExecutor lets its queue
get infinitely long. This can use excessive memory without
any performance benefit. This class bounds the thread queue
and only adds more items when needed.
Credit: https://www.bettercodebytes.com/theadpoolexecutor-with-a-bounded-queue-in-python/
"""
def __init__(self, bound, max_workers):
self.executor = ThreadPoolExecutor(max_workers=max_workers)
self.semaphore = BoundedSemaphore(bound + max_workers)
def future_done(self, future):
self.semaphore.release()
exception = future.exception()
if exception is not None:
raise exception
def submit(self, fn, *args, **kwargs):
self.semaphore.acquire()
try:
future = self.executor.submit(fn, *args, **kwargs)
except ValueError:
self.semaphore.release()
raise
else:
future.add_done_callback(lambda x: self.future_done(future))
return future
def shutdown(self, wait=True):
self.executor.shutdown(wait)
class ThreadPool:
""" Pool of threads consuming tasks from a queue """
def __init__(self, num_threads):
self._results_queue = Queue()
self._exceptions_queue = Queue()
self._tasks_queue = Queue()
self._sem = BoundedSemaphore(num_threads)
self._num_threads = num_threads
def add_task(self, func, *args, **kargs):
"""
Add a task to the queue. Calling this function can block
until workers have a room for processing new tasks. Blocking
the caller also prevents the latter from allocating a lot of
memory while workers are still busy running their assigned tasks.
"""
self._sem.acquire()
cleanup_func = self._sem.release
self._tasks_queue.put((func, args, kargs, cleanup_func))
def start_parallel(self):
""" Prepare threads to run tasks"""
for _ in range(self._num_threads):
Worker(
self._tasks_queue, self._results_queue, self._exceptions_queue,
)
def result(self):
""" Stop threads and return the result of all called tasks """
# Send None to all threads to cleanly stop them
for _ in range(self._num_threads):
self._tasks_queue.put(None)
# Wait for completion of all the tasks in the queue
self._tasks_queue.join()
# Check if one of the thread raised an exception, if yes
# raise it here in the function
if not self._exceptions_queue.empty():
raise self._exceptions_queue.get()
return self._results_queue
def _low_layer_execute(request: CloudSDKLowLayerRequest):
"""
执行单个底层请求
:param request: 内部请求对象
:return: 响应结果
"""
# 获取原生 sdk 对象
csp = request.parent.csp
action = request.parent.action
sdk_name = request.parent.csp_conf['native_sdk']
native_sdk_path = f'cloud.native_sdk.{csp}.{sdk_name}'
native_sdk = dynamic_import_class(native_sdk_path)()
# 设置并请求
native_sdk.set(request.parent.interface_conf, request.params)
req_limit = request.parent.csp_conf['req_limit']
key = f'{csp}:{action}'
if not semaphore_map.get(key):
semaphore_map[key] = BoundedSemaphore(req_limit)
sp = semaphore_map.get(key)
# 用于计算消耗时间
start_time = datetime_to_timestamp()
try_time = 3
resp = None
# 请求重试机制
while try_time:
with sp:
resp = native_sdk.request()
# 得到消耗时间
end_time = datetime_to_timestamp()
expend_time = end_time - start_time
# 将请求消耗时间控制在1秒左右，防止请求频率过高
if expend_time < 1:
time.sleep(1 - expend_time)
if 'Error' in resp:
try_time -= 1
else:
break
# 预处理原始数据
cleaner_name = request.parent.csp_conf['cleaner']
cleaner_path = f'cloud.cleaner.{csp}.{cleaner_name}'
cleaner = dynamic_import_class(cleaner_path)(request)
data = cleaner.clean(resp)
# 返回数据
return data
def bruteforce_ssh_user(target_host,
target_user,
path_dictionary,
n_processes=5,
max_fails=5,
waiting_time=3):
global found, fails, connexion_lock
connexion_lock = BoundedSemaphore(value=n_processes)
with open(path_dictionary) as dictionary:
for password in dictionary:
if found:
exit(0)
if fails > max_fails:
print("[!] Exiting: Too Many Socket Timeouts")
exit(0)
connexion_lock.acquire()
password = password.rstrip("\n")
Thread(target=connect,
args=(target_host, target_user, password,
waiting_time)).start()
class Resource:
def __init__(self):
self.MAX_CLIENTS = 3
self.sema = BoundedSemaphore(value=self.MAX_CLIENTS)
self.locked = False
def spaceAvailable(self):
return self.sema._Semaphore__value > 0
def acquire(self):
if self.spaceAvailable and not self.locked:
self.sema.acquire()
if not self.spaceAvailable():
self.locked = True
def release(self):
self.sema.release()
if self.sema._Semaphore__value == self.sema._initial_value:
self.locked = False
def isLocked(self):
return self.locked
def __init__(self):
"""
Creates an empty priority queue
Args:
None
Returns:
Nothing
"""
# We use this to speed up the queue operations
self.__data = dict()
# We use a semaphore to avoid compilation problems in Python 3.x
self.__semaphore = BoundedSemaphore(1)
self.__elements = 0
def __init__(self, uid):
self.uid = base58decode(uid)
self.ipcon = None
self.stack_id = 0
self.name = ''
self.firmware_version = [0, 0, 0]
self.binding_version = [0, 0, 0]
self.callbacks = {}
self.callbacks_format = {}
self.answer_type = -1
self.answer = None
self.answer_queue = Queue()
self.sem_write = BoundedSemaphore(value=1)
class semaphore(object):
""" Class encapsulating a semaphore to limit
number of resources.
"""
def __init__(self, *args):
self.sem = BoundedSemaphore(args[0])
def __call__(self, f):
def semfunc(*args, **kwargs):
try:
print "Trying to acquire sem=>", currentThread()
self.sem.acquire()
print "Acquired sem=>", currentThread()
try:
return f(*args, **kwargs)
except Exception, e:
raise
finally:
self.sem.release()
print "Realeaed sem=>", currentThread()
return semfunc
def __init__(self, args_):
self._args = args_
self.write2sql_thread = None
self.pool_sqlconnections = BoundedSemaphore(value=self._args.sql_max_connection)
self.userdata = {
'haveresponse' : False,
'starttime' : time.time()
}
self.mqttc, ret = self.mqtt_connect(
host=self._args.mqtt_host,
port=self._args.mqtt_port,
username=self._args.mqtt_username,
password=self._args.mqtt_password,
keepalive=self._args.mqtt_keepalive,
cafile=self._args.mqtt_cafile,
certfile=self._args.mqtt_certfile,
keyfile=self._args.mqtt_keyfile,
insecure=self._args.mqtt_insecure,
userdata=self.userdata
)
if ret != ExitCode.OK:
SignalHandler.exitus(ret, '{}:{} failed - [{}] {}'.format(self._args.mqtt_host, self._args.mqtt_port, ret, mqtt.error_string(ret)))
def __init__(self, storage_path=STORAGE_PATH):
self.log = logging.getLogger('%s.%s' % (__name__, self.__class__.__name__))
if not os.path.exists(storage_path):
self.log.debug("Creating directory (%s) for persistent storage" % (storage_path))
os.makedirs(storage_path)
os.chmod(storage_path, stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH)
elif not os.path.isdir(storage_path):
raise ImageFactoryException("Storage location (%s) already exists and is not a directory - cannot init persistence" % (storage_path))
else:
# TODO: verify that we can write to this location
pass
self.storage_path = storage_path
self.metadata_lock = BoundedSemaphore()
def discover(aDict):
"""Function docstring for discover TBD
Args:
- a_dom_id (required)
- network_id (required)
Output:
"""
from time import time
from threading import Thread, BoundedSemaphore
from zdcp.rest.ipam import network_discover as ipam_discover, address_allocate
from zdcp.devices.generic import Device
def __detect_thread(aIP, aDB, aSema):
__dev = Device(aIP)
aDB[aIP['ip']] = __dev.detect()['info']
aSema.release()
return True
start_time = int(time())
ipam = ipam_discover({'id': aDict['network_id']})
ret = {'errors': 0, 'start': ipam['start'], 'end': ipam['end']}
with DB() as db:
db.do("SELECT id,name FROM device_types")
devtypes = db.get_dict('name')
dev_list = {}
try:
sema = BoundedSemaphore(20)
for ip in ipam['addresses']:
sema.acquire()
t = Thread(target=__detect_thread, args=[ip, dev_list, sema])
t.name = "Detect %s" % ip
t.start()
for i in range(20):
sema.acquire()
except Exception as err:
ret['error'] = "Error:{}".format(str(err))
# We can now do inserts only (no update) as we skip existing :-)
with DB() as db:
sql = "INSERT INTO devices (a_dom_id, ipam_id, snmp, model, type_id, hostname) VALUES (" + aDict[
'a_dom_id'] + ",{},'{}','{}','{}','{}')"
count = 0
for ip, entry in dev_list.iteritems():
count += 1
alloc = address_allocate({
'ip': ip,
'network_id': aDict['network_id']
})
if alloc['success']:
db.do(
sql.format(alloc['id'], entry['snmp'], entry['model'],
devtypes[entry['type']]['id'],
"unknown_%i" % count))
ret['time'] = int(time()) - start_time
ret['found'] = len(dev_list)
return ret
class BoundedThreadPoolExecutor(concurrent.futures.ThreadPoolExecutor):
"""A ThreadPoolExecutor which has a limit on the number of jobs
which can be submitted at once
"""
def __init__(self, *args, queue_limit=10000, **kwargs):
self.semaphore = BoundedSemaphore(
kwargs.get('max_workers', 0) + queue_limit)
super().__init__(*args, **kwargs)
def submit(self, *args, **kwargs):
func, *args = args
self.semaphore.acquire()
try:
future = super().submit(func, *args, **kwargs)
except:
# if anything goes wrong, we need to release the semaphore
self.semaphore.release()
raise
else:
# release the semaphore once the thread ends
future.add_done_callback(lambda x: self.semaphore.release())
return future
class MaxQueuePool(object):
"""This Class wraps a concurrent.futures.Executor
limiting the size of its task queue.
If `max_queue_size` tasks are submitted, the next call to submit will
block until a previously submitted one is completed.
Brought in from:
* https://gist.github.com/noxdafox/4150eff0059ea43f6adbdd66e5d5e87e
See also:
* https://www.bettercodebytes.com/
theadpoolexecutor-with-a-bounded-queue-in-python/
* https://pypi.org/project/bounded-pool-executor/
* https://bugs.python.org/issue14119
* https://bugs.python.org/issue29595
* https://github.com/python/cpython/pull/143
"""
def __init__(self, executor, max_queue_size, max_workers=None):
self.pool = executor(max_workers=max_workers)
self.pool_queue = BoundedSemaphore(max_queue_size)
def submit(self, function, *args, **kwargs):
"""Submits a new task to the pool, blocks if Pool queue is full."""
self.pool_queue.acquire()
future = self.pool.submit(function, *args, **kwargs)
future.add_done_callback(self.pool_queue_callback)
return future
def pool_queue_callback(self, _):
"""Called once task is done, releases one queue slot."""
self.pool_queue.release()
def __enter__(self):
return self
def __exit__(self, exception_type, exception_value, traceback):
self.pool.__exit__(exception_type, exception_value, traceback)
class ProviderManaged(Provider):
max_parallel_requests = 2
def __init__(self, provider):
super().__init__(provider.provider_id)
self.provider = provider
self.state = ProviderStateEnum.active
self.bounded_semaphore = BoundedSemaphore(
value=self.max_parallel_requests)
self.is_alive = True
def exclude(self):
self.state = ProviderStateEnum.inactive
def include(self):
self.state = ProviderStateEnum.active
def get(self):
super().get()
print("release")
self.bounded_semaphore.release()
class Pool:
def __init__(self, count: int):
self.count = count
#池中放着链接备用
self.pool = [self._connect("conn-{}".format(i)) for i in range(count)]
self.bsemaphore = BoundedSemaphore(count)
#创建连接方法，返回一个连接对象
def _connect(self, conn_name):
return Conn(conn_name)
#获取一个链接
def get_conn(self):
self.bsemaphore.acquire()
self.pool.pop()
logging.info("从连接池拿走了一个连接~~~~~~~")
#归还一个连接
def return_conn(self, conn: Conn):
logging.info("归还了一个连接----------")
self.pool.append(conn)
self.bsemaphore.release()
def __init__(self):
try:
shutil.rmtree(osconfig.statisticsPath())
except:
pass
os.makedirs(osconfig.statisticsPath())
self.statsMap = {}
self.statsLock = BoundedSemaphore(1)
self.pptFailureCount = 0
self.pdfFailureCount = 0
self.retrieveFailureCount = 0
def __init__(self,
outlets,
beverage_names,
ingredient_to_quantity_map=None,
ingredient_to_threshold_for_notification=None):
self.id = id(self)
self.outlets = outlets
self.beverage_names = beverage_names
self.ingredient_to_quantity_map = ingredient_to_quantity_map or {}
self.ingredient_to_threshold_for_notification = ingredient_to_threshold_for_notification or {}
self.lock = Lock() # for handling concurrent access to quantity maps
self.outlet_semaphore = BoundedSemaphore(
value=outlets) # to allow maximum of #outlets dispense requests
class MyThread(Thread):
def __init__(self,nthreads):
self.threadLimiter = BoundedSemaphore(nthreads)
self.executing=[]
self.done=[]
self.lock=Lock()
def run(self,cmd):
self.threadLimiter.acquire()
try:
#print("executing: ",self.executing)
# print(id," : started")
self.Executemycode(cmd)
finally:
# print(id," : ended")
self.lock.acquire()
self.done.append(cmd)
self.executing.remove(cmd)
self.lock.release()
self.threadLimiter.release()
#print("done: \t\t\t\t\t",self.done)
def Executemycode(self,cmd):
self.lock.acquire()
self.executing.append(cmd)
self.lock.release()
result=subprocess.run(cmd,shell=True,check=True,
capture_output=True,text=True)
return 0
def executedList(self):
return self.done
def __init__(self,
num_workers=20,
filename=None,
ignore_cycles=False,
local=False,
output=None,
error=None,
silent=False,
perf=False,
keep_going=True):
self.pool = ThreadPool(num_workers=num_workers)
self.current_package = self.get_current_package()
self.current_project = {'name': None, 'path': None, 'version': None}
self.packages = {}
self.counter = 0
self.semaphore = BoundedSemaphore(1)
self.local = local
self.ignore_cycles = ignore_cycles
self.output = output
self.error = error
self.silent = silent
self.perf = perf
self.keep_going = keep_going
if self.perf is not False:
f = open(self.perf, 'w+')
f.close()
if output is not None:
if not os.path.exists(output):
print "path", output, "no exits"
sys.exit(-1)
if not os.path.isdir(output):
print "path", output, "no a valid directory"
sys.exit(-1)
self.get_current_project()
self.instanciate_packages(filename)
if self.local: self.get_local_graph()
self.check_cycles()
def __init__(self, cluster_params, query_timeout=2,
concurrent_queries=100):
self.__closed = False
self.__task_semaphore = BoundedSemaphore(concurrent_queries)
self.__cluster_params = cluster_params
self.__query_timeout = query_timeout
self.__connection_lock = RLock()
self.__cluster = None
self.__session = None
self.__connection_monitor_thread = Thread(
target=_monitor_control_connection, args=(weakref.ref(self),)
)
self.__connection_monitor_thread.start()
class semaphore(object):
""" Class encapsulating a semaphore to limit
number of resources """
def __init__(self, *args):
self.sem = BoundedSemaphore(args[0])
def __call__(self, f):
def semfunc(*args, **kwargs):
try:
# print 'Trying to acquire sem=>',currentThread()
self.sem.acquire()
# print 'Acquired sem=>',currentThread()
try:
return f(*args, **kwargs)
except Exception, e: #@UnusedVariable
raise
finally:
self.sem.release()
# print 'Released sem=>',currentThread()
return semfunc
def __new__(cls, *p, **k):
if cls.instance is None:
i = super(ReservationManager, cls).__new__(cls, *p, **k)
# initialize here, not in __init__()
i.log = logging.getLogger('%s.%s' %
(__name__, i.__class__.__name__))
i.default_minimum = cls.DEFAULT_MINIMUM
i._mounts = dict()
i.appconfig = ApplicationConfiguration().configuration
i._queues = dict(local=BoundedSemaphore(
i.appconfig.get('max_concurrent_local_sessions', 1)),
ec2=BoundedSemaphore(
i.appconfig.get('max_concurrent_ec2_sessions',
1)))
i._named_locks = {}
i._named_locks_lock = BoundedSemaphore()
# Initialize based on PID to prevent conflicts from multiple CLI runs on the same machine
# TODO: This is TinMan/Oz specific - move it to the plugin
i._listen_port = cls.MIN_PORT + (os.getpid() %
(cls.MAX_PORT - cls.MIN_PORT))
i._listen_port_lock = BoundedSemaphore()
cls.instance = i
return cls.instance
def __init__(self, n_process=2, n_threads=5, n_tasks=10, daemon=False): """ :param n_process: :type n_process: :param n_threads: :type n_threads: :param n_tasks: :type n_tasks: :param daemon: :type daemon: """ self.daemon = daemon self.n_taks = n_tasks self.n_threads = n_threads self.n_process = n_process self.sem_threads = BoundedSemaphore(self.n_threads) self.sem_tasks = asyncio.BoundedSemaphore(self.n_taks) self.running_process = []
def __init__(self, a, b):
print("init thread")
super(GTKThread, self).__init__()
self.mainloop = gobject.MainLoop()
signal.signal(signal.SIGQUIT, self.handler)
self.image = gtk.Image()
self.box1 = gtk.VBox(False, 0)
self.box1.pack_start(self.image)
self.quitButton = gtk.Button('Quit')
self.quitButton.connect('clicked', self.gtkQuitCB)
self.testLoopButton = gtk.Button('testLoop')
self.testLoopButton.connect('clicked', self.testLoopCB)
self.pngToArrayButton = gtk.Button('pngToArray')
self.pngToArrayButton.connect('clicked', self.pngToArrayCB)
self.readAndDisplayButton = gtk.Button('readAndDisplay')
self.readAndDisplayButton.connect('clicked', self.readAndDisplayCB)
self.box1.pack_start(self.testLoopButton, True, True, 0)
self.box1.pack_start(self.pngToArrayButton, True, True, 0)
self.box1.pack_start(self.readAndDisplayButton, True, True, 0)
self.box1.pack_start(self.quitButton, True, True, 0)
self.mainWindow = gtk.Window(gtk.WINDOW_TOPLEVEL)
self.task = 0
try:
self.mainWindow.set_icon_from_file('icon.png')
except:
print('could not find icon.png. using question mark icon.')
self.mainWindow.set_border_width(10)
self.mainWindow.move(1024, 0)
#self.mainWindow.connect("event",event)
#self.mainWindow.connect("expose_event",expose)
#self.mainWindow.connect("window_state_event",window_state_changed)
self.mainWindow.connect("destroy", lambda w: self.gtkQuitCB())
self.mainWindow.connect("delete_event", lambda a, b: self.gtkQuitCB())
self.mainWindow.add(self.box1)
self.mainWindow.show_all()
self.i = 0
self.j = 0
self.a = a
self.b = b
self.busySem = BoundedSemaphore(value=1)
self.updatePixelBuf()
self.setCropped(0, 0)
self.stop = False
self.start()
try:
self.mainloop.run()
except KeyboardInterrupt:
self.stop = True
self.mainloop.quit()
class semaphore(object):
""" Class encapsulating a semaphore to limit
number of resources """
def __init__(self, *args):
self.sem = BoundedSemaphore(args[0])
def __call__(self, f):
def semfunc(*args, **kwargs):
try:
print 'Trying to acquire sem=>',currentThread()
self.sem.acquire()
print 'Acquired sem=>',currentThread()
try:
return f(*args, **kwargs)
except Exception, e:
raise
finally:
self.sem.release()
print 'Released sem=>',currentThread()
return semfunc
def __init__(self, pin, active_high):
"""
Initializes a Buzzer object
Inputs: pin - The hardware pin the buzzer is connected to
"""
Thread.__init__(self)
# set to true when buzzer pin is initialized
self.pin_initialized = False
# Board pin where buzzer is located (-1 to disable)
self.pin = pin
# True if buzzer sounds when pin is HIGH; False if buzzer sounds when pin is LOW
self.active_high = active_high
# Set all default settings
self._set_default_settings()
# Used in buzz thread
self._current_buzz_list = []
self._buzz_sem = BoundedSemaphore(1)
self._buzz_sem.acquire()
self._buzz_cancel = False
self._buzz_data_lock = RLock()
self._buzz_condition = Condition()
# Running flag just to ensure that we stop buzzing when shutting down
self._running = True
class Client(Thread):
# variables de classe
drinking = 0
drink_mutex = Semaphore(1) # pour protéger les maj du compteur <drinking>
enter_mutex = BoundedSemaphore(1) # pour respecter la contrainte
def __init__(self, id, fifo):
super().__init__() # Thread.__init__(self)
self._id = id
self._fifo = fifo
def run(self):
self.to_arrive() # pour "espacer" les arrivees
Client.enter_mutex.acquire()
Client.drink_mutex.acquire()
if Client.drinking < Seats - 1:
Client.enter_mutex.release()
# "zone" où il ne doit pas y avoir plus <Seats> consommateurs
Client.drinking += 1
state = "s'installe et boit"
item = "client {0:3d} {2:30s} compteur = {1:d}".format(
self._id, Client.drinking, state)
self._fifo.put(item)
Client.drink_mutex.release()
# phase de consommation
self.drink()
# Sortie d'un client
Client.drink_mutex.acquire()
Client.drinking -= 1
state = "quitte sa place"
item = "client {0:3d} {2:30s} compteur = {1:d}".format(
self._id, Client.drinking, state)
self._fifo.put(item)
if Client.drinking == Seats - 1:
Client.enter_mutex.release()
Client.drink_mutex.release()
def drink(self):
sleep(uniform(1, 3))
def to_arrive(self):
sleep(uniform(0, 20))
# sleep(uniform(0, Nb_clients // 2))
item = 'client {:3d} arrive au bar'.format(self._id)
self._fifo.put(item)
def _after_init(self):
# Populate lists of the databases used, the parsers and the sources
for model in self.models:
for source in model.source:
self.sources.add(source)
for db in model.database:
self.databases.add(db)
# Restrict the amount of source workers working at the same time.
self.semaphore = BoundedSemaphore(self.num_sources)
for source in self.sources:
source.semaphore = self.semaphore
self.validate()
def __init__(self):
super().__init__()
self._sid = ""
self._hb_interval = -1 # Heartbeat interval
self._hb_timeout = -1 # Heartbeat timeout
self._transports = "" # Valid transports
self._client = None
self._client_ready = False
# This bounded semaphore will ensure that only one thread can be
# connecting to the client at a time
self._connection_semaphore = BoundedSemaphore(1)
self._socket_url_protocol = "http"
self._socket_url_base = ""
self._stopping = False
self._disconnect_thread = None
def map_task(source, func, thread_limit=10):
'''Run func in up to thread_limit threads, with data from
source arg passed into it.
The arg source must be iterable. map_task() will call next()
each time a free thread is available.
The function will block until all of the tasks are completed.
'''
assert thread_limit > 0
e = BoundedSemaphore(thread_limit)
task_list = []
for i in xrange(0, thread_limit):
task_list.append(task(func, e))
iterer = source.__iter__()
data = None
while 1:
try:
if data is None:
data = iterer.next()
t = get_free_task(task_list)
if t:
t.data = data
t.start()
data = None
else:
# print >> sys.stderr, 'waiting for e'
e.acquire()
e.release()
except StopIteration:
# iteration is stopped
# print >>sys.stderr, 'terminating'
for a_task in task_list:
# print >>sys.stderr, 'terminating ' + str(a_task)
a_task.terminate_and_join()
return
def __init__(self, item_number, person_capacity):
'''The store class.
Keyword arguments:
item_number -- The number of items that are in the store.
person_capacity -- The maximum amount of people allowed in the store.
'''
self.__item_number = item_number
self.__person_capacity = person_capacity
if Store.item_number is None:
Store.item_number = self.__item_number
if Store.people_in_store is None:
Store.people_in_store = BoundedSemaphore(self.__person_capacity)
class Store(object):
def __init__(self, item_number, person_capacity):
if type(item_number) is not int:
raise Exception("item_number is not an integer")
if type(person_capacity) is not int:
raise Exception("person_capacity is not an integer")
self.item_number = item_number
self.person_capacity = person_capacity
self.sema = BoundedSemaphore(value=self.person_capacity)
def enter(self):
self.sema.acquire()
def buy(self):
sleep(randint(5, 10))
if self.item_number:
purchase = True
self.item_number -= 1
else:
purchase = False
self.sema.release()
return purchase
def __init__(
self,
bounded_semaphore=None,
thread_pool_executor=None,
*args,
**kwargs
):
self.executor = (
thread_pool_executor
if thread_pool_executor is not None
else futures.ThreadPoolExecutor(*args, **kwargs)
)
self.semaphore = (
bounded_semaphore
if bounded_semaphore is not None
else BoundedSemaphore(value=self.executor._max_workers * 10)
)
def main():
webservers = Counter()
t = time.time()
lock = Lock()
semaphore = BoundedSemaphore(10)
threads = []
for host in get_hosts():
thread = Thread(target=get_webserver_for_host,
args=(host, webservers, lock, semaphore))
thread.start()
threads.append(thread)
for thread in threads:
thread.join()
print("Done in {}sec".format(time.time() - t))
print(str(webservers))
def __init__(self, queue, num_threads, timeout=0):
"""
Init thread worker
:param Queue.Queue queue: simple queue object
:param int num_threads: threads numbers
:param int timeout: delay timeout
"""
super(Worker, self).__init__()
self.__semaphore = BoundedSemaphore(num_threads)
self.__event = Event()
self.__event.set()
self.__empty = False
self.__running = True
self.__queue = queue
self.__timeout = timeout
self.counter = 0
class ThreadedGroup:
""" An object to keep an arbitary number of workers on an easily-split
task.
"""
def __init__(self):
""" Initialise self """
self._job_count = 0 # The number of active jobs.
self.lock = Lock() # Lock protecting job_count...
self.semaphore = BoundedSemaphore(THREAD_COUNT)
def start(self, func, *args, **kargs):
""" Start the given job. This will block until there are free
workers.
"""
def job_wrapper():
""" Run the job, then decrement the job count and release the
semaphore.
"""
try:
func(*args, **kargs)
finally:
with self.lock:
self._job_count -= 1
self.semaphore.release()
# Create the job.
job = Job(job_wrapper)
# Acquire the semaphore and start.
self.semaphore.acquire()
with self.lock:
self._job_count += 1
job.start()
def wait(self):
""" Wait until all of the jobs are finished """
print("Waiting for jobs to finish...")
while self.get_job_count() != 0:
# Block until another job ends.
self.semaphore.acquire()
print("Jobs finished!")
def get_job_count(self):
""" Return the current job count """
with self.lock:
return self._job_count
def _singleton_init(self):
self.log = logging.getLogger('%s.%s' % (__name__, self.__class__.__name__))
self.pending_uploads = dict()
self.pending_uploads_lock = BoundedSemaphore()
self.pim = PersistentImageManager.default_manager()
self.res = ReservationManager()
self.secondaries = { }
if os.path.isfile(SECONDARIES):
try:
secs = open(SECONDARIES, "r")
self.secondaries = json.load(secs)
secs.close()
except:
self.log.warning("Unable to load JSON for secondaries from %s", SECONDARIES)
if not 'targets' in self.secondaries:
self.secondaries['targets'] = { }
if not 'providers' in self.secondaries:
self.secondaries['providers'] = { }
class Results:
def __init__(self):
self.mutex = BoundedSemaphore(value=1)
self.__issues = []
def __len__(self):
return len(self.__issues)
# ------------------------------------------------
# data functions
# ------------------------------------------------
def add_result(self, result):
self.mutex.acquire()
self.__issues.append(result)
self.mutex.release()
def get_result(self, index):
self.mutex.acquire()
item = self.__issues[index]
self.mutex.release()
return item
class BlockingRequest():
def __init__(self):
self.semaphore = BoundedSemaphore(1)
self.exception = None
self.response = None
self.semaphore.acquire(True)
def response_callback(self, response):
self.response = response
self.semaphore.release()
def error_callback(self, exception):
self.exception = exception
self.semaphore.release()
''' returns the response or throws an exception '''
def await_response(self):
self.semaphore.acquire(True)
if self.exception :
raise self.exception
return self.response
def __init__(self, n_process=2, n_threads=5, n_tasks=10, daemon=False): """ :param n_process: :type n_process: :param n_threads: :type n_threads: :param n_tasks: :type n_tasks: :param daemon: :type daemon: """ self.daemon = daemon self.n_taks = n_tasks self.n_threads = n_threads self.n_process = n_process self.sem_threads = BoundedSemaphore(self.n_threads) self.sem_tasks = asyncio.BoundedSemaphore(self.n_taks) self.running_process = []
