class CGSBoundedSemaphore: def __init__(self,value): self.boundedSemaphore = BoundedSemaphore(value) self.datasetQueueLock = RLock() self.datasetQueue = [] def acquire(self,datasetId): try: self.datasetQueueLock.acquire() self.datasetQueue.append(datasetId) finally: self.datasetQueueLock.release() self.boundedSemaphore.acquire() def release(self,datasetId): try: self.datasetQueueLock.acquire() self.datasetQueue.remove(datasetId) except: pass finally: self.datasetQueueLock.release() self.boundedSemaphore.release() def getIndexDatasetId(self,datasetId): try: self.datasetQueueLock.acquire() return self.datasetQueue.index(datasetId) except: return -1 finally: self.datasetQueueLock.release() def status(self): return list(self.datasetQueue)
def create_document(main_url, max_connections=2, filepath=None): """Creates an EPUB document from a fanfic. main_url -- user given URL which should be the first chapter max_connections -- maximum number of simultaneous connections default: 2. This should be chosen with care as the Terms of Service of some of the websites states that you shouldn't cause more stress than a normal visitor. filepath -- optional path for the resulting Epub document By default filename is: %author - %title.epub in the current directory. %author and %title in the path are special, they're changed to the story author and title respectively.""" global dl_semaphore dl_semaphore = BoundedSemaphore(max_connections) parse, parse_ch1 = get_parser(main_url) html_ch1, chapter_num, story_info = get_chapter1(main_url, parse_ch1) chapters = {} chapters[1] = html_ch1 if story_info["cover_url"]: cover_image_req = Request( story_info["cover_url"], headers=story_info["cover_headers"]) cover_image = urlopen(cover_image_req).read() else: cover_image = open("default.jpg", "rb").read() with concurrent.futures.ThreadPoolExecutor(max_workers=max_connections+3) \ as executor: parse_chapters = [] download_urls = story_info["chapter_urls"] for ch in range(2, chapter_num+1): dl_semaphore.acquire() parse_chapters.append( executor.submit(get_chapter, download_urls[ch], ch, parse)) for future in concurrent.futures.as_completed(parse_chapters): html, chapter_no = future.result() chapters[chapter_no] = html if not filepath: filepath = "{} - {}.epub".format( INVALID_CHARS.sub("-", story_info["author"]), INVALID_CHARS.sub("-", story_info["title"])) else: filepath = filepath.replace( "%author", INVALID_CHARS.sub("-", story_info["author"])) filepath = filepath.replace( "%title", INVALID_CHARS.sub("-", story_info["title"])) with zipfile.ZipFile(filepath, "w") as f: f.writestr("mimetype", MIMETYPE) f.writestr("META-INF/container.xml", CONTAINER_XML) f.writestr("Content/titlepage.html", TITLEPAGE_HTML) f.writestr("Content/styles.css", STYLES_CSS) f.writestr("Content/cover.jpg", cover_image) f.writestr("Content/toc.ncx", toc(story_info, chapter_num)) f.writestr("Content/content.opf", contents(story_info, chapter_num)) for ch in range(1, chapter_num+1): f.writestr("Content/Chapters/ch{}.html".format(ch), chapters[ch])
def execute_semaphored_threads(): inputs = list(range(800, 1000)) print("Calculating from {} to {}".format(inputs[0], inputs[-1])) # only four threads at time pool_sema = BoundedSemaphore(value=4) threads = [] for i in inputs: # limit threads amount pool_sema.acquire() t = Thread(target=execute_fib, args=(i,)) threads.append(t) t.start() pool_sema.release() return threads
def __init__(self): self.__logger = logging.getLogger("framework.facade") self.mCallBacks = None self.mutex = BoundedSemaphore(value=1) self.controller = None # Messages queue self.__qmsg = Queue() self.jman = None # results storage self.__results = Results() # load plugins self.__logger.info("* Loading plugins") self.load_plugins() # load session self.__logger.info("* Loading session") self.__cur_session = DataModel() if Settings().get(Settings.SEC_GRL, Settings.AUTO_SESSION) == 'True': try: self.__cur_session = self.load_session() except Exception: pass
def __init__(self, repos): self.ui = getglobalui() self.repos = repos self.config = repos.getconfig() self.tunnel = repos.getpreauthtunnel() self.usessl = repos.getssl() self.username = repos.getuser() self.password = None self.passworderror = None self.goodpassword = None self.hostname = repos.gethost() self.port = repos.getport() if self.port == None: self.port = 993 if self.usessl else 143 self.sslclientcert = repos.getsslclientcert() self.sslclientkey = repos.getsslclientkey() self.sslcacertfile = repos.getsslcacertfile() if self.sslcacertfile is None: self.verifycert = None # disable cert verification self.delim = None self.root = None self.maxconnections = repos.getmaxconnections() self.availableconnections = [] self.assignedconnections = [] self.lastowner = {} self.semaphore = BoundedSemaphore(self.maxconnections) self.connectionlock = Lock() self.reference = repos.getreference() self.idlefolders = repos.getidlefolders() self.gss_step = self.GSS_STATE_STEP self.gss_vc = None self.gssapi = False
class VDOM_semaphore(object): def __init__(self, counter=1): self.__semaphore = BoundedSemaphore(counter) def lock(self): return self.__semaphore.acquire() def unlock(self): self.__semaphore.release() def __enter__(self): self.lock() return self def __exit__(self, extype, exvalue, traceback): self.unlock()
def __init__(self, storage_driver, table_info_repo, concurrent_tasks=1000, batch_chunk_size=25, schema_operation_timeout=300): self._storage_driver = storage_driver self._table_info_repo = table_info_repo self._batch_chunk_size = batch_chunk_size self._schema_operation_timeout = schema_operation_timeout self.__task_executor = ThreadPoolExecutor(concurrent_tasks) self.__task_semaphore = BoundedSemaphore(concurrent_tasks)
def __init__(self, item_number, person_capacity): if type(item_number) is not int: raise Exception("item_number is not an integer") if type(person_capacity) is not int: raise Exception("person_capacity is not an integer") self.item_number = item_number self.person_capacity = person_capacity self.sema = BoundedSemaphore(value=self.person_capacity)
class IDCardFactory(Factory): def __init__(self): self.owner = [] self.seqNum = 1 self.semaphore = BoundedSemaphore(1) def createProduct(self, owner): self.semaphore.acquire() card = IDCard(self.seqNum, owner) self.seqNum += 1 self.semaphore.release() return card def registerProduct(self, product): self.owner.append(product.getOwner()) def getOwners(self): return self.owner
def __init__(self, subscription): super(Latest.Sink, self).__init__(subscription) self.gate = RLock() self.semaphore = BoundedSemaphore(1) self.semaphore.acquire() self.notificationAvailable = False self.kind = None self.value = None self.error = None
def __init__(self, subscription): super(Next.Sink, self).__init__(subscription) self.gate = RLock() self.semaphore = BoundedSemaphore(1) self.semaphore.acquire() self.waiting = False self.kind = None self.value = None self.error = None
class MyThread(threading.Thread): def __init__(self,site): self.site = site threading.Thread.__init__(self) self.semaphore = BoundedSemaphore(value=MAXCONN) self.t = Tweetya() def run(self): link = self.t.parse(self.site) self.semaphore.acquire() urls = self.t.getlinks() for i in link: if not (i in urls): self.t.setlink(i) short = self.t.short(i) title = self.t.gettitle(short) self.t.auth(str(title)+' '+str(short)) self.semaphore.release()
def __init__(self, dAmn, logging=True, debug=False): super(Logger, self).__init__() self.dAmn = dAmn self.logging = logging self.mute_channels = [] self.debug = debug self.running = False self._run = False # Create locks self.qlock = BoundedSemaphore() #self.mlock = BoundedSemaphore() self.llock = BoundedSemaphore() # Create queues self.wqueue = [] # Output queue #self.mqueue = [] # Muter queue self.lqueue = [] # Logger queue. Just in case. # Just in case. self.subbing = False self.subthread = None
def __init__(self, n_process=2, n_threads=5, n_taks=10, daemon=False): self.daemon = daemon self.n_taks = n_taks self.n_threads = n_threads self.n_process = n_process self.sem_threads = BoundedSemaphore(self.n_threads) self.sem_tasks = asyncio.BoundedSemaphore(self.n_taks) self.running_process = []
def __init__(self, repos): self.ui = getglobalui() self.repos = repos self.config = repos.getconfig() self.preauth_tunnel = repos.getpreauthtunnel() self.transport_tunnel = repos.gettransporttunnel() if self.preauth_tunnel and self.transport_tunnel: raise OfflineImapError('%s: ' % repos + \ 'you must enable precisely one ' 'type of tunnel (preauth or transport), ' 'not both', OfflineImapError.ERROR.REPO) self.tunnel = \ self.preauth_tunnel if self.preauth_tunnel \ else self.transport_tunnel self.username = \ None if self.preauth_tunnel else repos.getuser() self.user_identity = repos.get_remote_identity() self.authmechs = repos.get_auth_mechanisms() self.password = None self.oauth2 = repos.getoauth2() self.oauth2url = repos.getoauth2url() if self.oauth2 else None self.oauth2clientid = repos.getoauth2clientid() if self.oauth2 else None self.oauth2clientsecret = repos.getoauth2clientsecret() if self.oauth2 else None self.oauth2refreshtoken = repos.getoauth2refreshtoken() if self.oauth2 else None self.passworderror = None self.goodpassword = None self.usessl = repos.getssl() self.hostname = \ None if self.preauth_tunnel else repos.gethost() self.port = repos.getport() if self.port == None: self.port = 993 if self.usessl else 143 self.sslclientcert = repos.getsslclientcert() self.sslclientkey = repos.getsslclientkey() self.sslcacertfile = repos.getsslcacertfile() self.sslversion = repos.getsslversion() if self.sslcacertfile is None: self.__verifycert = None # disable cert verification self.delim = None self.root = None self.maxconnections = repos.getmaxconnections() self.availableconnections = [] self.assignedconnections = [] self.lastowner = {} self.semaphore = BoundedSemaphore(self.maxconnections) self.connectionlock = Lock() self.reference = repos.getreference() self.idlefolders = repos.getidlefolders() self.gss_step = self.GSS_STATE_STEP self.gss_vc = None self.gssapi = False
def __init__(self, device): self.dev = device self.sts_semaphore = BoundedSemaphore(value=1) self.dev.set_configuration() cfg = self.dev.get_active_configuration() intf = cfg[(0, 0)] self.endpoint_out = usb.util.find_descriptor( intf, custom_match= \ lambda e: \ usb.util.endpoint_direction(e.bEndpointAddress) == \ usb.util.ENDPOINT_OUT) self.endpoint_in = usb.util.find_descriptor( intf, custom_match= \ lambda e: \ usb.util.endpoint_direction(e.bEndpointAddress) == \ usb.util.ENDPOINT_IN) self.cmd = Commands() # these events are checked by the thread assigned to this device and # cleared after each one has been serviced self.update_integ = Event() self.update_integ.set() self.update_average_scans = Event() self.update_average_scans.clear() self.change_units = Event() self.change_units.set() self.data_ready = Event() self.data_ready.clear() self.prev_integ = 1 self.dark_integ = 1 self.prev_temp = 0 self.dark_ref_taken = False self.light_reference = [] self.dark_reference = [] self.x_data = [] self.y_data = [] self.calibration_scan = [] self.dark_pixels = [] self.auto_integration = True self.avg_scans = 1 self.paired = False try: self.read() except Exception: pass self.name = self.get_device_alias() self.file_path = '' self.calib_coeff = [] self.wavelength_indices = [] self.irradiance_data = [] self.irrad_unit = 0 self.get_hot_pixel_indices() self.build_wavelength_indices() self.get_irradiance_calibration() # initialized to 10 ms integration period self.set_integration_period(10000)
def handle(self): self._verified = None self._apiExists = None self._success = False # whether the processing has been completed or not self._request = '' self._response = '' self._apicommand = False self._semaphore = BoundedSemaphore(value=1) self.cur_thread = None data = '' try: while not (('\n' in data) or ('\r' in data)): data = data + self.request.recv(1024) except: #if there is an error while reading, just exit return data = data.replace("\b", "") log_trace('I', '0005', "Request received", detail=data) self._request = data.strip()+'\n' #server.queueManager self.cur_thread = threading.current_thread() log_trace('D', '0006', "Request received", detail = self.cur_thread.name, data = self._request) if self._verified is None: self.testRequest() if self._verified == False: if len(self._response) < 6: log_trace('D', '0007', "Incorrect internal verification response", detail = self.cur_thread.name, data = self._response) self._response = "ERR204" self.request.sendall(self._response) self._semaphore.acquire() #insert itself to the Queue manager self.server.queueManager.newRequest(self._request, self.callback) #block till semaphore is open again self._semaphore.acquire() if self._success: log_trace('D', '0014', "Success - sendall() called", response=self._response) self.request.sendall(self._response) elif self._apicommand: #finally we can do API command processing here self._response = self.processAPICommand(self._response) self.request.sendall(self._response) else: if len(self._response) < 6: log_trace('C', '0013', "Incorrect result from request processing", detail=self._response, command=self._request) self._response = "ERR204" #TODO call a class constructor and block till its processing is completed self.request.sendall(self._response) self._semaphore.release()
def __init__(self, host='chat.deviantart.com', version='dAmnClient 0.3', port=3900, debug=False): """Initialise up in this motherfucka""" Thread.__init__(self) self.server.host = host self.server.version = version self.server.port = port self.sock = None self.packet = [] self.__buffer = str() self._lock = BoundedSemaphore() self._debug = debug
class rpiFIFOClass(deque): """ Implements the a Deque with BoundedSemaphore. Used as a FIFO buffer for the image file names (including the full path). Stores also the name of the current sub-folder. """ def __init__(self, *args): super(rpiFIFOClass,self).__init__(*args) self.FIFOSema = BoundedSemaphore() self.crtSubDir = '/' self.camID = '' def acquireSemaphore(self): self.FIFOSema.acquire() def releaseSemaphore(self): self.FIFOSema.release() def __del__(self): # self.FIFOSema.release() self.crtSubDir = ''
class BoundedThreadPoolExecutor(object): """ The default python ThreadPoolExecutor lets its queue get infinitely long. This can use excessive memory without any performance benefit. This class bounds the thread queue and only adds more items when needed. Credit: https://www.bettercodebytes.com/theadpoolexecutor-with-a-bounded-queue-in-python/ """ def __init__(self, bound, max_workers): self.executor = ThreadPoolExecutor(max_workers=max_workers) self.semaphore = BoundedSemaphore(bound + max_workers) def future_done(self, future): self.semaphore.release() exception = future.exception() if exception is not None: raise exception def submit(self, fn, *args, **kwargs): self.semaphore.acquire() try: future = self.executor.submit(fn, *args, **kwargs) except ValueError: self.semaphore.release() raise else: future.add_done_callback(lambda x: self.future_done(future)) return future def shutdown(self, wait=True): self.executor.shutdown(wait)
class ThreadPool: """ Pool of threads consuming tasks from a queue """ def __init__(self, num_threads): self._results_queue = Queue() self._exceptions_queue = Queue() self._tasks_queue = Queue() self._sem = BoundedSemaphore(num_threads) self._num_threads = num_threads def add_task(self, func, *args, **kargs): """ Add a task to the queue. Calling this function can block until workers have a room for processing new tasks. Blocking the caller also prevents the latter from allocating a lot of memory while workers are still busy running their assigned tasks. """ self._sem.acquire() cleanup_func = self._sem.release self._tasks_queue.put((func, args, kargs, cleanup_func)) def start_parallel(self): """ Prepare threads to run tasks""" for _ in range(self._num_threads): Worker( self._tasks_queue, self._results_queue, self._exceptions_queue, ) def result(self): """ Stop threads and return the result of all called tasks """ # Send None to all threads to cleanly stop them for _ in range(self._num_threads): self._tasks_queue.put(None) # Wait for completion of all the tasks in the queue self._tasks_queue.join() # Check if one of the thread raised an exception, if yes # raise it here in the function if not self._exceptions_queue.empty(): raise self._exceptions_queue.get() return self._results_queue
def _low_layer_execute(request: CloudSDKLowLayerRequest): """ 执行单个底层请求 :param request: 内部请求对象 :return: 响应结果 """ # 获取原生 sdk 对象 csp = request.parent.csp action = request.parent.action sdk_name = request.parent.csp_conf['native_sdk'] native_sdk_path = f'cloud.native_sdk.{csp}.{sdk_name}' native_sdk = dynamic_import_class(native_sdk_path)() # 设置并请求 native_sdk.set(request.parent.interface_conf, request.params) req_limit = request.parent.csp_conf['req_limit'] key = f'{csp}:{action}' if not semaphore_map.get(key): semaphore_map[key] = BoundedSemaphore(req_limit) sp = semaphore_map.get(key) # 用于计算消耗时间 start_time = datetime_to_timestamp() try_time = 3 resp = None # 请求重试机制 while try_time: with sp: resp = native_sdk.request() # 得到消耗时间 end_time = datetime_to_timestamp() expend_time = end_time - start_time # 将请求消耗时间控制在1秒左右,防止请求频率过高 if expend_time < 1: time.sleep(1 - expend_time) if 'Error' in resp: try_time -= 1 else: break # 预处理原始数据 cleaner_name = request.parent.csp_conf['cleaner'] cleaner_path = f'cloud.cleaner.{csp}.{cleaner_name}' cleaner = dynamic_import_class(cleaner_path)(request) data = cleaner.clean(resp) # 返回数据 return data
def bruteforce_ssh_user(target_host, target_user, path_dictionary, n_processes=5, max_fails=5, waiting_time=3): global found, fails, connexion_lock connexion_lock = BoundedSemaphore(value=n_processes) with open(path_dictionary) as dictionary: for password in dictionary: if found: exit(0) if fails > max_fails: print("[!] Exiting: Too Many Socket Timeouts") exit(0) connexion_lock.acquire() password = password.rstrip("\n") Thread(target=connect, args=(target_host, target_user, password, waiting_time)).start()
class Resource: def __init__(self): self.MAX_CLIENTS = 3 self.sema = BoundedSemaphore(value=self.MAX_CLIENTS) self.locked = False def spaceAvailable(self): return self.sema._Semaphore__value > 0 def acquire(self): if self.spaceAvailable and not self.locked: self.sema.acquire() if not self.spaceAvailable(): self.locked = True def release(self): self.sema.release() if self.sema._Semaphore__value == self.sema._initial_value: self.locked = False def isLocked(self): return self.locked
def __init__(self): """ Creates an empty priority queue Args: None Returns: Nothing """ # We use this to speed up the queue operations self.__data = dict() # We use a semaphore to avoid compilation problems in Python 3.x self.__semaphore = BoundedSemaphore(1) self.__elements = 0
def __init__(self, uid): self.uid = base58decode(uid) self.ipcon = None self.stack_id = 0 self.name = '' self.firmware_version = [0, 0, 0] self.binding_version = [0, 0, 0] self.callbacks = {} self.callbacks_format = {} self.answer_type = -1 self.answer = None self.answer_queue = Queue() self.sem_write = BoundedSemaphore(value=1)
class semaphore(object): """ Class encapsulating a semaphore to limit number of resources. """ def __init__(self, *args): self.sem = BoundedSemaphore(args[0]) def __call__(self, f): def semfunc(*args, **kwargs): try: print "Trying to acquire sem=>", currentThread() self.sem.acquire() print "Acquired sem=>", currentThread() try: return f(*args, **kwargs) except Exception, e: raise finally: self.sem.release() print "Realeaed sem=>", currentThread() return semfunc
def __init__(self, args_): self._args = args_ self.write2sql_thread = None self.pool_sqlconnections = BoundedSemaphore(value=self._args.sql_max_connection) self.userdata = { 'haveresponse' : False, 'starttime' : time.time() } self.mqttc, ret = self.mqtt_connect( host=self._args.mqtt_host, port=self._args.mqtt_port, username=self._args.mqtt_username, password=self._args.mqtt_password, keepalive=self._args.mqtt_keepalive, cafile=self._args.mqtt_cafile, certfile=self._args.mqtt_certfile, keyfile=self._args.mqtt_keyfile, insecure=self._args.mqtt_insecure, userdata=self.userdata ) if ret != ExitCode.OK: SignalHandler.exitus(ret, '{}:{} failed - [{}] {}'.format(self._args.mqtt_host, self._args.mqtt_port, ret, mqtt.error_string(ret)))
def __init__(self, storage_path=STORAGE_PATH): self.log = logging.getLogger('%s.%s' % (__name__, self.__class__.__name__)) if not os.path.exists(storage_path): self.log.debug("Creating directory (%s) for persistent storage" % (storage_path)) os.makedirs(storage_path) os.chmod(storage_path, stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH) elif not os.path.isdir(storage_path): raise ImageFactoryException("Storage location (%s) already exists and is not a directory - cannot init persistence" % (storage_path)) else: # TODO: verify that we can write to this location pass self.storage_path = storage_path self.metadata_lock = BoundedSemaphore()
def discover(aDict): """Function docstring for discover TBD Args: - a_dom_id (required) - network_id (required) Output: """ from time import time from threading import Thread, BoundedSemaphore from zdcp.rest.ipam import network_discover as ipam_discover, address_allocate from zdcp.devices.generic import Device def __detect_thread(aIP, aDB, aSema): __dev = Device(aIP) aDB[aIP['ip']] = __dev.detect()['info'] aSema.release() return True start_time = int(time()) ipam = ipam_discover({'id': aDict['network_id']}) ret = {'errors': 0, 'start': ipam['start'], 'end': ipam['end']} with DB() as db: db.do("SELECT id,name FROM device_types") devtypes = db.get_dict('name') dev_list = {} try: sema = BoundedSemaphore(20) for ip in ipam['addresses']: sema.acquire() t = Thread(target=__detect_thread, args=[ip, dev_list, sema]) t.name = "Detect %s" % ip t.start() for i in range(20): sema.acquire() except Exception as err: ret['error'] = "Error:{}".format(str(err)) # We can now do inserts only (no update) as we skip existing :-) with DB() as db: sql = "INSERT INTO devices (a_dom_id, ipam_id, snmp, model, type_id, hostname) VALUES (" + aDict[ 'a_dom_id'] + ",{},'{}','{}','{}','{}')" count = 0 for ip, entry in dev_list.iteritems(): count += 1 alloc = address_allocate({ 'ip': ip, 'network_id': aDict['network_id'] }) if alloc['success']: db.do( sql.format(alloc['id'], entry['snmp'], entry['model'], devtypes[entry['type']]['id'], "unknown_%i" % count)) ret['time'] = int(time()) - start_time ret['found'] = len(dev_list) return ret
class BoundedThreadPoolExecutor(concurrent.futures.ThreadPoolExecutor): """A ThreadPoolExecutor which has a limit on the number of jobs which can be submitted at once """ def __init__(self, *args, queue_limit=10000, **kwargs): self.semaphore = BoundedSemaphore( kwargs.get('max_workers', 0) + queue_limit) super().__init__(*args, **kwargs) def submit(self, *args, **kwargs): func, *args = args self.semaphore.acquire() try: future = super().submit(func, *args, **kwargs) except: # if anything goes wrong, we need to release the semaphore self.semaphore.release() raise else: # release the semaphore once the thread ends future.add_done_callback(lambda x: self.semaphore.release()) return future
class MaxQueuePool(object): """This Class wraps a concurrent.futures.Executor limiting the size of its task queue. If `max_queue_size` tasks are submitted, the next call to submit will block until a previously submitted one is completed. Brought in from: * https://gist.github.com/noxdafox/4150eff0059ea43f6adbdd66e5d5e87e See also: * https://www.bettercodebytes.com/ theadpoolexecutor-with-a-bounded-queue-in-python/ * https://pypi.org/project/bounded-pool-executor/ * https://bugs.python.org/issue14119 * https://bugs.python.org/issue29595 * https://github.com/python/cpython/pull/143 """ def __init__(self, executor, max_queue_size, max_workers=None): self.pool = executor(max_workers=max_workers) self.pool_queue = BoundedSemaphore(max_queue_size) def submit(self, function, *args, **kwargs): """Submits a new task to the pool, blocks if Pool queue is full.""" self.pool_queue.acquire() future = self.pool.submit(function, *args, **kwargs) future.add_done_callback(self.pool_queue_callback) return future def pool_queue_callback(self, _): """Called once task is done, releases one queue slot.""" self.pool_queue.release() def __enter__(self): return self def __exit__(self, exception_type, exception_value, traceback): self.pool.__exit__(exception_type, exception_value, traceback)
class ProviderManaged(Provider): max_parallel_requests = 2 def __init__(self, provider): super().__init__(provider.provider_id) self.provider = provider self.state = ProviderStateEnum.active self.bounded_semaphore = BoundedSemaphore( value=self.max_parallel_requests) self.is_alive = True def exclude(self): self.state = ProviderStateEnum.inactive def include(self): self.state = ProviderStateEnum.active def get(self): super().get() print("release") self.bounded_semaphore.release()
class Pool: def __init__(self, count: int): self.count = count #池中放着链接备用 self.pool = [self._connect("conn-{}".format(i)) for i in range(count)] self.bsemaphore = BoundedSemaphore(count) #创建连接方法,返回一个连接对象 def _connect(self, conn_name): return Conn(conn_name) #获取一个链接 def get_conn(self): self.bsemaphore.acquire() self.pool.pop() logging.info("从连接池拿走了一个连接~~~~~~~") #归还一个连接 def return_conn(self, conn: Conn): logging.info("归还了一个连接----------") self.pool.append(conn) self.bsemaphore.release()
def __init__(self): try: shutil.rmtree(osconfig.statisticsPath()) except: pass os.makedirs(osconfig.statisticsPath()) self.statsMap = {} self.statsLock = BoundedSemaphore(1) self.pptFailureCount = 0 self.pdfFailureCount = 0 self.retrieveFailureCount = 0
def __init__(self, outlets, beverage_names, ingredient_to_quantity_map=None, ingredient_to_threshold_for_notification=None): self.id = id(self) self.outlets = outlets self.beverage_names = beverage_names self.ingredient_to_quantity_map = ingredient_to_quantity_map or {} self.ingredient_to_threshold_for_notification = ingredient_to_threshold_for_notification or {} self.lock = Lock() # for handling concurrent access to quantity maps self.outlet_semaphore = BoundedSemaphore( value=outlets) # to allow maximum of #outlets dispense requests
class MyThread(Thread): def __init__(self,nthreads): self.threadLimiter = BoundedSemaphore(nthreads) self.executing=[] self.done=[] self.lock=Lock() def run(self,cmd): self.threadLimiter.acquire() try: #print("executing: ",self.executing) # print(id," : started") self.Executemycode(cmd) finally: # print(id," : ended") self.lock.acquire() self.done.append(cmd) self.executing.remove(cmd) self.lock.release() self.threadLimiter.release() #print("done: \t\t\t\t\t",self.done) def Executemycode(self,cmd): self.lock.acquire() self.executing.append(cmd) self.lock.release() result=subprocess.run(cmd,shell=True,check=True, capture_output=True,text=True) return 0 def executedList(self): return self.done
def __init__(self, num_workers=20, filename=None, ignore_cycles=False, local=False, output=None, error=None, silent=False, perf=False, keep_going=True): self.pool = ThreadPool(num_workers=num_workers) self.current_package = self.get_current_package() self.current_project = {'name': None, 'path': None, 'version': None} self.packages = {} self.counter = 0 self.semaphore = BoundedSemaphore(1) self.local = local self.ignore_cycles = ignore_cycles self.output = output self.error = error self.silent = silent self.perf = perf self.keep_going = keep_going if self.perf is not False: f = open(self.perf, 'w+') f.close() if output is not None: if not os.path.exists(output): print "path", output, "no exits" sys.exit(-1) if not os.path.isdir(output): print "path", output, "no a valid directory" sys.exit(-1) self.get_current_project() self.instanciate_packages(filename) if self.local: self.get_local_graph() self.check_cycles()
def __init__(self, cluster_params, query_timeout=2, concurrent_queries=100): self.__closed = False self.__task_semaphore = BoundedSemaphore(concurrent_queries) self.__cluster_params = cluster_params self.__query_timeout = query_timeout self.__connection_lock = RLock() self.__cluster = None self.__session = None self.__connection_monitor_thread = Thread( target=_monitor_control_connection, args=(weakref.ref(self),) ) self.__connection_monitor_thread.start()
class semaphore(object): """ Class encapsulating a semaphore to limit number of resources """ def __init__(self, *args): self.sem = BoundedSemaphore(args[0]) def __call__(self, f): def semfunc(*args, **kwargs): try: # print 'Trying to acquire sem=>',currentThread() self.sem.acquire() # print 'Acquired sem=>',currentThread() try: return f(*args, **kwargs) except Exception, e: #@UnusedVariable raise finally: self.sem.release() # print 'Released sem=>',currentThread() return semfunc
def __new__(cls, *p, **k): if cls.instance is None: i = super(ReservationManager, cls).__new__(cls, *p, **k) # initialize here, not in __init__() i.log = logging.getLogger('%s.%s' % (__name__, i.__class__.__name__)) i.default_minimum = cls.DEFAULT_MINIMUM i._mounts = dict() i.appconfig = ApplicationConfiguration().configuration i._queues = dict(local=BoundedSemaphore( i.appconfig.get('max_concurrent_local_sessions', 1)), ec2=BoundedSemaphore( i.appconfig.get('max_concurrent_ec2_sessions', 1))) i._named_locks = {} i._named_locks_lock = BoundedSemaphore() # Initialize based on PID to prevent conflicts from multiple CLI runs on the same machine # TODO: This is TinMan/Oz specific - move it to the plugin i._listen_port = cls.MIN_PORT + (os.getpid() % (cls.MAX_PORT - cls.MIN_PORT)) i._listen_port_lock = BoundedSemaphore() cls.instance = i return cls.instance
def __init__(self, n_process=2, n_threads=5, n_tasks=10, daemon=False): """ :param n_process: :type n_process: :param n_threads: :type n_threads: :param n_tasks: :type n_tasks: :param daemon: :type daemon: """ self.daemon = daemon self.n_taks = n_tasks self.n_threads = n_threads self.n_process = n_process self.sem_threads = BoundedSemaphore(self.n_threads) self.sem_tasks = asyncio.BoundedSemaphore(self.n_taks) self.running_process = []
def __init__(self, a, b): print("init thread") super(GTKThread, self).__init__() self.mainloop = gobject.MainLoop() signal.signal(signal.SIGQUIT, self.handler) self.image = gtk.Image() self.box1 = gtk.VBox(False, 0) self.box1.pack_start(self.image) self.quitButton = gtk.Button('Quit') self.quitButton.connect('clicked', self.gtkQuitCB) self.testLoopButton = gtk.Button('testLoop') self.testLoopButton.connect('clicked', self.testLoopCB) self.pngToArrayButton = gtk.Button('pngToArray') self.pngToArrayButton.connect('clicked', self.pngToArrayCB) self.readAndDisplayButton = gtk.Button('readAndDisplay') self.readAndDisplayButton.connect('clicked', self.readAndDisplayCB) self.box1.pack_start(self.testLoopButton, True, True, 0) self.box1.pack_start(self.pngToArrayButton, True, True, 0) self.box1.pack_start(self.readAndDisplayButton, True, True, 0) self.box1.pack_start(self.quitButton, True, True, 0) self.mainWindow = gtk.Window(gtk.WINDOW_TOPLEVEL) self.task = 0 try: self.mainWindow.set_icon_from_file('icon.png') except: print('could not find icon.png. using question mark icon.') self.mainWindow.set_border_width(10) self.mainWindow.move(1024, 0) #self.mainWindow.connect("event",event) #self.mainWindow.connect("expose_event",expose) #self.mainWindow.connect("window_state_event",window_state_changed) self.mainWindow.connect("destroy", lambda w: self.gtkQuitCB()) self.mainWindow.connect("delete_event", lambda a, b: self.gtkQuitCB()) self.mainWindow.add(self.box1) self.mainWindow.show_all() self.i = 0 self.j = 0 self.a = a self.b = b self.busySem = BoundedSemaphore(value=1) self.updatePixelBuf() self.setCropped(0, 0) self.stop = False self.start() try: self.mainloop.run() except KeyboardInterrupt: self.stop = True self.mainloop.quit()
class semaphore(object): """ Class encapsulating a semaphore to limit number of resources """ def __init__(self, *args): self.sem = BoundedSemaphore(args[0]) def __call__(self, f): def semfunc(*args, **kwargs): try: print 'Trying to acquire sem=>',currentThread() self.sem.acquire() print 'Acquired sem=>',currentThread() try: return f(*args, **kwargs) except Exception, e: raise finally: self.sem.release() print 'Released sem=>',currentThread() return semfunc
def __init__(self, pin, active_high): """ Initializes a Buzzer object Inputs: pin - The hardware pin the buzzer is connected to """ Thread.__init__(self) # set to true when buzzer pin is initialized self.pin_initialized = False # Board pin where buzzer is located (-1 to disable) self.pin = pin # True if buzzer sounds when pin is HIGH; False if buzzer sounds when pin is LOW self.active_high = active_high # Set all default settings self._set_default_settings() # Used in buzz thread self._current_buzz_list = [] self._buzz_sem = BoundedSemaphore(1) self._buzz_sem.acquire() self._buzz_cancel = False self._buzz_data_lock = RLock() self._buzz_condition = Condition() # Running flag just to ensure that we stop buzzing when shutting down self._running = True
class Client(Thread): # variables de classe drinking = 0 drink_mutex = Semaphore(1) # pour protéger les maj du compteur <drinking> enter_mutex = BoundedSemaphore(1) # pour respecter la contrainte def __init__(self, id, fifo): super().__init__() # Thread.__init__(self) self._id = id self._fifo = fifo def run(self): self.to_arrive() # pour "espacer" les arrivees Client.enter_mutex.acquire() Client.drink_mutex.acquire() if Client.drinking < Seats - 1: Client.enter_mutex.release() # "zone" où il ne doit pas y avoir plus <Seats> consommateurs Client.drinking += 1 state = "s'installe et boit" item = "client {0:3d} {2:30s} compteur = {1:d}".format( self._id, Client.drinking, state) self._fifo.put(item) Client.drink_mutex.release() # phase de consommation self.drink() # Sortie d'un client Client.drink_mutex.acquire() Client.drinking -= 1 state = "quitte sa place" item = "client {0:3d} {2:30s} compteur = {1:d}".format( self._id, Client.drinking, state) self._fifo.put(item) if Client.drinking == Seats - 1: Client.enter_mutex.release() Client.drink_mutex.release() def drink(self): sleep(uniform(1, 3)) def to_arrive(self): sleep(uniform(0, 20)) # sleep(uniform(0, Nb_clients // 2)) item = 'client {:3d} arrive au bar'.format(self._id) self._fifo.put(item)
def _after_init(self): # Populate lists of the databases used, the parsers and the sources for model in self.models: for source in model.source: self.sources.add(source) for db in model.database: self.databases.add(db) # Restrict the amount of source workers working at the same time. self.semaphore = BoundedSemaphore(self.num_sources) for source in self.sources: source.semaphore = self.semaphore self.validate()
def __init__(self): super().__init__() self._sid = "" self._hb_interval = -1 # Heartbeat interval self._hb_timeout = -1 # Heartbeat timeout self._transports = "" # Valid transports self._client = None self._client_ready = False # This bounded semaphore will ensure that only one thread can be # connecting to the client at a time self._connection_semaphore = BoundedSemaphore(1) self._socket_url_protocol = "http" self._socket_url_base = "" self._stopping = False self._disconnect_thread = None
def map_task(source, func, thread_limit=10): '''Run func in up to thread_limit threads, with data from source arg passed into it. The arg source must be iterable. map_task() will call next() each time a free thread is available. The function will block until all of the tasks are completed. ''' assert thread_limit > 0 e = BoundedSemaphore(thread_limit) task_list = [] for i in xrange(0, thread_limit): task_list.append(task(func, e)) iterer = source.__iter__() data = None while 1: try: if data is None: data = iterer.next() t = get_free_task(task_list) if t: t.data = data t.start() data = None else: # print >> sys.stderr, 'waiting for e' e.acquire() e.release() except StopIteration: # iteration is stopped # print >>sys.stderr, 'terminating' for a_task in task_list: # print >>sys.stderr, 'terminating ' + str(a_task) a_task.terminate_and_join() return
def __init__(self, item_number, person_capacity): '''The store class. Keyword arguments: item_number -- The number of items that are in the store. person_capacity -- The maximum amount of people allowed in the store. ''' self.__item_number = item_number self.__person_capacity = person_capacity if Store.item_number is None: Store.item_number = self.__item_number if Store.people_in_store is None: Store.people_in_store = BoundedSemaphore(self.__person_capacity)
class Store(object): def __init__(self, item_number, person_capacity): if type(item_number) is not int: raise Exception("item_number is not an integer") if type(person_capacity) is not int: raise Exception("person_capacity is not an integer") self.item_number = item_number self.person_capacity = person_capacity self.sema = BoundedSemaphore(value=self.person_capacity) def enter(self): self.sema.acquire() def buy(self): sleep(randint(5, 10)) if self.item_number: purchase = True self.item_number -= 1 else: purchase = False self.sema.release() return purchase
def __init__( self, bounded_semaphore=None, thread_pool_executor=None, *args, **kwargs ): self.executor = ( thread_pool_executor if thread_pool_executor is not None else futures.ThreadPoolExecutor(*args, **kwargs) ) self.semaphore = ( bounded_semaphore if bounded_semaphore is not None else BoundedSemaphore(value=self.executor._max_workers * 10) )
def main(): webservers = Counter() t = time.time() lock = Lock() semaphore = BoundedSemaphore(10) threads = [] for host in get_hosts(): thread = Thread(target=get_webserver_for_host, args=(host, webservers, lock, semaphore)) thread.start() threads.append(thread) for thread in threads: thread.join() print("Done in {}sec".format(time.time() - t)) print(str(webservers))
def __init__(self, queue, num_threads, timeout=0): """ Init thread worker :param Queue.Queue queue: simple queue object :param int num_threads: threads numbers :param int timeout: delay timeout """ super(Worker, self).__init__() self.__semaphore = BoundedSemaphore(num_threads) self.__event = Event() self.__event.set() self.__empty = False self.__running = True self.__queue = queue self.__timeout = timeout self.counter = 0
class ThreadedGroup: """ An object to keep an arbitary number of workers on an easily-split task. """ def __init__(self): """ Initialise self """ self._job_count = 0 # The number of active jobs. self.lock = Lock() # Lock protecting job_count... self.semaphore = BoundedSemaphore(THREAD_COUNT) def start(self, func, *args, **kargs): """ Start the given job. This will block until there are free workers. """ def job_wrapper(): """ Run the job, then decrement the job count and release the semaphore. """ try: func(*args, **kargs) finally: with self.lock: self._job_count -= 1 self.semaphore.release() # Create the job. job = Job(job_wrapper) # Acquire the semaphore and start. self.semaphore.acquire() with self.lock: self._job_count += 1 job.start() def wait(self): """ Wait until all of the jobs are finished """ print("Waiting for jobs to finish...") while self.get_job_count() != 0: # Block until another job ends. self.semaphore.acquire() print("Jobs finished!") def get_job_count(self): """ Return the current job count """ with self.lock: return self._job_count
def _singleton_init(self): self.log = logging.getLogger('%s.%s' % (__name__, self.__class__.__name__)) self.pending_uploads = dict() self.pending_uploads_lock = BoundedSemaphore() self.pim = PersistentImageManager.default_manager() self.res = ReservationManager() self.secondaries = { } if os.path.isfile(SECONDARIES): try: secs = open(SECONDARIES, "r") self.secondaries = json.load(secs) secs.close() except: self.log.warning("Unable to load JSON for secondaries from %s", SECONDARIES) if not 'targets' in self.secondaries: self.secondaries['targets'] = { } if not 'providers' in self.secondaries: self.secondaries['providers'] = { }
class Results: def __init__(self): self.mutex = BoundedSemaphore(value=1) self.__issues = [] def __len__(self): return len(self.__issues) # ------------------------------------------------ # data functions # ------------------------------------------------ def add_result(self, result): self.mutex.acquire() self.__issues.append(result) self.mutex.release() def get_result(self, index): self.mutex.acquire() item = self.__issues[index] self.mutex.release() return item
class BlockingRequest(): def __init__(self): self.semaphore = BoundedSemaphore(1) self.exception = None self.response = None self.semaphore.acquire(True) def response_callback(self, response): self.response = response self.semaphore.release() def error_callback(self, exception): self.exception = exception self.semaphore.release() ''' returns the response or throws an exception ''' def await_response(self): self.semaphore.acquire(True) if self.exception : raise self.exception return self.response