def reset_queue(self, queue):
""" Make sure the queue is empty """
try:
while True:
queue.get_nowait()
except:
pass
def remove_client(self, clientID):
txThread=self._thread.pop(clientID)
txThread.stop()
queue=self._rxqueue.pop(clientID)
while queue.qsize() > 0:
queue.get_nowait()
queue=self._txqueue.pop(clientID)
while queue.qsize() > 0:
queue.get_nowait()
self._logger.info("Client-ID:{0}; removed; number of clients:{1}".format(clientID, self._clientcounter))
def _task(self):
queue = self.queue
# queue can be overflow
with self.transaction_lock:
while not queue.empty():
queue.get_nowait()
queue.task_done()
queue.put(0)
# INTERNAL: start counter for task
while True:
wait = self.default_execute_wait
event = queue.get()
if event is None:
# STOP EVENT
while not queue.empty():
queue.get_nowait()
queue.task_done()
# TODO: warning not queued event?
# TODO: just new stop flag?
queue.task_done()
# for stop event.
return
time.sleep(wait)
# TODO: how to sleep automation?
# TODO: use some good schuler?
with self.transaction_lock:
current_transaction = self.current_transaction
if current_transaction is None:
with self.transaction() as current_transaction:
pass
if not current_transaction.operations:
queue.put(event + wait)
queue.task_done()
continue
if event >= self.status.default_execute_wait:
self.current_transaction = None
self.execute_transaction(current_transaction)
queue.put(0)
queue.task_done()
def _task(self):
queue = self.queue
# queue can be overflow
with self.transaction_lock:
while not queue.empty():
queue.get_nowait()
queue.task_done()
queue.put(0)
# INTERNAL: start counter for task
while True:
wait = self.default_execute_wait
event = queue.get()
if event is None:
# STOP EVENT
while not queue.empty():
queue.get_nowait()
queue.task_done()
# TODO: warning not queued event?
# TODO: just new stop flag?
queue.task_done()
# for stop event.
return
time.sleep(wait)
# TODO: how to sleep automation?
# TODO: use some good schuler?
with self.transaction_lock:
current_transaction = self.current_transaction
if current_transaction is None:
with self.transaction() as current_transaction:
pass
if not current_transaction.operations:
queue.put(event + wait)
queue.task_done()
continue
if event >= self.status.default_execute_wait:
self.current_transaction = None
self.execute_transaction(current_transaction)
queue.put(0)
queue.task_done()
def fnThreadLoop(i, queue, lock):
s = requests.Session()
while True:
#exit Thread when detect signal to quit.
while libextra.fnExitNow():
try:
r = queue.get_nowait()
break
except:
#libextra.fnQueueEmpty(1,lock)
time.sleep(0.1)
continue
if libextra.fnExitNow() == False:
break
id = r[0]
(status, gw_resp, log_msg) = fnJOB_CallAPI(s, r)
gw_resp = pymysql.escape_string(gw_resp)
#if (log_msg != ''):
# print(log_msg)
QueueUpdate.put((id, status, gw_resp))
queue.task_done()
def getFromQueue(queue):
result = None
try:
result = queue.get_nowait()
except:
pass
return result
def run(self):
while True:
self.__event.wait()
if self.shutdownFlag is True:
return
self.__event.clear()
while self._isEmpty() is False:
request = None
currentQueue = None
for queue in self.__requestQueueList:
try:
request = queue.get_nowait()
currentQueue = queue
break
except queue.Empty:
continue
if request is None:
continue
try:
self._handleRequest(request)
except Exception:
traceInfo = traceback.format_exc()
self.logger.error('Fail to handle request: %s', traceInfo)
currentQueue.task_done()
if self.__sleep_time > 0:
time.sleep(self.__sleep_time)
def fnThreadLoop(i, queue, lock):
s = requests.Session()
while True:
#exit Thread when detect signal to quit.
while libextra.fnExitNow():
try:
r = queue.get_nowait()
break
except:
#libextra.fnQueueEmpty(1,lock)
time.sleep(0.1)
continue
if libextra.fnExitNow() == False:
break
id = r[0]
(status, gw_resp, log_msg) = fnJOB_CallAPI(s,r)
gw_resp = pymysql.escape_string(gw_resp)
#if (log_msg != ''):
# print(log_msg)
QueueUpdate.put((id,status,gw_resp))
queue.task_done()
def climbMain(conn, queue):
fol = queue.get_nowait()
# get followers strings
followers = (getuserlists(username=fol.username, header=myHeader, ftype='followers'))
for val in followers:
# get followers info
userInfo = getpageinfo(username=val, header=myHeader)
print(userInfo['showname'])
user = Users(
username=userInfo['username'],
showname=userInfo['showname'],
followees=userInfo['followees'],
followers=userInfo['followers'],
focus=userInfo['focus'],
fr_status=0,
fe_status=0
)
conn.add(user)
conn.commit()
conn.query(Users).filter(Users.id == fol.id).\
update({Users.fr_status:1}, synchronize_session=False)
print('++++++'+fol.showname+'finished++++')
def worker(queue, handle=None):
try:
while True:
proxy = queue.get_nowait()
result = ""
counter[0] += 1
sys.stdout.write("\r%s\r" % ROTATION_CHARS[counter[0] % len(ROTATION_CHARS)])
sys.stdout.flush()
start = time.time()
candidate = "%s://%s:%s" % (proxy["proto"].replace("https", "http"), proxy["ip"], proxy["port"])
if not all((proxy["ip"], proxy["port"])) or re.search(r"[^:/\w.]", candidate):
continue
if not check_alive(proxy["ip"], proxy["port"]):
continue
if not FALLBACK_METHOD:
process = subprocess.Popen("curl -m %d -A \"%s\" --proxy %s %s" % (timeout, USER_AGENT, candidate, random_ifconfig()), shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
result, _ = process.communicate()
elif proxy["proto"] in ("http", "https"):
opener = build_opener(ProxyHandler({"http": candidate, "https": candidate}))
result = retrieve(random_ifconfig(), timeout=timeout, opener=opener)
if (result or "").strip() == proxy["ip"].encode("utf8"):
latency = time.time() - start
if latency < timeout:
sys.stdout.write("\r%s%s # latency: %.2f sec; country: %s; anonymity: %s (%s)\n" % (candidate, " " * (32 - len(candidate)), latency, ' '.join(_.capitalize() for _ in (proxy["country"].lower() or '-').split(' ')), proxy["type"], proxy["anonymity"]))
sys.stdout.flush()
if handle:
os.write(handle, "%s%s" % (candidate, os.linesep))
except:
pass
def _dequeue(self, queue):
"""Removes queue entries till an alive reference was found.
The referenced image holder will be returned in this case.
Otherwise if there wasn't found any alive reference
None will be returned.
Args:
queue (queue.Queue): the queue to operate on
Returns:
tuple of (ImageHolder, tuple of (width: int, height: int),
PostLoadImageProcessor):
an queued image holder or None, upper bound size or None,
the post load image processor or None
"""
holder_reference = None
image_holder = None
upper_bound_size = None
post_load_processor = None
while not queue.empty():
holder_reference, upper_bound_size, post_load_processor = \
queue.get_nowait()
image_holder = holder_reference and holder_reference()
if (holder_reference is None or image_holder is not None):
break
return image_holder, upper_bound_size, post_load_processor
def download_music(queue, folder, name):
while not queue.empty():
item = queue.get_nowait()
content = requests.get(item['url']).content
with open('%s/%s - %s.mp3' % (folder, item['artist'], item['song']),
'wb') as f:
f.write(content)
print('[线程%s]下载完成: %s - %s' % (name, item['artist'], item['song']))
def _get_results(self, queue):
'''Generator to yield one after the others all items currently
in the queue, without any waiting
'''
try:
while True:
yield queue.get_nowait()
except queue.Empty:
raise StopIteration
def tailStdOut(self):
result = []
if (self.m_scriptMonitor != None):
queue = self.m_scriptMonitor.getStdoutQueue()
for x in xrange(min(100, queue.qsize())):
try:
val = queue.get_nowait()
result.append(val)
except Empty:
pass
return {"tail": result}
def scrawler(queue, query):
time.sleep(1)
while not queue.empty():
try:
print('thread %s is running...' % threading.currentThread().name)
# 不阻塞的读取队列数据
url = queue.get_nowait()
getgroup(url, query)
except Exception as e:
print(e)
continue
def basic_incognito_algorithm(self, priority_queue):
self.init_C1_and_E1()
queue = priority_queue
marked_nodes = set()
for i in range(1, len(self.Q) + 1):
i_str = str(i)
self.cursor.execute("SELECT * FROM C" + i_str + "")
Si = set(self.cursor)
# no edge directed to a node => root
self.cursor.execute("SELECT C" + i_str + ".* FROM C" + i_str +
", E" + i_str + " WHERE C" + i_str +
".ID = E" + i_str + ".start EXCEPT SELECT C" +
i_str + ".* FROM C" + i_str + ", E" + i_str +
" WHERE C" + i_str + ".ID = E" + i_str +
".end ")
roots = set(self.cursor)
roots_in_queue = set()
for node in roots:
height = self.get_height_of_node(node)
# -height because priority queue shows the lowest first. Syntax: (priority number, data)
roots_in_queue.add((-height, node))
for upgraded_node in roots_in_queue:
queue.put_nowait(upgraded_node)
while not queue.empty():
upgraded_node = queue.get_nowait()
# [1] => pick 'node' in (-height, node);
node = upgraded_node[1]
if node[0] not in marked_nodes:
if node in roots:
frequency_set = self.frequency_set_of_T_wrt_attributes_of_node_using_T(
node)
else:
frequency_set = self.frequency_set_of_T_wrt_attributes_of_node_using_parent_s_frequency_set(
node, i)
if self.table_is_k_anonymous_wrt_attributes_of_node(
frequency_set):
self.mark_all_direct_generalizations_of_node(
marked_nodes, node, i)
else:
Si.remove(node)
self.insert_direct_generalization_of_node_in_queue(
node, queue, i, Si)
self.cursor.execute("DELETE FROM C" + str(i) +
" WHERE ID = " + str(node[0]))
self.graph_generation(Si, i)
marked_nodes = set()
def __call__(self, *args, **kwargs):
procs = []
queue = mp.Queue(self.n_gpus)
results = [None] * self.n_gpus
for dev in range(self.n_gpus):
p = mp.Process(
target=_run_wrapped,
args=(
self.func,
self.world_size > self.n_gpus,
self.master_ip,
self.port,
self.world_size,
dev + self.rank_start,
dev,
args,
kwargs,
queue,
),
)
p.start()
procs.append(p)
devs = list(range(self.n_gpus))
def terminate():
for dev in devs:
procs[dev].terminate()
devs.clear()
while len(devs) > 0:
left = []
# check all processes in one second
time_to_wait = 1.0 / len(devs)
for dev in devs:
procs[dev].join(time_to_wait)
code = procs[dev].exitcode
# terminate processes if one of them has failed
if code != 0 and code != None:
terminate()
assert (
code == 0 or code == None
), "subprocess {} exit with code {}".format(dev + self.rank_start, code)
if code == None:
left.append(dev)
elif queue.empty():
get_logger().warning(WARN_SUBPROCESS_EXIT_WITHOUT_RETURN)
else:
dev, ret = queue.get_nowait()
results[dev] = ret
devs = left
return results
def scrawler(queue):
time.sleep(1)
threedaysago = int(time.time()) - 3600 * 24 * 30
while not queue.empty():
try:
print('thread %s is running...' % threading.currentThread().name)
# 不阻塞的读取队列数据
url = queue.get_nowait()
getrentinfo(url, threedaysago)
except Exception as e:
print(e)
continue
def grab(cam, queue):
global running
global isStop
capture = cv.VideoCapture(cam)
while (1):
while (running):
capture.grab()
ret, img = capture.read()
if not ret:
break
if not queue.empty():
try:
queue.get_nowait()
except:
pass
img = cv.cvtColor(img, cv.COLOR_RGB2BGR)
queue.put(img)
while not queue.empty():
queue.get_nowait()
while isStop:
sys.exit()
def cryptobook_get(namespace):
queue = dpget(namespace, "/cryptobook/queue")
used = dpget(namespace, "/cryptobook/used")
try:
id, key, masher = queue.get_nowait()
used[id] = (key, masher)
dpset(namespace, "/key/private", key)
dpset(namespace, "/key/masher", masher)
dpset(namespace, "/key/id", id)
except queue.Empty:
namespace = cryptobook_add(namespace)
return cryptobook_get(namespace)
return namespace
def clap_worker(vc, queue):
print('creating worker')
# vc = message.guild.voice_client
# clap = discord.FFmpegPCMAudio(random.choice(files))
while vc.is_connected():
if not vc.is_playing() and not queue.empty():
try:
clap = queue.get_nowait()
vc.play(discord.FFmpegPCMAudio(clap), after=queue.task_done())
except:
pass
else:
time.sleep(1)
print('worker destroyed')
def _run():
while 1:
try:
row = queue.get_nowait()
try:
print(row)
print(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S.%f')
[:-3])
test_1()
except Exception as ee:
print(ee)
print(row[0], 'error')
except Exception as e:
print(e)
return
def work(queue):
while True:
if queue.empty():
if self.all_fetched_event.is_set():
break
else:
time.sleep(0.001)
try:
task = queue.get_nowait()
except Exception:
continue
if not os.path.exists(os.path.dirname(task.destination)):
os.makedirs(os.path.dirname(task.destination),
exist_ok=True)
with open(task.destination, 'wb') as f:
f.write(self.__download(task.url))
def read_sensor(self, gas, thermister, photoresister, tracking, queue):
data = {}
while True:
data.update({"Gas": gas.read()})
time.sleep(0.1)
data.update({"Thermister": thermister.read()})
time.sleep(0.1)
data.update({"Photoresister": photoresister.read()})
time.sleep(0.1)
data.update({"Tracking": tracking.read()})
time.sleep(0.1)
queue.put(data)
#print(queue)
time.sleep(0.1)
self.client.publish(self.pubtopic,
payload=json.dumps(queue.get_nowait()))
time.sleep(0.1)
def pollQueue(self, whatInbox):
"""This method checks all the queues from the outside world, and forwards any waiting data
to the child component. Returns False if we propogated a shutdown signal, true otherwise."""
parentSource = self.childInboxMapping[whatInbox]
queue = self.inQueues[whatInbox]
while not queue.empty():
if not self.outboxes[parentSource].isFull():
msg = queue.get_nowait() # won't fail, we're the only one reading from the queue.
try:
self.send(msg, parentSource)
except noSpaceInBox as e:
raise RuntimeError("Box delivery failed despite box (earlier) reporting being not full. Is more than one thread directly accessing boxes?")
if isinstance(msg, (Ipc.shutdownMicroprocess, Ipc.producerFinished)):
# print ("Quietly dieing?")
return False
else:
# if the component's inboxes are full, do something here. Preferably not succeed.
break
return True
def pollQueue(self, whatInbox):
"""This method checks all the queues from the outside world, and forwards any waiting data
to the child component. Returns False if we propogated a shutdown signal, true otherwise."""
parentSource = self.childInboxMapping[whatInbox]
queue = self.inQueues[whatInbox]
while not queue.empty():
if not self.outboxes[parentSource].isFull():
msg = queue.get_nowait() # won't fail, we're the only one reading from the queue.
try:
self.send(msg, parentSource)
except noSpaceInBox as e:
raise RuntimeError(
"Box delivery failed despite box (earlier) reporting being not full. Is more than one thread directly accessing boxes?"
)
if isinstance(msg, (Ipc.shutdownMicroprocess, Ipc.producerFinished)):
# print ("Quietly dieing?")
return False
else:
# if the component's inboxes are full, do something here. Preferably not succeed.
break
return True
def waitforqueue(queue, timeout=10000, maxitems=None, bot=None):
""" wait for results to arrive in a queue. return list of results. """
#if len(queue) > 1: return list(queue)
result = []
counter = 0
if not maxitems: maxitems = 100
logging.warn("waiting for queue: %s - %s" % (timeout, maxitems))
try:
while not len(queue):
if len(queue) > maxitems: break
if counter > timeout: break
time.sleep(0.001) ; counter += 10
logging.warn("waitforqueue - result is %s items (%s) - %s" % (len(queue), counter, str(queue)))
return queue
except (AttributeError, TypeError):
q = []
while 1:
if counter > timeout: break
try:
q.append(queue.get_nowait())
except queue.Empty: break
time.sleep(0.001) ; counter += 10
logging.warn("waitforqueue - result is %s items (%s) - %s" % (len(q), counter, str(q)))
return q
async def worker(name, client):
log = logging.getLogger(name)
while True:
try:
line = queue.get_nowait()
except Empty as e:
log.info('Queue is empty.')
await asyncio.sleep(1)
else:
line = line.decode('utf-8')
match = pattern.match(line)
if match is None:
log.info(f"No match found for {line.strip()}")
await asyncio.sleep(0.001)
else:
async with client.post(
"http://es01:9200/logs/_doc/",
data=json.dumps(match.groupdict()).encode('utf-8'),
headers={'Content-Type': 'application/json'}) as resp:
if resp.status != 201:
err = await resp.text()
log.info(f"{resp.status}: {err}")
else:
log.info("Upload succesful.")
def __call__(self, *args, **kwargs):
procs = []
queue = mp.Queue(self.n_gpus)
results = [None] * self.n_gpus
for dev in range(self.n_gpus):
p = mp.Process(
target=_run_wrapped,
args=(
self.func,
self.world_size > self.n_gpus,
self.master_ip,
self.port,
self.world_size,
dev + self.rank_start,
dev,
self.device_type,
args,
kwargs,
queue,
),
)
p.start()
procs.append(p)
devs = list(range(self.n_gpus))
def terminate():
for dev in devs:
procs[dev].terminate()
devs.clear()
result_count = 0
while len(devs) > 0:
left = []
# check all processes in one second
time_to_wait = 1.0 / len(devs)
for dev in devs:
procs[dev].join(time_to_wait)
code = procs[dev].exitcode
# terminate processes if one of them has failed
if code != 0 and code != None:
terminate()
assert (code == 0 or code
== None), "subprocess {} exit with code {}".format(
dev + self.rank_start, code)
if code == None:
left.append(dev)
# DO NOT delete it, multiprocess.Queue has small buffer
# fetch data early to avoid dead lock
if not queue.empty():
result_count += 1
dev, ret = queue.get_nowait()
results[dev] = ret
devs = left
while not queue.empty():
result_count += 1
dev, ret = queue.get_nowait()
results[dev] = ret
if result_count < self.n_gpus:
get_logger().warning(WARN_SUBPROCESS_EXIT_WITHOUT_RETURN)
return results
def clear_queue(queue):
try:
while True:
queue.get_nowait()
except Exception:
pass
class PriorityQueue(queue.Queue):
"Thread-safe priority queue"
def _put(self, item):
# insert in order
bisect.insort(self.queue, item)
#
# try it
queue = PriorityQueue(0)
# add items out of order
queue.put((20, "second"))
queue.put((10, "first"))
queue.put((30, "third"))
# print queue contents
try:
while 1:
print(queue.get_nowait())
except Empty:
pass
## third
## second
## first
