def close(self, error=None):
""" close down this channel with an optional error message.
Note that closing of a channel tied to remote_exec happens
automatically at the end of execution and cannot be done explicitely.
"""
if self._executing:
raise IOError("cannot explicitly close channel within remote_exec")
if self._closed:
self.gateway._trace(self, "ignoring redundant call to close()")
if not self._closed:
# state transition "opened/sendonly" --> "closed"
# threads warning: the channel might be closed under our feet,
# but it's never damaging to send too many CHANNEL_CLOSE messages
# however, if the other side triggered a close already, we
# do not send back a closed message.
if not self._receiveclosed.isSet():
put = self.gateway._send
if error is not None:
put(Message.CHANNEL_CLOSE_ERROR, self.id, dumps_internal(error))
else:
put(Message.CHANNEL_CLOSE, self.id)
self._trace("sent channel close message")
if isinstance(error, RemoteError):
self._remoteerrors.append(error)
self._closed = True # --> "closed"
self._receiveclosed.set()
queue = self._items
if queue is not None:
queue.put(ENDMARKER)
self.gateway._channelfactory._no_longer_opened(self.id)
def init(name, number=10):
global cnt
global visited
# blog_name = input('输入博客名称:')
# thread_num = input('输入启动线程数:')
blog_name = name.lower()
th_num = int(number)
url = 'http://blog.csdn.net/' + blog_name + '/'
opener = urllib.request.build_opener(urllib.request.HTTPHandler)
headers = [
('User-Agent', 'Mozilla/5.0 (Windows NT 6.3; WOW64; Trident/7.0; rv:11.0) like Gecko')
]
urllib.request.install_opener(opener)
opener.addheaders = headers
queue.put(url)
visited |= {url}
cnt = 0
for i in range(th_num):
t = CsdnBlogSpider(queue,opener,blog_name)
t.setDaemon(True)
t.start()
queue.join()
print('--------end!!!-----')
print('共抓取:' + str(cnt))
def data_link( self, threadName, delay, in_file, queue):
"Fetch data from a particular data link"
# Open and read file
ifile = open(in_file,'rb')
# Read data (get packets)
data = ifile.read(packet_size)
while data:
# Push bytes into Queue
queue.put(data)
# Sleep the thread
time.sleep(delay)
# Read data (get packets)
data = ifile.read(packet_size)
print("Fetching from %s" % threadName)
# Close file
ifile.close()
return
def ucs(source, target, graph):
""" Uniform-cost graph search """
queue = queue.PriorityQueue() # fringe
queue.put((0, source))
parent = {source:None}
visited = {}
while not queue.empty():
(d, v_in) = queue.get()
if v_in not in visited or d < visited[v_in]:
if v_in == target:
return (d, build_path(parent, target))
for v_out in graph.adj(v_in):
cost = graph.distance(v_in, v_out) + d
if v_out not in visited:
queue.put((cost, v_out))
parent[v_out] = v_in
visited[v_in] = cost
return None
def run_server(tmpdir, handler_class, stop_event, queue): # pragma: no cover
"""
Runs an HTTP server serving files from given tmpdir in a separate
process. When it's ready, it sends a URL to the server over a
queue so the main process (the HTTP client) can start making
requests of it.
"""
class HTTPRequestHandler(handler_class):
def translate_path(self, path):
path = handler_class.translate_path(self, path)
path = os.path.join(
tmpdir,
os.path.relpath(path, os.getcwd()))
return path
server = socketserver.TCPServer(("127.0.0.1", 0), HTTPRequestHandler)
domain, port = server.server_address
url = "http://{0}:{1}/".format(domain, port)
# Set a reasonable timeout so that invalid requests (which may occur during
# testing) do not cause the entire test suite to hang indefinitely
server.timeout = 0.1
queue.put(url)
# Using server.serve_forever does not work here since it ignores the
# timeout value set above. Having an explicit loop also allows us to kill
# the server from the parent thread.
while not stop_event.isSet():
server.handle_request()
server.server_close()
def use_cached_files(self, cache_key, results_dir=None):
if self._localcache.has(cache_key):
return self._localcache.use_cached_files(cache_key, results_dir)
queue = multiprocessing.Queue()
try:
response = self._request('GET', cache_key)
if response is not None:
threading.Thread(
target=_log_if_no_response,
args=(
60,
"\nStill downloading artifacts (either they're very large or the connection to the cache is slow)",
queue.get,
)
).start()
# Delegate storage and extraction to local cache
byte_iter = response.iter_content(self.READ_SIZE_BYTES)
res = self._localcache.store_and_use_artifact(cache_key, byte_iter, results_dir)
queue.put(None)
return res
except Exception as e:
logger.warn('\nError while reading from remote artifact cache: {0}\n'.format(e))
queue.put(None)
# TODO(peiyu): clean up partially downloaded local file if any
return UnreadableArtifact(cache_key, e)
return False
def producer(queue):
while True:
time.sleep(1)
print("Queue size: %d" % queue.qsize())
print("Put Widget")
widget = random.choice(('drum', 'stick', 'trombone', 'swordfish'))
queue.put(widget, block=False)
def multiCommand(commands):
maxAlterations = int(max([i[2] for i in commands]) * frameRate)
queueList = []
queueLock.acquire()
while not queue.empty():
queueList.append(queue.get())
queue.task_done()
appends = maxAlterations - len(queueList)
if appends > 0:
for i in range(abs(appends)):
queueList.append({})
for c in commands:
commandAlterations = int(c[2] * frameRate)
for i in range(c[0][0], c[0][1]):
start = pixels[i]
bridgeGenerator = bridgeValues(commandAlterations, start, c[1])
for m in range(commandAlterations):
queueList[m][i] = next(bridgeGenerator)
if appends < 0:
for r in range(abs(appends)):
if i in queueList[commandAlterations + r]:
del queueList[commandAlterations + r][i]
while queueList:
queue.put(queueList.pop(0))
queueLock.release()
def new_request(http_request):
form = forms.RequestForm()
items_formset = forms.ItemRequestFormSet()
if http_request.method == "POST":
form = forms.RequestForm(http_request.POST)
items_formset = forms.ItemRequestFormSet(http_request.POST)
if form.is_valid() and items_formset.is_valid():
logger.debug("creating the new request.")
request = form.save()
for item_data in items_formset.cleaned_data:
if item_data:
logger.debug("adding a new item.")
item_request = models.ItemRequest()
item_request.request = request
item_request.item = item_data['item']
item_request.quantity = item_data['quantity']
item_request.save()
logger.debug("new request created.")
logger.debug("Sending new request to the queue.")
queue.put(request)
form = forms.RequestForm()
items_formset = forms.ItemRequestFormSet()
return render(http_request, "new_request.html",
{'form': form, 'items_formset': items_formset})
def reader(pipe,queue): """Target for threading for scrolling BASH function below.""" try: with pipe: for line in iter(pipe.readline,b''): queue.put((pipe, line)) finally: queue.put(None)
def on_message(self, message):
#logging.info("got message %r", message)
parsed = tornado.escape.json_decode(message)
if 'command' in parsed:
command = parsed['command'].lower()
data = parsed['data'] if 'data' in parsed else None
req_id = parsed['req_id'] if 'req_id' in parsed else None
if hasattr(self, 'on_' + command):
getattr(self, 'on_' + command)(data, req_id)
elif hasattr(self, command + '_queue'):
queue = getattr(self, command + '_queue')
url = data.get('url')
logging.debug("putting response in queue %s", url)
if queue is not None:
if isinstance(queue, dict):
if url in queue:
queue[url].put(data)
else:
queue.put(data)
elif data and req_id:
if isinstance(data, dict):
args = data['args'] if 'args' in data else [data]
else:
args = [data]
logging.info("got callback[%s] (%r)" % (req_id, args,))
ServerSocketHandler.send('callback', args, req_id)
def support_test_send_reply_to(self, address, queue):
message = uuid.uuid4().hex
with self.context.socket(roles['speaker']) as socket:
socket.connect("tcp://%s" % address)
socket.send(nw0.sockets._serialise(message))
reply = nw0.sockets._unserialise(socket.recv())
queue.put(reply)
def command(queue, stop):
commands = ['error',
'move',
'set safety',
'fire',
'move reticle',
'store location',
'get reticle',
'get altitude',
'get azimuth',
'get safety',
'get locations',
'0x0',
'0x10',
'0x11',
'0x12',
'0x13',
'0x14']
count = 0
while True:
if stop.is_set():
break
if count % 10 == 11:
command = random.choice(commands)
queue.put([command, 0])
count += 1
time.sleep(1)
def absoluteFade(indexes, rgb, fadeTime):
'''Is given a color to fade to, and executes fade'''
if not fadeTime:
fadeTime = 1 / frameRate
for c in rgb:
c = makeEightBit(c)
#Calculates how many individual fade frames are needed
alterations = int(fadeTime * frameRate)
queueList = []
queueLock.acquire()
while not queue.empty():
queueList.append(queue.get())
queue.task_done()
#Amount of frames that need to be added to queue
appends = alterations - len(queueList)
#fill out the queue with blank dictionaries to populate
if appends > 0:
for i in range(abs(appends)):
queueList.append({})
#Iterate down indexes, figure out what items in queue need to be altered
for i in indexes:
#INVESTIGATE: THIS MIGHT BE THE SOURCE OF FLASHING ISSUES AT THE START OF A COMMAND
start = pixels[i]
bridgeGenerator = bridgeValues(alterations, start, rgb)
for m in range(alterations):
queueList[m][i] = next(bridgeGenerator)
#If this command overrides a previous command to the pixel, it should wipe any commands remaining
if appends < 0:
for r in range(abs(appends)):
if i in queueList[alterations + r]:
del queueList[alterations + r][i]
while queueList:
queue.put(queueList.pop(0))
queueLock.release()
def parse_potential_recent_deaths(self, refresh):
from queue import Queue, Empty
queue = Queue()
for name, info in self.chars.items():
if (info.is_online() or time.time() - info.last_online() < 1200) and info.vocation != 'N':
queue.put(name)
task_count = queue.qsize()
def get_info():
while True:
try:
name = queue.get(block=False)
tasks_left = queue.qsize()
except Empty:
return
info = tibiacom.char_info(name)
self.chars[name].deaths = info["deaths"]
refresh()
queue.task_done()
print("pzlock update: %d/%d" % ((task_count - tasks_left), task_count))
threads = []
for i in range(10):
thrd = threading.Thread(target=get_info)
thrd.start()
threads.append(thrd)
queue.join()
for t in threads:
t.join()
def running(queue):
for x in range(5):
text = 'message ' + str(x)
print('PUT:', text)
queue.put(text)
time.sleep(4)
queue.put('last')
def worker(url, queue): # get queue as argument
r = requests.get(url)
soup = BeautifulSoup(r.text, "html.parser")
data = soup.find("span", {"class": "text"}).get_text()
# send result to main thread using queue
queue.put(data)
def prepare_fitness(self):
"""
Here we request fitness calculation. Prepare place for result
and put our string into a queue. A running worker-thread will
pick it up from the queue and calculate.
"""
self.result_dict = {}
queue.put((str(self), self.result_dict))
def runlist_parser(queue,parser,url):
url2 = urlHandle(url)
try:
result = parser.Parse(url2)
if (result is not None) and (result != []) and (result["data"] is not None) and (result["data"] != []):
queue.put({"result":result,"url":url})
except Exception as e:
#continue
print(e)
def download_t(uid, cat, start, queue, t_name):
global EXIT
url = "http://api.douban.com/people/{0}/collection?cat={1}&tag=&status=&start-index={2}&max-results=50&alt=atom&apikey={3}".format(uid, cat, start,APIKEY)
filename = '{0}_{1}_{2}'.format(uid, cat, start)
while True:
try:
fh = open("api_limit", "a+")
fcntl.flock(fh.fileno(), LOCK_EX|LOCK_NB)
break
except IOError:
fh.close()
if(fh.tell() != 0):
os.lseek(fh.fileno(), -20, os.SEEK_END)
line = fh.read(19)
seconds = float(line.split()[1])
# just be careful!
while float(time.time()) - seconds <= 1.8:
pass
fh.write("[LOG] {0:<24} {1} {2}\n".format(time.ctime(), url[:-40], time.time().__repr__()[0:-4]))
fcntl.flock(fh, LOCK_UN)
fh.close()
#print('start dl:{0} {1}'.format(t_name, time.ctime()))
try:
req = urllib.request.Request(url, headers= {'Accept-encoding':'gzip'})
rec = urllib.request.urlopen(req)
compressed_data = rec.read()
compressed_fh = io.BytesIO(compressed_data)
gzipper = gzip.GzipFile(fileobj = compressed_fh);
data = gzipper.read().decode('utf8')
#data = urllib.request.urlopen(url).read().decode('utf8')
except (urllib.error.URLError, ValueError) as e:
if hasattr(e, 'reason'):
print("<h4>Cannot connected to the server</h4>")
print("<h4>url:</h4>",url)
EXIT = True
return
if hasattr(e, 'code'):
print("<h4>Return code:",e.code,"error</h4>")
print("<h4>",e.msg,"</h4>")
print("<h4>url:</h4>",url)
EXIT = True
return
fw = open(ROOTDIR + r'htdocs/cache_datas/' + filename, "w+", encoding = 'utf8')
fw.write(data)
fw.close()
#print('creat file:' ,filename, time.ctime())
if start == 1:
fr = open(ROOTDIR + r'htdocs/cache_datas/' + filename, 'r', encoding = 'utf8')
for line in fr.readlines():
if 'totalResults' in line:
max_item = int(line.split('>')[1].split('<')[0])
dl_nr = max_item//50
break
if max_item > 50:
for i in range(dl_nr):
start += 50
queue.put([uid, cat, start, queue])
def test():
for i in range(500):
queue.put('init pro ' + str(i))
for i in range(2):
p = Producer()
p.start()
for i in range(5):
c = Consumer()
c.start()
def support_test_send_news_to(self, address, topic, queue):
with self.context.socket(roles['subscriber']) as socket:
socket.connect("tcp://%s" % address)
socket.subscribe = topic.encode("utf-8")
while True:
topic, data = nw0.sockets._unserialise_for_pubsub(socket.recv_multipart())
queue.put((topic, data))
if data is not None:
break
def run(queue,get_list_info,html_text):
try:
result = get_list_info(html_text)
if result != []:
queue.put(result)
except Exception as e:
#import traceback
#traceback.print_exc()
print(e)
def main(): for tr in trains: queue.put(tr) for i in range(DAEMONS): t = Thread(queue, print_lock) t.setDaemon(True) t.start() queue.join()
def crawl(queue):
global log
log = open('crawl.log.txt', 'w')
for category in catalog.values():
for link in process(category):
queue.put(link)
log.close()
def getRows(self, table_name, queue):
rows = 0
query = "SELECT count(1) as rows_table FROM {};".format(table_name)
cur = self.conn.cursor()
cur.execute(query)
for rows_table in cur:
rows = rows_table[0]
cur.close()
queue.put(int(rows))
return int(rows)
def time_connection(server, queue):
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
s.connect(server)
print ("Connected to " + str(server))
if queue.empty:
queue.put(server)
s.close()
except:
print(str(server) + " does not respond!")
def userinput(self, queue, lock):
i = 0
uinput = ""
while self.running and uinput != "terminate":
try:
blocking = True
uinput = input(">>>")
queue.put(str(uinput))
blocking = False
except EOFError:
i += 1
def echoreverse(ip, queue, data):
# Reverse the data.
# result = reversed(data)
# Convert each byte to a character.
# result = [chr(b) for b in reversed(data)]
# Turn the list of characters into a string with "" in between each char.
# result = "".join([chr(b) for b in reversed(data)])
# Turn the whole thing back into bytes with UTF-8 encoding.
# result = bytes("".join([chr(b) for b in reversed(data)]), "UTF-8")
# Put the result back into the queue of data to be sent back:
queue.put(bytes("".join([chr(b) for b in reversed(data)]), "UTF-8"))
def _add_depandent(self,queue):
for value in self.file_set:
_list=[]
for seq in getattr(self,'seq_{}'.format(value)):
for data_dict in self.data:
if data_dict['id']==seq and data_dict['product']==value:
# if data_dict['case_id']==seq and data_dict['file']==value:
_list.append(data_dict)
self.data.remove(data_dict)
break
queue.put(_list)
def _task(self):
queue = self.queue
# queue can be overflow
with self.transaction_lock:
while not queue.empty():
queue.get_nowait()
queue.task_done()
queue.put(0)
# INTERNAL: start counter for task
while True:
wait = self.default_execute_wait
event = queue.get()
if event is None:
# STOP EVENT
while not queue.empty():
queue.get_nowait()
queue.task_done()
# TODO: warning not queued event?
# TODO: just new stop flag?
queue.task_done()
# for stop event.
return
time.sleep(wait)
# TODO: how to sleep automation?
# TODO: use some good schuler?
with self.transaction_lock:
current_transaction = self.current_transaction
if current_transaction is None:
with self.transaction() as current_transaction:
pass
if not current_transaction.operations:
queue.put(event + wait)
queue.task_done()
continue
if event >= self.status.default_execute_wait:
self.current_transaction = None
self.execute_transaction(current_transaction)
queue.put(0)
queue.task_done()
def minDepth(self, root: TreeNode) -> int:
if root == None:
return 0
import queue
queue = queue.Queue()
queue.put((root, 1))
while queue:
node, depth = queue.get()
if node.left == None and node.right == None:
return depth
else:
if node.left != None:
queue.put((node.left, depth + 1))
if node.right != None:
queue.put((node.right, depth + 1))
def program():
# Threads without locking
print("Threads without locking:")
thread_1 = ExampleThread(1, "Thread 1", 1)
thread_2 = ExampleThread(2, "Thread 2", 2)
thread_1.start()
thread_2.start()
# Do something and then block until the threads are finished
time.sleep(1)
thread_1.join()
thread_2.join()
# Threads with locking
print("\nThreads with locking:")
thread_3 = ExampleThreadLock(1, "Thread 3", 1)
thread_4 = ExampleThreadLock(2, "Thread 4", 2)
thread_3.start()
thread_4.start()
# Do something and then block until the threads are finished
time.sleep(5)
thread_3.join()
thread_4.join()
# Threads with queueing
print("Threads with queueing:")
thread_5 = ExampleThreadQueue(1, "Thread 5", queue)
thread_6 = ExampleThreadQueue(2, "Thread 6", queue)
thread_5.start()
thread_6.start()
# Add items to the queue
source = [
"Pigeondust",
"Ill Sugi",
"J'adore Banania",
]
for item in source:
queue.put(item)
# Add items that allow the threads to terminate when the work is done
queue.put(None)
queue.put(None)
# Block until all jobs in the queue are finished
queue.join()
def single_check_expried(Device, trial: int):
r = ""
while True:
q = queue.Queue()
t = Thread(target=lambda queue, Device, trial: queue.put(Device.status[
"provider"].check_expired(Device, trial)),
args=(
q,
Device,
trial,
))
Device.status["cThread"].append(t)
while not (check_ready(t.getName(), Device.status["cThread"])):
pass
t.start()
t.join()
Device.status["cThread"].pop(0)
r = q.get()
if r != "" and r != None:
break
return ({Device.data["phone_number"]: r})
def _executor(self, queue, host, args, kwargs, exception_handler):
result = None
exception = None
exception_occurred = False
try:
result = host(*args, **kwargs)
except Exception as e:
exception_occurred = True
exception = e
if not exception_occurred:
queue.put(result)
elif exception_occurred and exception_handler is not None:
queue.put(None)
queue.put(exception)
else:
raise exception
def process_main(self, queue):
# Scramble seed so it's not a copy of the parent's seed
np.random.seed()
# np.random.seed(1)
try:
while True:
queue.put(self.data_fn())
except StopIteration as e:
# Should only manually throw when want to close queue
queue.put(e)
#raise e
return
except Exception as e:
queue.put(StopIteration())
#raise e
return
def verticalOrder(self, root):
"""
:type root: TreeNode
:rtype: List[List[int]]
"""
results = collections.defaultdict(list)
import queue
queue = queue.Queue()
queue.put((root, 0))
while not queue.empty():
node, x = queue.get()
if node:
results[x].append(node.val)
queue.put((node.left, x - 1))
queue.put((node.right, x + 1))
return [results[i] for i in sorted(results)]
def keys_exist_or_assert(keys, server, bucket_name, test, queue=None, collection=None):
# we should try out at least three times
log = logger.Logger.get_logger()
# verify all the keys
client = MemcachedClientHelper.proxy_client(server, bucket_name)
# populate key
retry = 1
keys_left_to_verify = []
keys_left_to_verify.extend(copy.deepcopy(keys))
log_count = 0
while retry < 6 and len(keys_left_to_verify) > 0:
msg = "trying to verify {0} keys - attempt #{1} : {2} keys left to verify"
log.info(msg.format(len(keys), retry, len(keys_left_to_verify)))
keys_not_verified = []
for key in keys_left_to_verify:
try:
client.get(key=key, collection=collection)
except mc_bin_client.MemcachedError as error:
keys_not_verified.append(key)
if log_count < 100:
log.error("key {0} does not exist because {1}".format(key, error))
log_count += 1
retry += 1
keys_left_to_verify = keys_not_verified
if len(keys_left_to_verify) > 0:
log_count = 0
for key in keys_left_to_verify:
log.error("key {0} not found".format(key))
log_count += 1
if log_count > 100:
break
msg = "unable to verify {0} keys".format(len(keys_left_to_verify))
log.error(msg)
if test:
queue.put(False)
test.fail(msg=msg)
if queue is None:
return False
else:
queue.put(False)
log.info("verified that {0} keys exist".format(len(keys)))
if queue is None:
return True
else:
queue.put(True)
def _runner(self, queue, target, target_args, target_kwargs,
exception_handler):
result = None
exception = None
exception_occurred = False
try:
result = target(*target_args, **target_kwargs)
except Exception as e:
exception_occurred = True
exception = e
if not exception_occurred:
queue.put(result)
elif exception_occurred and exception_handler is not None:
queue.put(None)
queue.put(exception)
else:
raise exception
def check_balances(list_Device):
db.query(sim_model.Sim).update({sim_model.Sim.check: "checking"})
db.commit()
data = []
q = queue.Queue()
thread_list = []
for Device in list_Device:
t = Thread(
target=lambda queue, Device: queue.put(sim.check_balance(Device)),
args=(
q,
Device,
))
thread_list.append(t)
t.start()
for t in thread_list:
t.join()
for t in thread_list:
r = q.get()
print("-------------")
print(r)
data.append(r)
return {"status": "done"}
def takecommand():
r = sr.Recognizer()
with sr.Microphone() as source:
queue.put("Kurama:- Listening...\n\n")
r.pause_threshold = 1
audio = r.listen(source)
try:
queue.put("Kurama:- Recognizing...\n\n")
query = r.recognize_google(audio, language='en-in')
print(f"You-->{query}\n")
except Exception as e:
queue.put("Kurama:- Sorry couldn\'t recognise that !\n\n")
speak("Sorry couldn\'t recognise that !")
query = "\quit"
return query
def mergeSort(data):
global queue
last = time.perf_counter()
n = len(data)
step = 1
while (step < n):
i = 0
while (i < n - step):
a = data[i:i + step]
b = data[i + step:min(i + 2 * step, n)]
for k in range(i, min(i + 2 * step, n)):
if (len(a) > 0 and len(b) > 0):
if (a[0] > b[0]):
data[k] = b[0]
b.remove(b[0])
#sound()
else:
data[k] = a[0]
a.remove(a[0])
else:
for p in range(k, min(i + 2 * step, n)):
if (len(b) > 0):
data[p] = b[0]
b.remove(b[0])
for j in range(k, min(i + 2 * step, n)):
if (len(a) > 0):
data[j] = a[0]
a.remove(a[0])
if not sounds:
queue.put(lambda: DrawList(data, i, k), sound(data[k]))
else:
queue.put(lambda: DrawList(data, i, k))
time.sleep(ANIMATION_DELAY / 1000)
if not sorting:
print("Ending sort...")
return
i = i + 2 * step
step = step * 2
print("Sorted with Merge Sort in ", time.perf_counter() - last, "seconds.")
with queue.mutex:
queue.queue.clear()
queue.put(lambda: EndSort())
return
def searchRange(self, root, k1, k2):
# write your code here
import queue
if root is None:
return []
queue = queue.Queue()
queue.put(root)
result = []
while not queue.empty():
node = queue.get()
if node.val <= k2 and node.val >= k1:
result.append(node.val)
if node.left:
queue.put(node.left)
if node.right:
queue.put(node.right)
return result
def scanner(self, queue, parent, npes, track, sensor, dtype, dt_tol,
dL_tol):
timeout = 900
self.logger.debug("Scanner, pid: {}".format(os.getpid()))
filenames = bloom_filter(track, sensor, dtype, dt_tol, dL_tol)
self.logger.debug("Finished bloom filter")
with ProcessPoolExecutor(max_workers=npes,
timeout=timeout) as executor:
results = []
for f in filenames:
self.logger.info("Scanning: {}".format(f))
if (len(results) >= npes) and parent.is_alive():
idx = [r.done() for r in results]
while not np.any(idx):
time.sleep(1)
idx = [r.done() for r in results]
tmp = results.pop(idx.index(True)).result()
self.logger.debug("Finished reading another file")
if not tmp.empty:
self.logger.warning("Found {} matchs".format(len(tmp)))
queue.put(tmp)
self.logger.debug("Getting {}".format(f))
ds = self.db[f].compute()
if not parent.is_alive():
return
self.logger.debug("Submitting a new inrange process")
results.append(
executor.submit(matchup, track, ds, dL_tol, dt_tol))
for tmp in (r.result(timeout) for r in results):
if not parent.is_alive():
return
self.logger.debug("Finished reading another file")
if not tmp.empty:
self.logger.warning("Found {} matchs".format(len(tmp)))
queue.put(tmp)
self.logger.debug("Finished scanning all potential matchups.")
queue.put("END")
def expand_url(tweet_id, tweet, url, queue):
expanded_url = url['expanded_url']
try:
if (is_shortened(expanded_url)):
# queue.put([tweet_id, tweet, url['expanded_url']])
data = open_url(expanded_url)
print('--shortened--')
print(expanded_url)
queue.put([tweet_id, tweet, data])
else:
print('--no need--')
print(expanded_url)
queue.put([tweet_id, tweet, expanded_url])
except timeout:
print('socket timed out - URL %s', url)
except requests.ConnectionError:
print('try again')
data = open_url(expanded_url)
queue.put([tweet_id, tweet, data])
except Exception as e:
print(url)
print(e)
def find_astar_route(source, target, compute_dist, h):
parentDict = dict()
start = help.find_junction(my_road, source)
goal = help.find_junction(my_road, target)
queue = help.My_Priority_Queue()
f_score = dict()
g_score = dict()
for junc in my_road.junctions():
g_score[junc.index] = sys.maxsize
f_score[junc.index] = sys.maxsize
f_score[start.index] = h(start.lat, start.lon, goal.lat, goal.lon)
g_score[start.index] = 0
score_in_queue = dict()
queue.put((f_score[start.index], start))
score_in_queue[start.index] = h(start.lat, start.lon, goal.lat, goal.lon)
while queue:
score_f, node = queue.get()
if node.index == target:
path = help.find_path(source, target, parentDict)
return path
neighbors = help.compute_neighbors(node)
for neighbor in neighbors:
i = help.find_junction(my_road, neighbor)
distance = compute_dist(my_road, node, i)
new_cost = g_score[node.index] + distance
if new_cost < g_score[i.index]:
parentDict[i.index] = node
g_score[i.index] = new_cost
f_score[i.index] = g_score[i.index] + h(i.lat, i.lon, goal.lat, goal.lon)
if not queue.__contains__(i):
queue.put((f_score[i.index], i))
score_in_queue[i.index] = f_score[i.index]
else:
if score_in_queue[i.index] > f_score[i.index]:
queue.__remove__(i)
parentDict[i.index] = node
queue.put((f_score[i.index], i))
score_in_queue[i.index] = f_score[i.index]
return "NO PATH"
def bubbleSort(data):
global queue
last = time.perf_counter()
for e in range(len(data) - 1, 0, -1):
for i in range(1, len(data)):
if data[i - 1] > data[i]:
data[i], data[i - 1] = data[i - 1], data[i]
if not sounds:
queue.put(lambda: DrawList(data, e, i), sound(data[i - 1]))
else:
queue.put(lambda: DrawList(data, e, i))
time.sleep(ANIMATION_DELAY / 1000)
if not sorting:
print("Ending sort...")
return
print("Sorted with Bubble Sort in", time.perf_counter() - last, "seconds.")
with queue.mutex:
queue.queue.clear()
queue.put(lambda: EndSort())
return
def partition(arr, l, h):
global queue
i = (l - 1)
x = arr[h]
for j in range(l, h):
if arr[j] <= x:
# increment index of smaller element
i = i + 1
arr[i], arr[j] = arr[j], arr[i]
if not sounds:
queue.put(lambda: DrawList(arr, i, j), sound(arr[i]))
else:
queue.put(lambda: DrawList(arr, i, j))
arr[i + 1], arr[h] = arr[h], arr[i + 1]
queue.put(lambda: DrawList(arr, i, j))
time.sleep(ANIMATION_DELAY / 1000)
return (i + 1)
def parallel_run(max_shots, report_rate, runs, service):
threads = []
# if service == "EC2":
# start_EC2()
diff = 0
partition = int(max_shots / runs)
ten_percent = partition * 0.1
if (report_rate < ten_percent):
diff = int(ten_percent)
print('diff is 10 per cent :', diff)
else:
diff = report_rate
print('diff is report rate :', diff)
starting_point = 1
shots = partition
for i in range(0, runs):
if i == 0:
shots = partition + diff
elif i == runs - 1:
shots = max_shots
else:
shots = shots + partition
# if i == runs-1:
# shots = partition - diff
print("shots:", RoundFraction(shots, report_rate), " report rate: ",
report_rate, " starting point: ",
RoundFraction(starting_point, report_rate) + 1, " count",
shots - starting_point)
t = ThreadUrl(queue, i, RoundFraction(shots, report_rate), report_rate,
RoundFraction(starting_point, report_rate) + 1, service)
threads.append(t)
t.setDaemon(True)
t.start()
starting_point = shots + 1
for x in range(0, runs):
queue.put(count)
queue.join()
ids = [t.task_id for t in threads]
results = [t.data for t in threads]
long_result = []
task_id = []
for i, result in enumerate(results):
str_result_data = JSON.loads(result)
if service == "Lambda":
result_data = JSON.loads(str_result_data)
else:
result_data = str_result_data
long_result = long_result + result_data["random_list"]
for num in range(0, len(result_data["random_list"])):
task_id.append(ids[i] + 1)
# print("task id: ", ids[i],result_data["random_list"].__len__(), result_data["random_list"])
estimated_values = []
Total_shots = []
In_Circle = []
for value in long_result:
Total_shots.append(value[1])
In_Circle.append(value[0])
estimated_value = (value[0] / value[1]) * 4
estimated_values.append(estimated_value)
data = {
'task_id': task_id,
'In_Circle': In_Circle,
'Total_shots': Total_shots,
'estimated_values': estimated_values
}
df = pd.DataFrame(
data,
columns=['task_id', 'In_Circle', 'Total_shots', 'estimated_values'])
if service == "EC2":
stop_EC2()
return df
def myPublisher(queue):
while not queue.full():
queue.put(1)
print("{} Appended 1 to queue:{}".format(threading.current_thread(),
queue.qsize()))
time.sleep(1)
def _th_create_package(self,item,queue,semaphore): p=Package.Package(item) queue.put(p)
# year = input("year:")
# IP = input("IP start:")
year = input("爬取的年份：")
number_of_thread = 15
# 创建包括number_of_thread个线程的线程池
for i in range(number_of_thread):
Thread = threading.Thread
t = Thread(target=do_job)
t.start()
# 模拟创建线程池0.1秒后塞进任务到队列
time.sleep(0.1)
cnt = 0
for i in lines:
# 检查文件命名，防止网站资源有特殊字符本地无法保存
file_pattern_compile = re.compile(r'[\\/:\*\?"<>\|]')
journal_name = re.sub(file_pattern_compile, '', i[:-1])
if not os.path.exists(journal_name):
os.makedirs(journal_name)
task = MyTask(0, journal_name, cnt)
queue.put(task)
cnt += 1
if cnt == number_of_thread * 2:
# break
time.sleep(600)
cnt = 0
time.sleep(10)
queue.join()
# uses Vivek's skeleton code and replaces the "run" fcn w/ ftp parts
# ftp code: https://docs.python.org/2/library/ftplib.html
class WorkerThread(threading.Thread):
def __init__(self, queue):
threading.Thread.__init__(self)
self.queue = queue
def run(self):
while True:
site = self.queue.get()
print("Attempting login for: {}".format(site))
ftp = FTP(site) # connect to host, default port
ftp.login() # user anonymous, passwd anonymous@
ftp.retrlines('LIST') # list directory contents
self.queue.task_done()
print("\n\n")
queue = queue.Queue()
for i in range(5):
worker = WorkerThread(queue)
worker.setDaemon(True)
worker.start()
for site in ftp_sites:
queue.put(site)
queue.join()
print("\nDone.")
for url in urls:
try:
self.download('http:' + url)
except Exception as error:
print(error)
self.queue.task_done()
if self.queue.empty():
break
def download(self, url):
img = self.fetch(url)
name = url.split(r'/')[-1]
print(name)
with open(name, 'wb') as f:
f.write(img)
if __name__ == '__main__':
threads = []
queue = queue.Queue()
num = threading.Semaphore(4)
URL = 'http://jandan.net/ooxx/page-{}#comments'
meizhi = Meizhi(queue, num)
for num in range(2407, 2410):
queue.put(URL.format(num))
meizhi.start()
queue.join()
print(time.time() - start)
# 36.42306613922119
def thread_worker(queue, gen):
for val in gen:
queue.put(val)
queue.put(None)
def create_jobs():
for x in job_to_do:
queue.put(x)
#add the value in list to the queue
queue.join()
def producer(queue, event):
while not event.is_set():
message = random.randint(1, 99)
print("Producer received data: ", message)
queue.put(message)
s = m.get_server()
s.serve_forever()
# 发送数据
from multiprocessing.managers import BaseManager
class QueueManager(BaseManager):
pass
QueueManager.register('get_queue')
m = QueueManager(address=('127.0.0.1', 50000), authkey='abc'.encode("utf8"))
m.connect()
queue = m.get_queue()
queue.put('hello')
# 接收数据
from multiprocessing.managers import BaseManager
class QueueManager(BaseManager):
pass
QueueManager.register('get_queue')
m = QueueManager(address=('127.0.0.1', 50000), authkey='abc'.encode("utf8"))
m.connect()
queue = m.get_queue()
print(queue.get())
print(m.address)
# -*- coding: UTF-8 -*-
# filename: pcm2wav date: 2018/11/22 12:54
# author: FD
import os
import queue
import wave
if __name__ == '__main__':
queue = queue.Queue()
rootDir = '.'
queue.put(rootDir)
while not queue.empty():
node = queue.get()
for filename in os.listdir(node):
nextpath = os.path.join(node, filename)
if os.path.isdir(nextpath):
queue.put(nextpath)
elif nextpath.endswith('.pcm'):
filenamenosuffix = nextpath[0:nextpath.index('.pcm')]
wavfilepath = filenamenosuffix + ".wav"
with open(nextpath, 'rb') as pcmfile:
pcmdata = pcmfile.read()
with wave.open(wavfilepath, 'wb') as wavfile:
wavfile.setparams((1, 2, 48000, 0, 'NONE', 'NONE'))
wavfile.writeframes(pcmdata)
def getAnswer(self, questionId):
# 每次取10条回答
limit = 10
# 获取答案时的偏移量
offset = 0
# 开始时假设当前有这么多的回答
total = 2 * limit
# 我们当前已记录的回答数量
record_num = 0
# 标题
title = questionId
# 获取本地路径
dir = os.path.abspath(local_path)
# 获取图片的正则
image_pattern = re.compile(r'<img.*?actualsrc="(https.*?.jpg)".*?/>',
re.S)
print(f"开始下载问题： {title}...")
while record_num < total:
# 请求
url = 'https://www.zhihu.com/api/v4/questions/' + questionId + '/answers?sort_by=default&include=data[*].is_normal,voteup_count,content&limit=' + str(
limit) + '&offset=' + str(offset)
response = self.session.get(url, headers=self.headers, timeout=10)
response = json.loads(response.content)
# 获取总回答数
try:
total = response['paging']['totals']
except KeyError as e:
continue
now = time.strftime('%Y-%m-%d %H:%M:%S')
print(
f'{now} --- {title}的总回答数：{total}， 当前读取偏移量：{offset}， 剩余：{total - offset}， 当前队列空余容量：{100000 - queue.qsize()}'
)
# 获取实体信息
datas = response['data']
if datas is not None:
if total <= 0:
break
for data in datas:
# 标题 替换掉非法字符
title = data['question']['title'].replace("?", "？")
# 作者名称
author = data['author']['name']
answer = data['content']
print(f'读取用户：{author}')
# 组装下载路径
img_path = os.path.join(dir, title)
img_path = os.path.join(img_path, author)
# 判断文件夹是否存在
if not os.path.exists(img_path):
os.makedirs(img_path)
# 获取图片链接
results = re.findall(image_pattern, answer)
for img_url in results:
# 获取文件名
filename = os.path.basename(img_url)
file_path = os.path.join(img_path, filename)
if os.path.exists(file_path):
print(
f'{current_thread().getName()} - {author} - {filename}已存在'
)
else:
# 生产数据
queue.put({
'img_url': img_url,
'file_path': file_path,
'title': title,
'author': author
})
# 请求的向后偏移量
offset += len(datas)
record_num += len(datas)
# 随机沉睡1-3s
time.sleep(random.choice([1, 2, 3]))
# 如果获取的数组size小于limit,循环结束
if len(datas) < limit:
break
print(f"问题：{title} 下载完毕...")
dones.append(title)
