def enque(self, task):
queue = self.queue
if self.timeout == 0:
self.execute_task(task)
else:
queue.put_nowait(task)
self.execute_next()
async def enqueue_websocket_messages(endpoint,
queue,
ready,
close,
extra_headers=None,
verify=True):
websocket = await connect(endpoint, extra_headers, verify)
pending = {websocket.recv()}
# signal to the controlling process that we're ready
ready.set()
while True:
done, pending = await asyncio.wait(pending, timeout=0.1)
if done:
for msg in done:
queue.put_nowait(msg.result())
pending.add(websocket.recv())
if close.is_set():
break
for msg in pending:
msg.cancel()
await websocket.close()
def runInstaBot(params):
queue = params['queue']
try:
bot = InstaBot(**params)
bot.new_auto_mod()
except:
queue.put_nowait("Exception")
def _dream_process(self, queue, assoc=True, dump=True):
dream_re = re.compile(
r'^(\d+),(\d+),([0-9A-Fa-f]{12}),([0-9A-Fa-f]{12}),$')
cmd = [ tools['dream'] ] \
+ ([ '--a' ] if assoc else []) \
+ ([ '--d', f'{self.phy}-{self.static_token}.cap' ] \
if dump else []) \
+ [ '-tcbs' ] \
+ [ self.mon ]
dream = Popen(cmd,
bufsize=-1,
stdin=DEVNULL,
stdout=PIPE,
stderr=DEVNULL,
cwd=basepath,
text=True)
assert dream
try:
for line in iter(dream.stdout.readline, ''):
line = line.rstrip()
match = dream_re.match(line)
if match:
queue.put_nowait(match.groups())
except KeyboardInterrupt:
pass
def oncePerMinute():
global next_interval_time
while True:
try:
# Sleep for the remainder of the time until the next
# interval, prevents timer drift. The calculated time
# to sleep could be negative if our clock got updated
# by ntptime so just sleep one minute in that case.
next_interval_time += 60
sleep_time = next_interval_time - time.time()
if sleep_time < 0:
sleep_time = 60
time.sleep(sleep_time)
device_time = str(datetime.datetime.now())
current_cpm = cpm
pressure = getPressure()
sea_level_pressure = pressure / pow(1.0 - altitude_in_meters / 44330.0, 5.255)
body = urllib.parse.urlencode({'cpm': current_cpm, 'device_time': device_time, 'pressure': '{0:0.2f}'.format(pressure), 'sea_level_pressure': '{0:0.2f}'.format(sea_level_pressure)})
queue.put_nowait(body)
except:
print("Unexpected onePerMinute error: {0}".format(sys.exc_info()[0]))
else:
print("Queued sample")
def _threads_work(it, queue):
while True:
try:
item = next(it)
except Exception as e:
queue.put_nowait((None, e))
break
queue.put_nowait((item, None))
def cryptobook_add(namespace):
queue = dpget(namespace, "/cryptobook/queue")
key, masher = return_key(namespace)
try:
queue.put_nowait((str(uuid.uuid4()), key, masher))
except queue.Full:
pass
return namespace
def reader(container):
log = logging.getLogger('reader')
stream = container.logs(stream=True)
while True:
try:
queue.put_nowait(next(stream))
except StopIteration as e:
log.debug("No more logs")
time.sleep(0.001)
def put(self, msg, boxname = "inbox"):
"""Places an object on a queue which will be directed to a named inbox on the wrapped component."""
# print ("self.put msg", repr(msg), "boxname", boxname)
if self.alive:
queue = self.inQueues[boxname]
queue.put_nowait(msg)
self.inputComponent.whatInbox.put_nowait(boxname)
else:
raise AttributeError("shutdown was previously called, or we were never activated.")
def multiCIF_thread_wrapper(*args, **kwargs):
""" Wrapper to put results of multiCIF into a queue to be retrieved """
queue = args[0]
root_win = args[1]
try:
results = multiCIF.main(*args[2:], **kwargs)
queue.put(results)
except ValueError as e:
root_win.after(5, tkinter.messagebox.showerror, "Error", " ".join(e.args))
queue.put_nowait(None)
def basic_incognito_algorithm(self, priority_queue):
self.init_C1_and_E1()
queue = priority_queue
marked_nodes = set()
for i in range(1, len(self.Q) + 1):
i_str = str(i)
self.cursor.execute("SELECT * FROM C" + i_str + "")
Si = set(self.cursor)
# no edge directed to a node => root
self.cursor.execute("SELECT C" + i_str + ".* FROM C" + i_str +
", E" + i_str + " WHERE C" + i_str +
".ID = E" + i_str + ".start EXCEPT SELECT C" +
i_str + ".* FROM C" + i_str + ", E" + i_str +
" WHERE C" + i_str + ".ID = E" + i_str +
".end ")
roots = set(self.cursor)
roots_in_queue = set()
for node in roots:
height = self.get_height_of_node(node)
# -height because priority queue shows the lowest first. Syntax: (priority number, data)
roots_in_queue.add((-height, node))
for upgraded_node in roots_in_queue:
queue.put_nowait(upgraded_node)
while not queue.empty():
upgraded_node = queue.get_nowait()
# [1] => pick 'node' in (-height, node);
node = upgraded_node[1]
if node[0] not in marked_nodes:
if node in roots:
frequency_set = self.frequency_set_of_T_wrt_attributes_of_node_using_T(
node)
else:
frequency_set = self.frequency_set_of_T_wrt_attributes_of_node_using_parent_s_frequency_set(
node, i)
if self.table_is_k_anonymous_wrt_attributes_of_node(
frequency_set):
self.mark_all_direct_generalizations_of_node(
marked_nodes, node, i)
else:
Si.remove(node)
self.insert_direct_generalization_of_node_in_queue(
node, queue, i, Si)
self.cursor.execute("DELETE FROM C" + str(i) +
" WHERE ID = " + str(node[0]))
self.graph_generation(Si, i)
marked_nodes = set()
def get_many(mp_queue, queue, n):
"""
Safely gets n items from the queue
and inserts them into another one
(if there are less items takes all of them)
(if there are no items returns False)
"""
if mp_queue.empty():
return False
for i in range(n):
if mp_queue.empty():
break
item = mp_queue.get()
queue.put_nowait(item)
return True
async def _main(self):
async with aiohttp.ClientSession() as session:
queue = asyncio.Queue()
while True:
[queue.put_nowait(url) for url in self.urls]
tasks = []
for i in range(self.concurrency):
task = asyncio.create_task(
self._check_health(queue, session))
tasks.append(task)
# Wait until the queue is fully processed.
started_at = time.monotonic()
await queue.join()
total_time = time.monotonic() - started_at
# Cancel our worker tasks.
for task in tasks:
task.cancel()
# Wait until all worker tasks are cancelled.
await asyncio.gather(*tasks, return_exceptions=True)
print('====')
print(f'total time: {total_time:.2f} seconds')
await asyncio.sleep(self.interval)
def sendPendingOutput(self):
"""This method will take any outgoing data sent to us from a child component and stick it on a queue
to the outside world."""
if python_lang_type == 2:
items = self.childOutboxMapping.iteritems()
else:
items = self.childOutboxMapping.items()
for childSource, parentSink in items:
queue = self.outQueues[childSource]
while self.dataReady(parentSink):
if not queue.full():
msg = self.recv(parentSink)
# TODO - what happens when the wrapped component terminates itself? We keep on going. Not optimal.
queue.put_nowait(msg)
else:
break
def connect(port, queue, event):
datas = {}
count = 0
data_count = 0
serial_portname = port
serial_port = serial.Serial(serial_portname, baudrate=115200, timeout=5.0)
while event.is_set():
input = serial_port.readline().decode(encoding='ascii',
errors='ignore').rstrip()
if len(input) == 0:
print("Serial device timed out, no data received.")
else:
if is_json(input):
json_obj = ujson.loads(input)
data = {}
time = 0
for item in json_obj:
ID = item['node']
time = item['time']
datas.setdefault(f'x{ID}', []).append(item['X'])
datas.setdefault(f'y{ID}', []).append(item['Y'])
datas.setdefault(f'z{ID}', []).append(item['Z'])
if (count >= 30):
for key in datas:
data[key] = datas[key][-15:]
df = pd.DataFrame(data)
features = [extract_features(df)]
df_test = pd.DataFrame(features, columns=feature_name())
pred = knn_clf.predict(
df_test.iloc[:, 0:].to_numpy(dtype=float))
data_count += 1
queue.put_nowait(f'{pred[0]}')
print(f'{time}:{pred[0]}')
if data_count == 14:
dash_thread = Thread(target=dashboard,
args=(time, pred[0]))
dash_thread.start()
data_count = 0
else:
print(count)
progress = count / 30
queue.put_nowait(f'{progress:.2%}')
count += 1
else:
print(input)
def api_post_index():
errors = []
uri = flask.request.get_json()
print(uri)
uri = uri['spotify_uri']
*_, uri_type, _ = uri.split(':')
if 'http' in uri_type:
# given a url, so split by '/'
*_, uri_type, _ = uri.split('/')
try:
tracks = globals()[f'handle_{uri_type}_uri'](uri)
except spotipy.client.SpotifyException as e:
app.logger.error(f'received error from spotify: {str(e)}')
errors.append(str(e))
else:
for track in tracks:
track['artist'] = ', '.join(artist['name']
for artist in track['artists'])
duration = datetime.timedelta(milliseconds=track['duration_ms'])
track['duration'] = '{:02d}:{:02d}'.format(
(duration.seconds % 3600) // 60,
duration.seconds % 60,
)
duration = float(track['duration_ms']) / 1000
queue.put_nowait((track, duration - 2))
app.logger.debug(f'current queue size: {queue.unfinished_tasks}')
ret = {}
if errors:
ret['success'] = 0
ret['errors'] = errors
else:
ret['success'] = 1
ret['queue'] = list(queue.queue)
return flask.jsonify(ret)
def queueRandomOutputParams(config, trainingMorphsList, queue):
inputCnt = config.getShape()[0]
outputCnt = config.getShape()[1]
inputParams = [0] * inputCnt
newFace = copy.deepcopy(config.getBaseFace())
# Choose random number to decide what modification we apply
rand = random.random()
# select which morphs to modify
modifyIdxs = random.sample(range(len(newFace.morphFloats)),
random.randint(1, 25))
if len(trainingMorphsList) > 10:
randomIdxs = random.sample(range(len(trainingMorphsList)), 2)
newFaceMorphs = trainingMorphsList[randomIdxs[0]]
newFace.importFloatList(newFaceMorphs)
if rand < .6:
face2Morphs = trainingMorphsList[randomIdxs[1]]
face2 = copy.deepcopy(config.getBaseFace())
face2.importFloatList(face2Morphs)
mate(newFace, face2, modifyIdxs)
queue.put_nowait(inputParams + newFace.morphFloats)
elif rand < .9:
for idx in modifyIdxs:
newFace.changeMorph(idx, -1 + 2 * random.random())
queue.put_nowait(inputParams + newFace.morphFloats)
elif rand < .95:
numSteps = 5 #random.randint(5,15)
for idx in modifyIdxs:
face2 = copy.deepcopy(newFace)
minVal = face2.morphInfo[idx]['min']
maxVal = face2.morphInfo[idx]['max']
stepSize = (maxVal - minVal) / numSteps
face2.morphFloats[idx] = minVal
queue.put(inputParams + face2.morphFloats)
for step in range(numSteps):
face2.changeMorph(idx, stepSize)
queue.put(inputParams + face2.morphFloats)
else:
mutate(newFace, modifyIdxs)
queue.put_nowait(inputParams + newFace.morphFloats)
else:
# 90% chance to use baseface, otherwise completely random morphs.
if rand < .9:
mutate(newFace, modifyIdxs)
else:
newFace.randomize()
queue.put_nowait(inputParams + newFace.morphFloats)
def source_queue(self, procs, queues):
while True:
for index, proc in enumerate(procs):
try:
queue = queues[index]
h, w, c = self.shape[index]
frames_size = w * h * c
nv12_data = proc.stdout.read(frames_size)
if len(nv12_data) != frames_size:
# logger.error("source_queue read error index %s len %s", index, len(nv12_data))
continue
frame = np.frombuffer(nv12_data, dtype=np.uint8)
img = np.array(frame, dtype=np.uint8).reshape((h, w, c))
if DEBUG:
queue.put(img)
else:
queue.put_nowait(img)
except Exception as e:
# logger.error("source_queue queue full index %s", index)
pass
def insert_direct_generalization_of_node_in_queue(self, node, queue, i,
Si):
i_str = str(i)
self.cursor.execute("SELECT E" + i_str + ".end FROM C" + i_str +
", E" + i_str + " WHERE ID = E" + i_str +
".start and ID = " + str(node[0]))
nodes_to_put = set(self.cursor)
Si_indices = set()
for node in Si:
Si_indices.add(node[0])
for node_to_put in nodes_to_put:
# node_to_put == (ID,) -.-
if node_to_put[0] not in Si_indices:
continue
node_to_put = node_to_put[0]
self.cursor.execute("SELECT * FROM C" + i_str + " WHERE ID = " +
str(node_to_put))
node = (list(self.cursor)[0])
queue.put_nowait((-self.get_height_of_node(node), node))
def post_index():
errors = []
uri = flask.request.form.get('spotify_uri')
*_, uri_type, _ = uri.split(':')
if 'http' in uri_type:
# given a url, so split by '/'
*_, uri_type, _ = uri.split('/')
try:
tracks = globals()[f'handle_{uri_type}_uri'](uri)
except spotipy.client.SpotifyException as e:
app.logger.error(f'received error from spotify: {str(e)}')
errors.append(str(e))
else:
for track in tracks:
track['artist'] = ', '.join(artist['name']
for artist in track['artists'])
duration = datetime.timedelta(milliseconds=track['duration_ms'])
track['duration'] = '{:02d}:{:02d}'.format(
(duration.seconds % 3600) // 60,
duration.seconds % 60,
)
duration = float(track['duration_ms']) / 1000
queue.put_nowait((track, duration - 2))
app.logger.debug(f'current queue size: {queue.unfinished_tasks}')
if errors:
return flask.render_template('index.html',
queue=queue.queue,
errors=errors)
else:
return flask.redirect(flask.url_for('get_index'))
def get_rating_review_stats(kwargs=None, queue=None):
rnrArray = {}
try:
rnr = RatingsAndReviewsAPI(service_root=REVIEWS_SERVER)
sat_res = rnr.server_status()
if (Globals.DEBUG_SWITCH):
LOG.debug("review and rating service stat:%s", sat_res)
statlist = rnr.review_stats() #distroseries='any'
index = 0
for stat in statlist:
rnrStat = ReviewRatingStat(stat.package_name)
rnrStat.ratings_average = float(stat.ratings_average)
rnrStat.ratings_total = int(stat.ratings_total)
rnrStat.pkgname = stat.package_name
rnrStat.reviews_total = 0
rnrStat.useful = 0
rnrArray[stat.package_name] = rnrStat
index += 1
try:
queue.put_nowait(rnrStat)
except queue.Full:
if (Globals.DEBUG_SWITCH):
print("queue put exception")
if (Globals.DEBUG_SWITCH):
LOG.debug("got the rating and review list count:%d",
len(rnrArray))
return rnrArray
except Exception as e:
LOG.error("exception when getting rating and review stat...")
if (Globals.DEBUG_SWITCH):
print(e.args)
return {}
def oncePerMinute():
global next_interval_time
while True:
try:
# Sleep for the remainder of the time until the next
# interval, prevents timer drift. The calculated time
# to sleep could be negative if our clock got updated
# by ntptime so just sleep one minute in that case.
next_interval_time += 60
sleep_time = next_interval_time - time.time()
if sleep_time < 0:
sleep_time = 60
time.sleep(sleep_time)
device_time = str(datetime.datetime.now())
current_cpm = cpm
pressure = getPressure()
sea_level_pressure = pressure / pow(
1.0 - altitude_in_meters / 44330.0, 5.255)
body = urllib.parse.urlencode({
'cpm':
current_cpm,
'device_time':
device_time,
'pressure':
'{0:0.2f}'.format(pressure),
'sea_level_pressure':
'{0:0.2f}'.format(sea_level_pressure)
})
queue.put_nowait(body)
except:
print("Unexpected onePerMinute error: {0}".format(
sys.exc_info()[0]))
else:
print("Queued sample")
def multiCIF_stop(process, listener, queue):
""" Stop multiprocess, and safely end logging listener """
process.terminate()
process.join()
queue.put_nowait(None)
listener.join()
def MyMightyQueue(item=None):
if item.startswith("download"):
queue.put_nowait(item)
else:
queue.put(item)
def rebuild_all():
queue.put_nowait(("rebuild_all", ()))
def stop_task():
queue.put_nowait(("exit", ()))
def log_chat(event, metadata):
queue.put_nowait(
("log_chat", (datetime.datetime.now(pytz.utc), event, metadata)))
def clear_chat_log(nick):
queue.put_nowait(("clear_chat_log", (datetime.datetime.now(pytz.utc), nick)))
def get_screenshots(kwargs, queue=None):
screenshots = []
pkgname = kwargs['packagename']
version = kwargs['version']
cachedir = kwargs['cachedir']
thumbnail = kwargs['thumbnail']
screenshot = kwargs['screenshot']
thumbnailfile = kwargs['thumbnailfile']
screenshotfile = kwargs['screenshotfile']
screenshot_path_list = []
#if thumbnail and screenshot are not null,only get them
if (thumbnail and screenshot and thumbnailfile and screenshotfile):
try:
# if not os.path.exists(thumbnailfile):
# urlFile = urllib.request.urlopen(thumbnail)
# rawContent = urlFile.read()
# if rawContent:
# localFile = open(thumbnailfile,"wb")
# localFile.write(rawContent)
# localFile.close()
# else:
# print("get_screenshots,exists:",thumbnailfile)
screenshot_path_list.append(thumbnailfile)
queue.put_nowait(thumbnailfile)
# if not os.path.exists(screenshotfile):
# urlFile = urllib.request.urlopen(screenshot)
# rawContent = urlFile.read()
# if rawContent:
# localFile = open(screenshotfile,"wb")
# localFile.write(rawContent)
# localFile.close()
# else:
# print("get_screenshots,exists:",screenshotfile)
screenshot_path_list.append(screenshotfile)
queue.put_nowait(screenshotfile)
except urllib.error.HTTPError as e:
if (Globals.DEBUG_SWITCH):
print(e.code)
except urllib.error.URLError as e:
if (Globals.DEBUG_SWITCH):
print(str(e))
return screenshot_path_list
else:
#get urls of screenshots
screenshotURL = SCREENSHOT_JSON_URL % pkgname
rawContent = None
try:
urlFile = urllib.request.urlopen(screenshotURL)
rawContent = urlFile.read()
if not rawContent:
return []
except urllib.error.HTTPError as e:
if (Globals.DEBUG_SWITCH):
print(e.code)
except urllib.error.URLError as e:
if (Globals.DEBUG_SWITCH):
print(str(e))
if rawContent is None:
return []
try:
jsonContent = json.loads(rawContent)
except ValueError as e:
if (Globals.DEBUG_SWITCH):
print("can not decode: '%s' (%s)" % (rawContent, e))
jsonContent = None
return []
if isinstance(jsonContent, dict):
# a list of screenshots as listsed online
screenshots = jsonContent['screenshots']
else:
# fallback to a list of screenshots as supplied by the axi
screenshots = []
#we should choose the suitable ones for showing
#????
screenshot_path_list = []
try:
for item in screenshots:
filename = item['small_image_url'].split(pkgname + '/')[1]
destfile = cachedir + pkgname + item[
'version'] + "_" + filename
if not os.path.exists(destfile):
urlFile = urllib.request.urlopen(
item['small_image_url'])
rawContent = urlFile.read()
if not rawContent:
continue
localFile = open(destfile, "wb")
localFile.write(rawContent)
localFile.close()
screenshot_path_list.append(destfile)
queue.put_nowait(destfile)
except urllib.error.HTTPError as e:
if (Globals.DEBUG_SWITCH):
print(e.code)
except urllib.error.URLError as e:
if (Globals.DEBUG_SWITCH):
print(str(e))
return screenshot_path_list
def checkqueues(self, queues, resultlist):
""" check if resultlist is to be sent to the queues. if so do it! """
for queue in queues:
for item in resultlist: queue.put_nowait(item)
return True
return False
def stop_task():
queue.put_nowait(("exit", ()))
def receive_data(pin): queue.put_nowait(datetime.datetime.now())
def rebuild_all(period=7):
queue.put_nowait(("rebuild_all", (period, )))
def clear_chat_log_msg(msgid):
queue.put_nowait(("clear_chat_log_msg", (msgid, )))
def clear_chat_log(nick):
queue.put_nowait(
("clear_chat_log", (datetime.datetime.now(pytz.utc), nick)))
def rebuild_all(period=7):
queue.put_nowait(("rebuild_all", (period,)))
def receive_data(pin):
queue.put_nowait(datetime.datetime.now())
def clear_chat_log_msg(msgid):
queue.put_nowait(("clear_chat_log_msg", (msgid,)))
def log_chat(event, metadata):
queue.put_nowait(("log_chat", (datetime.datetime.now(pytz.utc), event, metadata)))
def rebuild_all():
queue.put_nowait(("rebuild_all", ()))
def send_all(self,msg): for queue in self.active_connections: queue.put_nowait(msg)
