def game_player(opcodes: OpCodes):
"""
Create the processor to play the game
"""
painting = defaultdict(lambda: 0)
input_queue = SimpleQueue()
output_queue = SimpleQueue()
processor = Processor(input_queue, output_queue)
# let the robot run in a separate thread
thread = threading.Thread(target=processor, name='Game', args=(opcodes, ))
thread.start()
thread.join()
while not output_queue.empty():
# not sure where the robot will break, so
x = output_queue.get()
y = output_queue.get()
tile_id = output_queue.get()
painting[(x, y)] = tile_id
return painting
def stream(method, url, **kwargs):
"""Replace httpx.stream.
Usage:
stream = poolrequests.stream(...)
response = next(stream)
for chunk in stream:
...
httpx.Client.stream requires to write the httpx.HTTPTransport version of the
the httpx.AsyncHTTPTransport declared above.
"""
q = SimpleQueue()
future = asyncio.run_coroutine_threadsafe(
stream_chunk_to_queue(get_network(), q, method, url, **kwargs),
get_loop())
# yield response
response = q.get()
if isinstance(response, Exception):
raise response
response.close = MethodType(_close_response_method, response)
yield response
# yield chunks
chunk_or_exception = q.get()
while chunk_or_exception is not None:
if isinstance(chunk_or_exception, Exception):
raise chunk_or_exception
yield chunk_or_exception
chunk_or_exception = q.get()
future.result()
def main():
# setup worker and result queues
queue = SimpleQueue()
rtn_queue = SimpleQueue()
for i in range(FETCH_COUNT):
queue.put(i)
queue.put(-1)
# bring up workers
with ThreadPoolExecutor(max_workers=WORKERS) as executor:
for i in range(WORKERS):
executor.submit(fetch_thread, queue, rtn_queue)
print(f"Worker {i} online.")
print("Finished.")
# collect results
rtn_queue.put(-1)
rtn = rtn_queue.get()
succ = 0
fail = 0
while rtn != -1:
if rtn == 1:
succ += 1
else:
fail += 1
rtn = rtn_queue.get()
# final report
print(f"[SUCC] {succ} [FAIL] {fail}")
class Alice(Thread):
def __init__(self, outbox=None):
Thread.__init__(self)
self.inbox = SimpleQueue()
self.outbox = outbox
self.p, self.g = dhlib._groups[randint(0, 2)]
self.Ka, self.KA = dhlib.gen_key(self.p, self.g)
def run(self):
assert self.outbox is not None
self.outbox.put((self.p, self.g))
assert self.inbox.get() == 'ACK'
self.outbox.put(self.KA)
KB = self.inbox.get()
s = pow(KB, self.Ka, self.p)
key = SHA1.new(int_to_bytes(s)).digest()[:BS]
print('Alice: key', key.hex())
iv = get_random_bytes(BS)
msg = b'Lorem ipsum dolor sit amet, consectetur adipiscing elit. Sed non risus. Suspendisse lectus tortor, dignissim sit amet, adipiscing nec, ultricies sed, dolor.'
ciph = AES.new(key, AES.MODE_CBC, iv).encrypt(pad(msg, BS))
self.outbox.put((ciph, iv))
ciph, iv = self.inbox.get()
msgB = unpad(AES.new(key, AES.MODE_CBC, iv).decrypt(ciph), BS)
assert msg == msgB
print('Alice: msg ok, done')
def game_player(game_input: OpCodes, opcodes: OpCodes):
"""
Create the processor to play the game
"""
input_queue = SimpleQueue()
output_queue = SimpleQueue()
processor = Processor(input_queue, output_queue)
map(input_queue.putm game_input)
# let the robot run in a separate thread
thread = threading.Thread(target=processor, args=(opcodes, ), daemon=True)
thread.start()
score = 0
while not output_queue.empty():
# not sure where the robot will break, so
x = output_queue.get()
y = output_queue.get()
tile_id = output_queue.get()
if x == -1 and y == 0:
score = max(tile_id, score)
return (score, ) # must return a tuple
class SRPClient(Thread):
def __init__(self, outbox=None):
Thread.__init__(self)
self.inbox = SimpleQueue()
self.outbox = outbox
self.N = _N
self.g = _G
self.Ka, self.KA = dhlib.gen_key(self.N, self.g) # client DH key
self.k = _K
self.mel = b'*****@*****.**'
self.pwd = b's3cr3t_p4ssw0rd'
def run(self):
assert self.outbox is not None
self.outbox.put((self.mel, self.KA))
salt, B = self.inbox.get()
u = int.from_bytes(SHA256.new(int_to_bytes(self.KA)+int_to_bytes(B)).digest(), 'big')
# using the salt, retrieve the server db entry key
Kx = int.from_bytes(SHA256.new(salt+self.pwd).digest(), 'big')
KX = pow(self.g, Kx, self.N)
sec = pow(B-self.k*KX, self.Ka+u*Kx, self.N)
# sec = (B-k*KX)^(Ka+u*Kx) = KB^(Ka+u*Kx) = g^(Kb*(Ka+u*Kx))
key = SHA256.new(int_to_bytes(sec)).digest()
print('Client: key', key.hex())
mac = HMAC.new(key, salt, SHA256).digest()
self.outbox.put(mac)
ok = self.inbox.get()
assert ok == b'OK'
print('Client: ok, done')
class SimplifiedSRPServer(Thread):
def __init__(self, outbox=None):
Thread.__init__(self)
self.inbox = SimpleQueue()
self.outbox = outbox
self.N = _N
self.g = _G
self.Kb, self.KB = dhlib.gen_key(self.N, self.g) # server DH key
# precomp. server db
self.DB = {}
for mel, pwd in IDS:
salt = get_random_bytes(BS)
Kx = int.from_bytes(SHA256.new(salt + pwd).digest(), 'big')
KX = pow(self.g, Kx, self.N)
self.DB[mel] = (salt, KX) # salt & pwd DH "public" key
def run(self):
assert self.outbox is not None
# get client id and retrieve its entry
mel, KA = self.inbox.get()
assert mel in self.DB
salt, KX = self.DB[mel]
# in this simplified version, we do not use k to mix KB and KX,
# we directly use the server "public" key KB instead
u = int.from_bytes(get_random_bytes(128), 'big')
self.outbox.put((salt, self.KB, u))
sec = pow(KA * pow(KX, u, self.N), self.Kb, self.N)
# sec = g^((Ka+Kx*u)*Kb)
key = SHA256.new(int_to_bytes(sec)).digest()
print('Server: key', key.hex())
mac = self.inbox.get()
HMAC.new(key, salt, SHA256).verify(mac)
print('Server: ok')
self.outbox.put(b'OK')
print('Server: done')
class SimplifiedSRPClient(Thread):
def __init__(self, outbox=None):
Thread.__init__(self)
self.inbox = SimpleQueue()
self.outbox = outbox
self.N = _N
self.g = _G
self.Ka, self.KA = dhlib.gen_key(self.N, self.g) # client DH key
self.mel, self.pwd = random.choice(IDS)
print(f'Client: {self.mel.decode()}:{self.pwd.decode()}')
def run(self):
assert self.outbox is not None
self.outbox.put((self.mel, self.KA))
salt, KB, u = self.inbox.get()
# using the salt, retrieve the server db entry key
Kx = int.from_bytes(SHA256.new(salt + self.pwd).digest(), 'big')
sec = pow(KB, self.Ka + u * Kx, self.N)
# sec = KB^(Ka+u*Kx) = g^(Kb*(Ka+u*Kx))
key = SHA256.new(int_to_bytes(sec)).digest()
print('Client: key', key.hex())
mac = HMAC.new(key, salt, SHA256).digest()
self.outbox.put(mac)
ok = self.inbox.get()
assert ok == b'OK'
print('Client: ok, done')
def checkValidString(self, s: str) -> bool:
open_parant_count = 0
star_queue = SimpleQueue()
for index, c in enumerate(s):
if c == '*':
star_queue.put(index)
elif c == '(':
open_parant_count += 1
elif open_parant_count > 0:
open_parant_count -= 1
elif not star_queue.empty():
pos = star_queue.get()
s = s[:pos] + '(' + s[pos + 1:]
else:
return False
open_parant_count = 0
star_queue = SimpleQueue()
for index, c in enumerate(s[::-1]):
if c == '*':
star_queue.put(index)
elif c == ')':
open_parant_count += 1
elif open_parant_count > 0:
open_parant_count -= 1
elif not star_queue.empty():
pos = star_queue.get()
s = s[:pos] + '(' + s[pos + 1:]
else:
return False
return True
def reproduce(chromosome1, chromosome2): # crossover_index = int(CROSSOVER_POINT * len(chromosome1)) # child1 = chromosome1[0:crossover_index] + chromosome2[crossover_index:] # child2 = chromosome2[0:crossover_index] + chromosome1[crossover_index:] # return child1, child2 pos1, pos2 = random.sample([x for x in range(1, len(chromosome1) - 1)], k=2) if pos1 > pos2: pos1, pos2 = pos2, pos1 genesFrom1 = chromosome1[pos1:pos2+1] genesFrom2 = SimpleQueue() for i in range(len(chromosome2)): gene = chromosome2[i] if gene not in genesFrom1: genesFrom2.put(gene) child = [] for i in range(pos1): child.append(genesFrom2.get()) child = child + genesFrom1 for i in range(len(child), len(chromosome2)): child.append(genesFrom2.get()) return child
class Eve(Thread):
def __init__(self, outA=None, outB=None):
Thread.__init__(self)
self.inA = SimpleQueue()
self.inB = SimpleQueue()
self.outA = outA
self.outB = outB
def run(self):
assert self.outA is not None
assert self.outB is not None
p, g, KA = self.inA.get()
self.outB.put((p, g, p)) # KA := p (= 0)
KB = self.inB.get()
self.outA.put(p) # KB := p (= 0)
ciph, iv = self.inA.get()
self.outB.put((ciph, iv))
key = SHA1.new(b'').digest()[:BS] # s = 0
msg = unpad(AES.new(key, AES.MODE_CBC, iv).decrypt(ciph), BS)
print('Eve: ', msg)
ciph, iv = self.inB.get()
self.outA.put((ciph, iv))
msg = unpad(AES.new(key, AES.MODE_CBC, iv).decrypt(ciph), BS)
print('Eve: ', msg)
print('Eve: done')
def game_player(scr, *args, opcodes: OpCodes):
"""
Create the processor to play the game
"""
painting = defaultdict(lambda: 0)
input_queue = SimpleQueue()
output_queue = SimpleQueue()
processor = Processor(input_queue, output_queue)
# let the robot run in a separate thread
thread = threading.Thread(target=processor,
name='Game',
args=(opcodes, ),
daemon=True)
thread.start()
curses.curs_set(0)
scr.nodelay(True)
scr.border(0)
max_y = 0
score = None
while thread.is_alive():
ch = scr.getch()
if ch == ord(','):
input_queue.put(-1)
elif ch == ord('.'):
input_queue.put(1)
elif ch == ord(' '):
input_queue.put(0)
elif ch == ord('q'):
return 'exit'
while not output_queue.empty():
# not sure where the robot will break, so
x = output_queue.get()
y = output_queue.get()
tile_id = output_queue.get()
if not (x == -1 and y == 0):
painting[(y + 1, x + 1)] = tile_id
max_y = max(max_y, y + 1)
else:
score = tile_id
for location, tile_id in painting.items():
scr.addstr(*location, SYMBOL_MAP[tile_id], curses.A_NORMAL)
if score is not None:
scr.addstr(max_y + 4, 1, str(score), curses.A_BOLD)
scr.clrtoeol()
return painting
def robot_painter(opcodes):
"""
Create a processor for the robot and run the opcodes.
There's no idea of where the robot starts, so let's assume (0, 0) for now. We can record locationss in
a dictionary, and record output values etc.
"""
painting = defaultdict(lambda: 0)
current_location = (0, 0)
movement = deque([
(1, 0), # up
(0, 1), # right
(-1, 0), # down
(0, -1), # left
])
input_queue = SimpleQueue()
output_queue = SimpleQueue()
processor = Processor(input_queue, output_queue)
# let the robot run in a separate thread
thread = threading.Thread(target=processor, name='Robot', args=(opcodes, ))
thread.start()
while True:
# not sure where the robot will break, so
if not thread.is_alive():
break
# first, give the system the colour of the current location
input_queue.put(painting[current_location])
# not sure where the robot will break, so
if not thread.is_alive():
break
# get the new colour
painting[current_location] = output_queue.get()
# not sure where the robot will break, so
if not thread.is_alive():
break
# move the robot
new_direction = output_queue.get()
if new_direction == 0:
movement.rotate(1)
elif new_direction == 1:
movement.rotate(-1)
else:
raise ValueError('Movement direction not modified')
current_location = (current_location[0] + movement[0][0],
current_location[1] + movement[0][1])
return len(painting)
def mergeKLists(self, lists: List[ListNode]) -> ListNode:
if len(lists) == 0: return None
q = SimpleQueue()
for l in lists:
q.put(l)
while(q.qsize() > 1):
q.put(self.merge(q.get(), q.get()))
return q.get()
def run_init_workflow(app_facade, data_facade):
queue = SimpleQueue()
workflow_id = IdentityService.random()
workflow = app_facade.workflow_factory.make_init_workflow(
workflow_id, app_facade, data_facade)
app_facade.evt_dispatcher.observe(
{
workflow.Completed: queue.put,
workflow.Aborted: queue.put
}, times=1)
app_facade.workflow_manager.start(workflow)
queue.get()
return workflow.is_COMPLETED()
def bfs(self, start, target, visited):
# Create an empty queue.
q = SimpleQueue()
# Put the starting userID in a list in our queue.
q.put([start])
# While the queue is not empty....
while not q.empty():
# Grab the current path.
path = q.get()
# Grab the last vertex in the path.
curr_vertex = path[-1]
# If current vertex is the target, return the path.
if curr_vertex == target:
visited[target] = path
return
# Iterate through all neighbors.
for social_connection in self.friendships[curr_vertex]:
if social_connection != start and social_connection not in path:
# Create new path and add current neighbor.
new_path = list(path)
new_path.append(social_connection)
# Add that new path to the queue.
q.put(new_path)
def backward(self, target_node, deriv_pass, ancestors=None):
"""Perform a backward computation to compute the derivatives for all
parameters that affect the target node
Parameters
----------
target_node : integer
The id of the node under consideration
deriv_pass : np.ndarray (nparams)
A gradient vector
Returns
-------
derivative : np.ndarray(nparams)
A vector containing the derivative of all parameters
"""
if ancestors is None:
ancestors = nx.ancestors(self.graph, target_node)
anc_and_target = ancestors.union(set([target_node]))
# Evaluate the node
self.graph.nodes[target_node]['pass_down'] = deriv_pass
# Gradient with respect to beta
self.graph.nodes[target_node]['derivative'] = \
np.dot(self.graph.nodes[target_node]['pass_down'],
self.graph.nodes[target_node]['pre_grad'])
queue = SimpleQueue()
queue.put(target_node)
for node in ancestors:
self.graph.nodes[node]['children_left'] = set(
self.graph.successors(node)).intersection(anc_and_target)
self.graph.nodes[node]['pass_down'] = 0.0
self.graph.nodes[node]['derivative'] = 0.0
while not queue.empty():
node = queue.get()
pass_down = self.graph.nodes[node]['pass_down']
for parent in self.graph.predecessors(node):
self.graph.nodes[parent]['pass_down'] += \
pass_down * self.graph.edges[parent, node]['eval']
self.graph.edges[parent, node]['derivative'] = \
np.dot(pass_down,self.graph.edges[parent, node]['pre_grad'])
self.graph.nodes[parent]['derivative'] += \
np.dot(pass_down * self.graph.edges[parent, node]['eval'],
self.graph.nodes[parent]['pre_grad'])
self.graph.nodes[parent]['children_left'].remove(node)
if self.graph.nodes[parent]['children_left'] == set():
queue.put(parent)
return self.get_derivative()
def ladderLength(self, beginWord: str, endWord: str,
wordList: List[str]) -> int:
q = SimpleQueue()
wl = set([beginWord] + wordList)
g = defaultdict(list)
l = len(beginWord)
for w in wl:
for i in range(l):
for c in 'abcdefghijklmnopqrstuvwxyz':
nw = w[:i] + c + w[i + 1:]
if nw == w or nw not in wl:
continue
g[w].append(nw)
g[nw].append(w)
q.put((beginWord, 1))
seen = set([beginWord])
while q.qsize():
w, dist = q.get()
for nw in g[w]:
if nw == endWord:
return dist + 1
# print(w, '->', nw)
if nw in seen:
continue
seen.add(nw)
q.put((nw, dist + 1))
return 0
def shortestPath(self, nid1, nid2):
if nid1 == nid2:
return [nid1]
Q = SimpleQueue()
P = [-1 for _ in range(self.order())]
visited = [False for _ in range(self.order())]
Q.put(nid1)
visited[nid1] = True
while not Q.empty():
nid = Q.get()
for next_nid in self._nodes[nid].outneighbours:
if not visited[next_nid]:
if next_nid == nid2:
path = [next_nid]
while nid > -1:
path.append(nid)
nid = P[nid]
return path
P[next_nid] = nid
Q.put(next_nid)
visited[next_nid] = True
return None
class RDProxyClientProtocol:
def __init__(self, server, client_addr):
self.server = server
self.client_addr = client_addr
self.transport = None
self.q = SimpleQueue()
def connection_made(self, transport):
self.transport = transport
self.dispatch_message()
def queue_message(self, message):
self.q.put(message)
self.dispatch_message()
def dispatch_message(self):
if self.transport != None:
while not self.q.empty():
item = self.q.get()
if random.random() >= DROP_RATE:
self.transport.sendto(item)
def datagram_received(self, data, addr):
if random.random() >= DROP_RATE:
self.server.transport.sendto(data, self.client_addr)
def error_received(self, exc):
print('Error received:', exc)
def connection_lost(self, exc):
pass
class BoundedBlockingQueue(object):
def __init__(self, capacity: int):
self.enque_lock = Lock()
self.deque_lock = Lock()
self.cap = capacity
self.q = SimpleQueue()
self.deque_lock.acquire()
def enqueue(self, element: int) -> None:
self.enque_lock.acquire()
self.q.put(element)
if self.q.qsize() < self.cap:
self.enque_lock.release()
if self.deque_lock.locked():
self.deque_lock.release()
def dequeue(self) -> int:
self.deque_lock.acquire()
val = None
if self.q.qsize() > 0:
val = self.q.get()
if self.q.qsize():
self.deque_lock.release()
if val and self.enque_lock.locked():
self.enque_lock.release()
return val
def size(self) -> int:
return self.q.qsize()
def get_details(start_pos, keys, doors, walkable):
q = SimpleQueue()
seen = set()
details = dict()
q.put({"position": start_pos, "distance": 0, "keys_needed": set()})
# queue = [{"position": (int, int), "distance": int, "keys_needed": {string}}]
while not q.empty():
item = q.get()
q_pos = item["position"]
if q_pos in seen:
continue
seen.add(q_pos)
q_distance = item["distance"]
q_keys = item["keys_needed"]
if q_pos in keys.keys():
details[q_pos] = {"distance": q_distance, "keys_needed": q_keys}
if q_pos in doors.keys():
q_keys.add(doors[q_pos].lower())
for n_pos in get_orthogonal_neighbours(q_pos):
if n_pos in walkable - seen:
q.put(
{"position": n_pos, "distance": q_distance + 1, "keys_needed": q_keys.copy()})
return details
async def main():
global devices
global client
global connector
client = ButtplugClient("League of Legends")
connector = CustomWebsocketConnection("ws://127.0.0.1:12345")
client.device_added_handler += device_added
client.device_removed_handler += device_removed
try:
await client.connect(connector)
except ButtplugClientConnectorError as e:
print("Could not connect to server (check Intiface desktop), exiting: {}".format(e.message))
return
await scan_and_pick_device()
queue = SimpleQueue()
queue2 = SimpleQueue()
lol = League_Thread(queue)
lol.start()
bu = Buttplug_Thread(queue, chosen_device, queue2)
bu.start()
while True:
item = queue2.get()
#print(item)
if item == "stop":
await chosen_device.send_stop_device_cmd()
else:
await chosen_device.send_vibrate_cmd(item)
def ac3(self, arcs=None):
"""
Update `self.domains` such that each variable is arc consistent.
If `arcs` is None, begin with initial list of all arcs in the problem.
Otherwise, use `arcs` as the initial list of arcs to make consistent.
Return True if arc consistency is enforced and no domains are empty;
return False if one or more domains end up empty.
"""
# Create arcs if None
if arcs is None:
arcs = SimpleQueue()
for x in combinations(self.crossword.variables, 2):
arcs.put(x)
while not arcs.empty():
x, y = arcs.get()
if self.revise(x, y):
if not self.domains[x]:
return False
neighbors = self.crossword.neighbors(x)
neighbors.remove(y)
for x_neighbor in neighbors:
arcs.put((x_neighbor, x))
return True
def escape_maze():
maze = []
maze.append(['#', '#', 'O', '#', '#', '#', '#'])
maze.append(['#', ' ', ' ', ' ', '#', ' ', '#'])
maze.append(['#', ' ', '#', ' ', '#', ' ', '#'])
maze.append(['#', ' ', '#', ' ', ' ', ' ', '#'])
maze.append(['#', ' ', '#', '#', '#', ' ', '#'])
maze.append(['#', ' ', ' ', ' ', '#', ' ', '#'])
maze.append(['#', '#', '#', '#', '#', 'X', '#'])
directions = [(1, 0), (-1, 0), (0, 1), (0, -1)]
visited = [row[:] for row in maze]
p_end = (6, 5)
p0 = (0, 2)
q = SimpleQueue()
q.put(p0)
time = 0
while not q.empty():
for _ in range(q.qsize()):
x = q.get()
visited[x[0]][x[1]] = str(time)
if x[0] == p_end[0] and x[1] == p_end[1]:
return visited
for d in directions:
p1 = (x[0] + d[0], x[1] + d[1])
if visited[p1[0]][p1[1]] in ' X':
q.put(p1)
time += 1
return None
def processing():
threads = []
tasks_queue = SimpleQueue()
results_queue = SimpleQueue()
for _ in range(16):
thread = threading.Thread(target=thread_main,
args=(
tasks_queue,
results_queue,
))
threads.append(thread)
for _ in range(tasks_count()):
task = retrieve_task()
tasks_queue.put(task)
for thread in threads:
thread.start()
for thread in threads:
thread.join()
for _ in range(results_queue.qsize()):
result = results_queue.get(block=False)
populate_result(result)
def slidingPuzzle(self, board: List[List[int]]) -> int:
endstate = tuple([tuple([1, 2, 3]), tuple([4, 5, 0])])
q = SimpleQueue()
q.put(tuple(map(tuple, board)))
d = {}
d[tuple(map(tuple, board))] = 0
def genstate(s):
s = list(map(list, s))
for i in range(2):
for j in range(3):
if s[i][j] != 0:
continue
for ni, nj in zip([i - 1, i, i + 1, i],
[j, j + 1, j, j - 1]):
if ni < 0 or ni > 1 or nj < 0 or nj > 2:
continue
news = list(map(list, s))
news[ni][nj], news[i][j] = news[i][j], news[ni][nj]
yield tuple(map(tuple, news))
while q.qsize():
state = q.get()
if state == endstate:
return d[state]
for nxtstate in genstate(state):
if nxtstate == endstate:
return d[state] + 1
if nxtstate in d:
continue
d[nxtstate] = d[state] + 1
q.put(nxtstate)
return -1
def bfs(self, node: List[int]):
def rotate_wheel(node: List[int], depth: int, pos: int, incre: int):
assert incre in [1, -1]
new_node = node.copy()
new_node[pos] = (node[pos] + incre) % 10
new_node_str = ''.join(map(str, new_node))
if new_node_str == self.target:
self.min_depth = min(self.min_depth, depth + 1)
self.visited[new_node_str] = None
elif new_node_str not in self.deadends and \
new_node_str not in self.visited:
q.put([new_node, depth + 1])
self.visited[new_node_str] = None
q = SimpleQueue()
q.put([node, 0])
while not q.empty():
node, depth = q.get()
rotate_wheel(node, depth, 0, 1)
rotate_wheel(node, depth, 0, -1)
rotate_wheel(node, depth, 1, 1)
rotate_wheel(node, depth, 1, -1)
rotate_wheel(node, depth, 2, 1)
rotate_wheel(node, depth, 2, -1)
rotate_wheel(node, depth, 3, 1)
rotate_wheel(node, depth, 3, -1)
if self.min_depth == depth + 1: break
def run_all(folder, s_target):
sql_queue = SimpleQueue()
size = 0
# Ordered scripts are formatted like {number}-name.sql
# Scripts without a number prefix will be ran last
for filename in sorted(glob.iglob(f'{folder}/**/*.sql', recursive=True), key=get_file_order):
sql_queue.put(get_sql(filename))
size += 1
if size == 0:
return
num_tries = 0
max_tries = size * 2
while not sql_queue.empty() and num_tries < max_tries:
try:
sql = sql_queue.get()
s_target.execute(sql)
s_target.commit()
num_tries = 0
except ProgrammingError as e:
message = str(e.orig).strip()
if 'relation' in message and 'does not exist' in message:
s_target.rollback()
print(f'Object does not exist yet: {message}. Re-queueing...')
sql_queue.put(sql)
num_tries += 1
else:
raise
if num_tries >= max_tries:
print(
f'Number of attempts exceeded configured threshold of {max_tries}')
sys.exit(1)
def run_all(folder, s_target):
sql_queue = SimpleQueue()
size = 0
for filename in glob.iglob(f'{folder}/**/*.sql', recursive=True):
sql_queue.put(get_sql(filename))
size += 1
num_tries = 0
max_tries = size * 2
while not sql_queue.empty() and num_tries < max_tries:
try:
sql = sql_queue.get()
s_target.execute(sql)
s_target.commit()
num_tries = 0
except ProgrammingError as e:
message = str(e.orig).strip()
if 'relation' in message and 'does not exist' in message:
s_target.rollback()
print(f'Object does not exist yet: {message}. Re-queueing...')
sql_queue.put(sql)
num_tries += 1
else:
raise
if num_tries >= max_tries:
print(f'Number of attempts exceeded configured threshold of {max_tries}')
sys.exit(1)
class BufferedReader(Listener):
"""
A BufferedReader is a subclass of :class:`~can.Listener` which implements a
**message buffer**: that is, when the :class:`can.BufferedReader` instance is
notified of a new message it pushes it into a queue of messages waiting to
be serviced. The messages can then be fetched with
:meth:`~can.BufferedReader.get_message`.
Putting in messages after :meth:`~can.BufferedReader.stop` has be called will raise
an exception, see :meth:`~can.BufferedReader.on_message_received`.
:attr bool is_stopped: ``True`` iff the reader has been stopped
"""
def __init__(self):
# set to "infinite" size
self.buffer = SimpleQueue()
self.is_stopped = False
def on_message_received(self, msg):
"""Append a message to the buffer.
:raises: BufferError
if the reader has already been stopped
"""
if self.is_stopped:
raise RuntimeError("reader has already been stopped")
else:
self.buffer.put(msg)
def get_message(self, timeout=0.5):
"""
Attempts to retrieve the latest message received by the instance. If no message is
available it blocks for given timeout or until a message is received, or else
returns None (whichever is shorter). This method does not block after
:meth:`can.BufferedReader.stop` has been called.
:param float timeout: The number of seconds to wait for a new message.
:rytpe: can.Message or None
:return: the message if there is one, or None if there is not.
"""
try:
return self.buffer.get(block=not self.is_stopped, timeout=timeout)
except Empty:
return None
def stop(self):
"""Prohibits any more additions to this reader.
"""
self.is_stopped = True
class NetworkConnection(object):
class State(object):
kCreated = 0
kInit = 1
kHandshake = 2
kSynchronized = 3
kActive = 4
kDead = 5
def __init__(self, uid, stream, notifier, handshake, get_entry_type, verbose=False):
# logging debugging
self.m_verbose = verbose
self.m_uid = uid
self.m_stream = stream
self.m_notifier = notifier
self.m_handshake = handshake
self.m_get_entry_type = get_entry_type
self.m_active = False
self.m_proto_rev = 0x0300
self.state = self.State.kCreated
self.m_state_mutex = threading.Lock()
self.m_last_update = 0
self.m_outgoing = Queue()
self.m_process_incoming = None
self.m_read_thread = None
self.m_write_thread = None
self.m_remote_id_mutex = threading.Lock()
self.m_remote_id = None
self.m_last_post = 0
self.m_pending_mutex = threading.Lock()
self.m_pending_outgoing = []
self.m_pending_update = {}
# Condition variables for shutdown
self.m_shutdown_mutex = threading.Lock()
# Not needed in python
# self.m_read_shutdown_cv = threading.Condition()
# self.m_write_shutdown_cv = threading.Condition()
self.m_read_shutdown = False
self.m_write_shutdown = False
# turn off Nagle algorithm; we bundle packets for transmission
try:
self.m_stream.setNoDelay()
except IOError as e:
logger.warning("Setting TCP_NODELAY: %s", e)
def start(self):
if self.m_active:
return
self.m_active = True
self.set_state(self.State.kInit)
# clear queue
try:
while True:
self.m_outgoing.get_nowait()
except Empty:
pass
# reset shutdown flags
with self.m_shutdown_mutex:
self.m_read_shutdown = False
self.m_write_shutdown = False
# start threads
self.m_write_thread = SafeThread(
target=self._writeThreadMain, name="nt-net-write"
)
self.m_read_thread = SafeThread(target=self._readThreadMain, name="nt-net-read")
def __repr__(self):
try:
return "<NetworkConnection 0x%x %s>" % (id(self), self.info())
except Exception:
return "<NetworkConnection 0x%x ???>" % id(self)
def stop(self):
logger.debug("NetworkConnection stopping (%s)", self)
if not self.m_active:
return
self.set_state(self.State.kDead)
self.m_active = False
# closing the stream so the read thread terminates
self.m_stream.close()
# send an empty outgoing message set so the write thread terminates
self.m_outgoing.put([])
# wait for threads to terminate, timeout
self.m_write_thread.join(1)
if self.m_write_thread.is_alive():
logger.warning("%s did not die", self.m_write_thread.name)
self.m_read_thread.join(1)
if self.m_read_thread.is_alive():
logger.warning("%s did not die", self.m_write_thread.name)
# clear queue
try:
while True:
self.m_outgoing.get_nowait()
except Empty:
pass
def get_proto_rev(self):
return self.m_proto_rev
def get_stream(self):
return self.m_stream
def info(self):
return ConnectionInfo(
self.remote_id(),
self.m_stream.getPeerIP(),
self.m_stream.getPeerPort(),
self.m_last_update,
self.m_proto_rev,
)
def is_connected(self):
return self.state == self.State.kActive
def last_update(self):
return self.m_last_update
def set_process_incoming(self, func):
self.m_process_incoming = func
def set_proto_rev(self, proto_rev):
self.m_proto_rev = proto_rev
def set_state(self, state):
with self.m_state_mutex:
State = self.State
# Don't update state any more once we've died
if self.state == State.kDead:
return
# One-shot notify state changes
if self.state != State.kActive and state == State.kActive:
info = self.info()
self.m_notifier.notifyConnection(True, info)
logger.info(
"CONNECTED %s port %s (%s)",
info.remote_ip,
info.remote_port,
info.remote_id,
)
elif self.state != State.kDead and state == State.kDead:
info = self.info()
self.m_notifier.notifyConnection(False, info)
logger.info(
"DISCONNECTED %s port %s (%s)",
info.remote_ip,
info.remote_port,
info.remote_id,
)
if self.m_verbose:
logger.debug(
"%s: %s -> %s", self, _state_map[self.state], _state_map[state]
)
self.state = state
# python optimization: don't use getter here
# def state(self):
# return self.m_state
def remote_id(self):
with self.m_remote_id_mutex:
return self.m_remote_id
def set_remote_id(self, remote_id):
with self.m_remote_id_mutex:
self.m_remote_id = remote_id
def uid(self):
return self.m_uid
def _sendMessages(self, msgs):
self.m_outgoing.put(msgs)
def _readThreadMain(self):
decoder = WireCodec(self.m_proto_rev)
verbose = self.m_verbose
def _getMessage():
decoder.set_proto_rev(self.m_proto_rev)
try:
return Message.read(self.m_stream, decoder, self.m_get_entry_type)
except IOError as e:
logger.warning("read error in handshake: %s", e)
# terminate connection on bad message
self.m_stream.close()
return None
self.set_state(self.State.kHandshake)
try:
handshake_success = self.m_handshake(self, _getMessage, self._sendMessages)
except Exception:
logger.exception("Unhandled exception during handshake")
handshake_success = False
if not handshake_success:
self.set_state(self.State.kDead)
self.m_active = False
else:
self.set_state(self.State.kActive)
try:
while self.m_active:
if not self.m_stream:
break
decoder.set_proto_rev(self.m_proto_rev)
try:
msg = Message.read(
self.m_stream, decoder, self.m_get_entry_type
)
except Exception as e:
if not isinstance(e, StreamEOF):
if verbose:
logger.exception("read error")
else:
logger.warning("read error: %s", e)
# terminate connection on bad message
self.m_stream.close()
break
if verbose:
logger.debug(
"%s received type=%s with str=%s id=%s seq_num=%s value=%s",
self.m_stream.sock_type,
msgtype_str(msg.type),
msg.str,
msg.id,
msg.seq_num_uid,
msg.value,
)
self.m_last_update = monotonic()
self.m_process_incoming(msg, self)
except IOError as e:
# connection died probably
logger.debug("IOError in read thread: %s", e)
except Exception:
logger.warning("Unhandled exception in read thread", exc_info=True)
self.set_state(self.State.kDead)
self.m_active = False
# also kill write thread
self.m_outgoing.put([])
with self.m_shutdown_mutex:
self.m_read_shutdown = True
def _writeThreadMain(self):
encoder = WireCodec(self.m_proto_rev)
verbose = self.m_verbose
out = []
try:
while self.m_active:
msgs = self.m_outgoing.get()
if verbose:
logger.debug("write thread woke up")
if msgs:
logger.debug(
"%s sending %s messages", self.m_stream.sock_type, len(msgs)
)
if not msgs:
continue
encoder.set_proto_rev(self.m_proto_rev)
# python-optimization: checking verbose causes extra overhead
if verbose:
for msg in msgs:
if msg:
logger.debug(
"%s sending type=%s with str=%s id=%s seq_num=%s value=%s",
self.m_stream.sock_type,
msgtype_str(msg.type),
msg.str,
msg.id,
msg.seq_num_uid,
msg.value,
)
Message.write(msg, out, encoder)
else:
for msg in msgs:
if msg:
Message.write(msg, out, encoder)
if not self.m_stream:
break
if not out:
continue
self.m_stream.send(b"".join(out))
del out[:]
# if verbose:
# logger.debug('send %s bytes', encoder.size())
except IOError as e:
# connection died probably
if not isinstance(e, StreamEOF):
logger.debug("IOError in write thread: %s", e)
except Exception:
logger.warning("Unhandled exception in write thread", exc_info=True)
self.set_state(self.State.kDead)
self.m_active = False
self.m_stream.close() # also kill read thread
with self.m_shutdown_mutex:
self.m_write_shutdown = True
def queueOutgoing(self, msg):
with self.m_pending_mutex:
# Merge with previous. One case we don't combine: delete/assign loop.
msgtype = msg.type
if msgtype in [kEntryAssign, kEntryUpdate]:
# don't do this for unassigned id's
msg_id = msg.id
if msg_id == 0xFFFF:
self.m_pending_outgoing.append(msg)
return
mpend = self.m_pending_update.get(msg_id)
if mpend is not None and mpend.first != 0:
# overwrite the previous one for this id
oldidx = mpend.first - 1
oldmsg = self.m_pending_outgoing[oldidx]
if (
oldmsg
and oldmsg.type == kEntryAssign
and msgtype == kEntryUpdate
):
# need to update assignment with seq_num and value
oldmsg = Message.entryAssign(
oldmsg.str, msg_id, msg.seq_num_uid, msg.value, oldmsg.flags
)
else:
oldmsg = msg # easy update
self.m_pending_outgoing[oldidx] = oldmsg
else:
# new, remember it
pos = len(self.m_pending_outgoing)
self.m_pending_outgoing.append(msg)
self.m_pending_update[msg_id] = Pair(pos + 1, 0)
elif msgtype == kEntryDelete:
# don't do this for unassigned id's
msg_id = msg.id
if msg_id == 0xFFFF:
self.m_pending_outgoing.append(msg)
return
# clear previous updates
mpend = self.m_pending_update.get(msg_id)
if mpend is not None:
if mpend.first != 0:
self.m_pending_outgoing[mpend.first - 1] = None
if mpend.second != 0:
self.m_pending_outgoing[mpend.second - 1] = None
self.m_pending_update[msg_id] = _empty_pair
# add deletion
self.m_pending_outgoing.append(msg)
elif msgtype == kFlagsUpdate:
# don't do this for unassigned id's
msg_id = msg.id
if id == 0xFFFF:
self.m_pending_outgoing.append(msg)
return
mpend = self.m_pending_update.get(msg_id)
if mpend is not None and mpend.second != 0:
# overwrite the previous one for this id
self.m_pending_outgoing[mpend.second - 1] = msg
else:
# new, remember it
pos = len(self.m_pending_outgoing)
self.m_pending_outgoing.append(msg)
self.m_pending_update[msg_id] = Pair(0, pos + 1)
elif msgtype == kClearEntries:
# knock out all previous assigns/updates!
for i, m in enumerate(self.m_pending_outgoing):
if not m:
continue
t = m.type
if t in [
kEntryAssign,
kEntryUpdate,
kFlagsUpdate,
kEntryDelete,
kClearEntries,
]:
self.m_pending_outgoing[i] = None
self.m_pending_update.clear()
self.m_pending_outgoing.append(msg)
else:
self.m_pending_outgoing.append(msg)
def postOutgoing(self, keep_alive):
with self.m_pending_mutex:
# optimization: don't call monotonic unless needed
# now = monotonic()
if not self.m_pending_outgoing:
if not keep_alive:
return
# send keep-alives once a second (if no other messages have been sent)
now = monotonic()
if (now - self.m_last_post) < 1.0:
return
self.m_outgoing.put((Message.keepAlive(),))
else:
now = monotonic()
self.m_outgoing.put(self.m_pending_outgoing)
self.m_pending_outgoing = []
self.m_pending_update.clear()
self.m_last_post = now
