def do_start_proxy():
log("do_start_proxy()")
message_queue = MQueue()
try:
ioe = client_proto.wait_for_io_threads_exit(5+self._socket_timeout)
if not ioe:
log.error("Error: some network IO threads have failed to terminate")
return
client_conn.set_active(True)
process = ProxyInstanceProcess(uid, gid, env_options, session_options, self._socket_dir,
self.video_encoders, self.csc_modules,
client_conn, disp_desc, client_state, cipher, encryption_key, server_conn, c, message_queue)
log("starting %s from pid=%s", process, os.getpid())
self.processes[process] = (display, message_queue)
process.start()
log("process started")
popen = process._popen
assert popen
#when this process dies, run reap to update our list of proxy processes:
self.child_reaper.add_process(popen, "xpra-proxy-%s" % display, "xpra-proxy-instance", True, True, self.reap)
finally:
#now we can close our handle on the connection:
client_conn.close()
server_conn.close()
message_queue.put("socket-handover-complete")
def do_start_proxy():
log("do_start_proxy()")
try:
#stop IO in proxy:
#(it may take up to _socket_timeout until the thread exits)
client_conn.set_active(False)
ioe = client_proto.wait_for_io_threads_exit(
0.1 + self._socket_timeout)
if not ioe:
log.error("IO threads have failed to terminate!")
return
#now we can go back to using blocking sockets:
self.set_socket_timeout(client_conn, None)
client_conn.set_active(True)
assert uid != 0 and gid != 0
message_queue = MQueue()
process = ProxyInstanceProcess(
uid, gid, env_options, session_options, self._socket_dir,
self.video_encoders, self.csc_modules, client_conn,
client_state, cipher, encryption_key, server_conn, c,
message_queue)
log("starting %s from pid=%s", process, os.getpid())
process.start()
log("process started")
#FIXME: remove processes that have terminated
self.processes[process] = (display, message_queue)
finally:
#now we can close our handle on the connection:
client_conn.close()
server_conn.close()
def evolve_p_s(settings: EnhanceModelSettings, optimal_cost, cum_cost, ev_set:EvolveSettings, secondaries=None):
cons = []
returnq = MQueue()
new_set = EnhanceModelSettings(item_store=GearItemStore())
new_set.set_state_json(settings.get_state_json())
if secondaries is not None:
new_set[new_set.P_FAIL_STACKER_SECONDARY] = secondaries
new_set[settings.P_R_FAIL_STACKERS] = []
new_set[settings.P_FAIL_STACKERS] = []
new_set[settings.P_R_STACKER_SECONDARY] = []
new_set[settings.P_ENHANCE_ME] = []
new_set[settings.P_R_ENHANCE_ME] = []
new_set[settings.P_R_FOR_PROFIT] = []
new_set[settings.P_R_FOR_PROFIT] = []
new_set[settings.P_ALTS] = []
new_set[settings.P_NADERR_BAND] = []
new_set[settings.P_VALKS] = {}
new_primaries = [None]*len(optimal_cost)
num_proc = ev_set.num_procs
for i in range(num_proc):
cons.append(MPipe(False))
procs = []
for i in range(num_proc):
procs.append(Process(target=evolve_multi_process_landing, args=(cons[i-1][0], cons[i][1], returnq, new_set.get_state_json(), new_primaries, optimal_cost, cum_cost, ev_set)))
return returnq, procs
def start_proxy_process():
log("start_proxy_process()")
message_queue = MQueue()
client_conn = None
try:
#no other packets should be arriving until the proxy instance responds to the initial hello packet
def unexpected_packet(packet):
if packet:
log.warn("Warning: received an unexpected packet")
log.warn(" from the proxy connection %s:",
client_proto)
log.warn(" %s", repr_ellipsized(packet))
client_proto.close()
client_conn = client_proto.steal_connection(unexpected_packet)
client_state = client_proto.save_state()
log("start_proxy_process(..) client connection=%s",
client_conn)
log("start_proxy_process(..) client state=%s", client_state)
ioe = client_proto.wait_for_io_threads_exit(
5 + self._socket_timeout)
if not ioe:
log.error(
"Error: some network IO threads have failed to terminate"
)
client_proto.close()
return
client_conn.set_active(True)
from xpra.server.proxy.proxy_instance_process import ProxyInstanceProcess
process = ProxyInstanceProcess(
uid, gid, env_options, session_options, self._socket_dir,
self.video_encoders, self.pings, client_conn, disp_desc,
client_state, cipher, encryption_key, server_conn, c,
message_queue)
log("starting %s from pid=%s", process, os.getpid())
self.instances[process] = (True, display, message_queue)
process.start()
log("ProxyInstanceProcess started")
popen = process._popen
assert popen
#when this process dies, run reap to update our list of proxy instances:
self.child_reaper.add_process(popen, "xpra-proxy-%s" % display,
"xpra-proxy-instance", True,
True, self.reap)
except Exception as e:
log("start_proxy_process() failed", exc_info=True)
log.error("Error starting proxy instance process:")
log.error(" %s", e)
message_queue.put("error: %s" % e)
message_queue.put("stop")
finally:
#now we can close our handle on the connection:
log("handover complete: closing connection from proxy server")
if client_conn:
client_conn.close()
server_conn.close()
log("sending socket-handover-complete")
message_queue.put("socket-handover-complete")
def __init__(self, queue=None):
super(PygameThread, self).__init__()
self.window = pygame.display.set_mode((WINDOW_WIDTH, WINDOW_HEIGHT))
self.clock = pygame.time.Clock()
self.lock = Lock()
self._queue = queue
if not queue:
self._queue = MQueue(maxsize=2)
self.sprites = []
self.spriteGroup = pygame.sprite.Group()
def __init__(self,
proc_count: Optional[int] = None,
address: Optional[bytes] = None,
id_generator: Optional[Callable[[], Iterator[Union[
str, int]]]] = lambda: count()):
self.log = logger.new()
self.job_data = MinerJob()
self.signal = Value('B')
self.queue = MQueue()
self.proc_count = proc_count
self.job = {}
self.miners = []
self.loop = None
self.address = address
self._iter_id = id_generator and id_generator() or None
def __init__(self, cls, *args, **kwargs):
super(New_Process_Actor, self).__init__()
self.cls = cls
self.args = list(args)
self.kwargs = kwargs
self.mqueue = MQueue()
self.mevent = MEvent()
if 'input_channel' not in kwargs:
kwargs['input_channel'] = self.args[0]
chan = kwargs['input_channel']
kwargs['input_channel'] = self.mqueue
print 'chan: ', chan
self.c2p = Channel2Process(chan, self.mevent, self.mqueue)
self.c2p.start()
def do_start_proxy():
log("do_start_proxy()")
message_queue = MQueue()
try:
ioe = client_proto.wait_for_io_threads_exit(5+self._socket_timeout)
if not ioe:
log.error("some network IO threads have failed to terminate!")
return
client_conn.set_active(True)
assert uid!=0 and gid!=0
process = ProxyInstanceProcess(uid, gid, env_options, session_options, self._socket_dir,
self.video_encoders, self.csc_modules,
client_conn, client_state, cipher, encryption_key, server_conn, c, message_queue)
log("starting %s from pid=%s", process, os.getpid())
self.processes[process] = (display, message_queue)
process.start()
log("process started")
finally:
#now we can close our handle on the connection:
client_conn.close()
server_conn.close()
message_queue.put("socket-handover-complete")
def __init__(self, max_size):
super().__init__(max_size)
self.queue = MQueue(maxsize=max_size)
import requests
from lxml import etree
from queue import Queue as TQueue
from multiprocessing import Process
from multiprocessing import JoinableQueue as MQueue
import urllib.parse as urlparse
from io import BytesIO
import threading
import time
import re
import multiprocessing
import os
url_queue = MQueue()
html_queue = MQueue()
class HTMLProcessor(Process):
def __init__(self, inputQue, outputQue, host, release, proc_id, **kwargs):
super(HTMLProcessor, self).__init__()
self.inputQue = inputQue
self.outputQue = outputQue
self.host = host
self.release = release
self.kwargs = kwargs
self.parser = etree.HTMLParser()
self.base = None
self.proc_id = 'Process%s' % proc_id
self.filename = 'proc%s_log.txt' % proc_id
frame = mqframes.get_nowait()
suit.blackout()
for pixels in frame:
strip.setPixelColor(pixels[0], pixels[1])
strip.show()
if __name__ == '__main__':
s = Suit()
connr, conns = Pipe()
strip = s.strip
strip.begin()
mqtrigger = MQueue(maxsize=3)
mqanimation = MQueue()
mqframes = MQueue()
pa = Process(target=getFreqInfo, args=(s, mqtrigger))
pb = Process(target=buildAnimations, args=(mqtrigger, mqanimation, connr))
p2 = Process(target=playAnimations, args=(s, mqanimation, mqframes))
pc = Process(target=ampCheck, args=(conns, ))
pf = Process(target=playFrames, args=(s, mqframes))
pa.start()
pb.start()
p2.start()
pc.start()
pf.start()
pa.join()
pb.join()
p2.join()
# queue.Queue – Locking Semantics for Parallel Computing
q = Queue()
q.put('eat')
q.put('sleep')
q.put('code')
print(q)
print(q.get())
print(q.get())
print(q.get())
try:
print(q.get_nowait())
except Exception as e:
print(e)
try:
print(q.get())
except Exception as e:
print(e)
# multiprocessing.Queue – Shared Job Queues
q = MQueue()
q.put('eat')
q.put('sleep')
q.put('code')
print(q)
print(q.get())
print(q.get())
print(q.get())
print(q.get())
def main(*args, **kwargs):
global key
global thread
global DownloadThread
keyword = args
key = keyword[0]
mod = kwargs['mod']
thread = int(kwargs['thread'])
DownloadThread = int(kwargs['downloadthread'])
manage = Manager()
manage2 = ManagerCookie()
# cookiepool = utils.CookiePool('tianyanchaCookie')
# lock = Lock()
# listinfolock = MLock()
# downlock = MLock()
cookiepool = manage2.CookiePool('tianyanchaCookie')
listinfolock = manage.Lock()
rlk_list = {}
for i in cookiepool.getWeightData()['tianyanchaCookieDict_Spider']:
rlk_list[i] = RLock()
logger.info(
"[*] thead %s | DownloadThead %s | DataThread %s" %
(kwargs['thread'], kwargs['downloadthread'], kwargs['datathread']))
logger.info("[*] 正在进行基线检测")
if settings.RD.exists('tianyanchaDownCookie'):
getAllPages(search_url.format(page=1, key=key),
cookiepool=cookiepool,
user='******')
else:
get_cookie(mod, user='******', browsermod=False)
for k in ['Spider', 'List']:
if cookiepool.exists(k):
getAllPages(search_url.format(page=1, key=key),
cookiepool=cookiepool,
user=k)
else:
get_cookie(mod, browsermod=False)
thList = []
logger.info("[*] 开始收集信息")
listQue = MQueue(1000)
filterQue = MQueue(10)
ThreadFlag = Value("d", 1)
# for i in range(thread):
# i = threading.Thread(target=
# , args=('Spider', listQue, rlk_list, cookiepool, ThreadFlag))
# i.start()
# thList.append(i)
p = Process(target=SpiderURL, args=(filterQue, ))
p.start()
thList.append(p)
time.sleep(2)
logger.info("[*] 开始爬取信息")
for i in range(DownloadThread):
x = Process(target=SpiderThreadPages,
args=('List', filterQue, listQue, listinfolock, cookiepool,
ThreadFlag))
x.start()
thList.append(x)
for i in thList:
i.join()
logger.info('Waiting for all subprocesses done...')
def runner(event_stopper):
try:
with open('config.json', 'r') as f:
config = json.load(f)
except Exception as err:
logger.exception(err)
event_stopper.set()
return
# start another process to render the graphical representation
# of what the camera sees
queue = MQueue(5)
video_process = Process(target=video_worker, args=(queue, event_stopper))
video_process.start()
try:
# initialize devices
pixy = pixy2.Pixy2I2C(address=0x54)
gopigo3 = EasyGoPiGo3()
logger.info('initialized pixy2 and gopigo3')
# turn on the built-in LED for better visibility
pixy.set_lamp(1)
logger.info('turn on pixy2 lamp')
# set it to follow white lanes
pixy.set_mode(pixy2.LINE_MODE_WHITE_LINE)
logger.info('set pixy2 line mode on white')
# get the number of frames per second
fps = pixy.get_fps()
period = 1 / fps
logger.info('pixy2 running at {} fps'.format(fps))
# camera resolution
width, height = pixy.get_resolution()
logger.info('pixy2 camera width={} height={}'.format(width, height))
# keep previous measurements for smoothing out the data
previous_angle = 0
previous_magnitude = 0
previous_ema_angle = 0
previous_ema_magnitude = 0
frame_index = 0
alfa = config['alfa']
yotta = config['yotta'] # magnitude importance
logger.info('alfa (exp. moving average factor) = {:3.2f}'.format(alfa))
logger.info('yotta (magnitude importance) = {:3.2f}'.format(yotta))
# the 2 PIDs
c1 = PID(Kp=config['pid1']['Kp'],
Ki=config['pid1']['Ki'],
Kd=config['pid1']['Kd'],
previous_error=0.0,
integral_area=0.0)
c2 = PID(Kp=config['pid2']['Kp'],
Ki=config['pid2']['Ki'],
Kd=config['pid2']['Kd'],
previous_error=0.0,
integral_area=0.0)
pid_switch_point = config['pid-switch-point']
logger.info('PID 1 = {}'.format(str(c1)))
logger.info('PID 2 = {}'.format(str(c2)))
logger.info('PID switchpoint = {}'.format(pid_switch_point))
set_point = 0
max_motor_speed = config['max-motor-speed']
logger.info('max motor speed = {}'.format(max_motor_speed))
while not event_stopper.is_set():
# track each step's time
start = time()
# get the main features from the pixy2
features = pixy.get_main_features(features=1)
frame = None
if features:
if 'vectors' in features:
frame = features['vectors']
else:
frame = []
frame_index += 1
else:
gopigo3.stop()
sleep(period)
continue
# calculate the heading of the lane and detect how long the lane is
lanes, win_size, roi = detect_lanes_from_vector_frame(
frame, height, width)
angle, magnitude = calculate_direction_and_magnitude(
lanes, win_size, previous_angle, previous_magnitude)
# run the determined heading and magnitude through an exponential moving average function
# this has the effect of smoothing the results and of ignoring the short term noise
ema_angle = exponential_moving_average(angle, previous_ema_angle,
alfa)
ema_magnitude = exponential_moving_average(magnitude,
previous_ema_magnitude,
alfa)
# save the measurements for the next iteration
previous_angle = angle
previous_magnitude = magnitude
previous_ema_angle = ema_angle
previous_ema_magnitude = ema_magnitude
angle = ema_angle
magnitude = ema_magnitude
# push the processed information to the graphical renderer (on a separate process)
if not queue.full():
queue.put({
'frame_idx': frame_index,
'frame': frame,
'resolution': (height, width),
'fps': fps,
'angle': angle,
'magnitude': magnitude,
'lanes': lanes,
'win_size': win_size
})
# run the pid controller
error = angle - set_point
c1.integral_area += error
c2.integral_area += error
pid1 = True
if magnitude < pid_switch_point:
c = c1
c2.integral_area = 0.0
else:
c = c2
c1.integral_area = 0.0
pid1 = False
# calculate the correction
correction = c.Kp * error + c.Ki * c.integral_area + c.Kd * (
error - c.previous_error)
c1.previous_error = error
c2.previous_error = error
# determine motor speeds
left_motor_speed = int((
(1 - yotta) + yotta * magnitude / 100) * max_motor_speed +
correction)
right_motor_speed = int((
(1 - yotta) + yotta * magnitude / 100) * max_motor_speed -
correction)
logger.debug(
'using PID {} | angle: {:3d} magnitude: {:3d} | left speed: {:3d} right speed: {:3d}'
.format(1 if pid1 else 0, int(angle), int(magnitude),
left_motor_speed, right_motor_speed))
# actuate the motors
gopigo3.set_motor_dps(gopigo3.MOTOR_LEFT, dps=left_motor_speed)
gopigo3.set_motor_dps(gopigo3.MOTOR_RIGHT, dps=right_motor_speed)
# make it run at a specific loop frequency
# equal to camera's fps
end = time()
diff = end - start
remaining_time = period - diff
if remaining_time > 0:
sleep(remaining_time)
# and turn the built-in LED off when the program ends
# and stop the gopigo3
pixy.set_lamp(0)
gopigo3.stop()
except Exception as e:
event_stopper.set()
logger.exception(e)
finally:
video_process.join()
