def checkmultiprocess(ipqueue,cacheResult):
if ipqueue.qsize() == 0:
return
processlist = []
"如果ip数小于512，只使用一个子进程，否则则使用指定进程数，每个进程处理平均值的数量ip"
max_threads = g_maxthreads
maxprocess = g_useprocess
if ipqueue.qsize() < g_maxthreads:
max_threads = ipqueue.qsize()
maxprocess = 1
else:
max_threads = (ipqueue.qsize() + g_useprocess) / g_useprocess
if max_threads > g_maxthreads:
max_threads = g_maxthreads
#multiprocessing.log_to_stderr(logging.DEBUG)
for i in xrange(0,maxprocess):
p = Process(target=callsingleprocess,args=(ipqueue,cacheResult,max_threads))
p.daemon = True
processlist.append(p)
p.start()
try:
for p in processlist:
p.join()
except KeyboardInterrupt:
PRINT("need wait all process end...")
for p in processlist:
if p.is_alive():
p.terminate()
def build_rec_multi(options, familylist):
'''
Run "build_reconciled_tree" on multiple threads.
'''
global q
for fam in familylist:
q.put(fam)
global n
for i in range(options.threads):
p = Process(target=build_reconciled_tree, name='%i' % (i+1), args = (options,))
p.start()
sleep(options.threads*0.05)
q.join()
while n.qsize() > 0:
for _ in range(10):
if n.qsize() > 0:
print n.get(),
print ""
sleep(options.threads*0.1)
if p.is_alive() and q.empty():
sleep(options.threads*0.2)
if p.is_alive() and q.empty():
p.terminate()
return None
def gentree_multi(options, familylist, gene_sp):
'''
Run "pli2tree" on multiple threads.
'''
global q
for fam in familylist:
q.put(fam)
global n
print "# Building tree from multiple alignment for the following families: \n"
for i in range(options.threads):
p = Process(target=pli2tree, name='%i' % (i+1), args = (options, gene_sp))
p.start()
sleep(options.threads*0.05)
q.join()
while n.qsize() > 0:
for _ in range(10):
if n.qsize() > 0:
print n.get(),
print ""
sleep(options.threads*0.05)
if p.is_alive() and q.empty():
sleep(options.threads*0.2)
if p.is_alive() and q.empty():
p.terminate()
return None
def main():
"""
Creates instances of the above methods and occassionally checks for crashed
worker processes & relaunches.
"""
worker_process = list()
get_update_process = Process(target=get_updates)
get_update_process.start()
for i in range(0, int(CONFIG['BOT_CONFIG']['workers'])):
worker_process.append(Process(target=process_updates))
worker_process[i].start()
time_worker = Process(target=check_time_args)
time_worker.start()
while RUNNING.value:
time.sleep(30)
for index, worker in enumerate(worker_process):
if not worker.is_alive():
del worker_process[index]
worker_process.append(Process(target=process_updates))
worker_process[-1].start()
if not time_worker.is_alive():
time_worker = Process(target=check_time_args)
time_worker.start()
if not get_update_process.is_alive():
get_update_process = Process(target=get_updates)
get_update_process.start()
get_update_process.join()
time_worker.join()
for worker in worker_process:
worker.join()
def runFunction(self, func, args, timeout=3600):
def target(func, args, retQ):
ret= func(*args)
retQ.put(ret)
retQ = Queue()
process = Process(target=target, args=(func, args, retQ))
process.start()
try:
ret = retQ.get(block=True, timeout=timeout)
except Empty:
ret = (-1, "function timeout, killed")
try:
if process.is_alive():
process.terminate()
process.join(2)
if process.is_alive():
os.kill(int(process.pid), signal.SIGKILL)
process.join(2)
except:
if process.is_alive():
try:
os.kill(int(process.pid), signal.SIGKILL)
except:
pass
process.join(2)
return ret
def main(configs, timeout=10 * 60):
makedirs('data/results', exist_ok=True)
for config in configs:
outpath = resultpath(config)
if path.isfile(outpath):
continue
out_queue = Queue()
def worker():
result = None
try:
result = run_classification(config)
except BaseException as exc:
print(traceback.format_exc(), file=stderr)
result = {'error': repr(exc)}
out_queue.put(result)
p = Process(target=worker)
try:
p.start()
p.join(timeout)
if not p.is_alive():
result = out_queue.get()
else:
result = {'error': 'timed out'}
out_obj = { 'config': config \
, 'result': result }
with gzip.open(outpath, mode='xt') as f:
json.dump(utils.namedtuples_replaced(out_obj), f, cls=NamedtupleJSONEncoder, indent=4)
finally:
if p.is_alive():
p.terminate()
def run(self):
timeout = None
try:
timeout = self.timeout
except:
pass
sleeptime = 5
p = Process(target=self.run_)
p.start()
if not timeout:
p.join()
status = 'normal'
else:
total_sleep = 0
while total_sleep < timeout:
time.sleep(sleeptime)
total_sleep = total_sleep + sleeptime
if not p.is_alive():
p.join()
status = 'normal'
break
if p.is_alive():
p.terminate()
status = 'timeout'
else:
p.join()
status = 'normal'
return status
def mixer_and_priority_synthesis(s_ast, synrkt, bugs, mutators, score):
queue = Queue()
mixer = Process(target=mixer_synthesis, args=(s_ast, synrkt, bugs,
mutator, True, queue))
priority = Process(target=priority_synthesis, args=(s_ast, synrkt, bugs,
mutator, score, True, queue))
mixer.start()
priority.start()
while True:
if not queue.empty():
# terminate all processes
if mixer.is_alive():
mixer.terminate()
if priority.is_alive():
priority.terminate()
# display results
(fixes, synthesizer) = queue.get()
if fixes:
for fix in fixes:
print "At line " + str(fix.lineno) + " and offset " + str(fix.col_offset)
print "\t " + SourceVisitor().visit(fix)
else:
print "No solution found!"
print "Winner: " + synthesizer
break
time.sleep(1)
def create_tables_force(engine, delay, retries):
"""Create the tables and **KILL ANY BLOCKING PROCESSES**.
This command will spawn a process to create the new tables in
order to find out which process is blocking us. If we didn't do
this concurrently, then the table creation will have disappeared
by the time we tried to find its blocker in the postgres backend
tables.
"""
logger.info('Running table creator named %s', app_name)
logger.warning('Running with force=True option %s', app_name)
from multiprocessing import Process
p = Process(target=create_graph_tables, args=(engine, delay))
p.start()
time.sleep(delay)
if p.is_alive():
logger.warning('Table creation blocked!')
kill_blocking_psql_backend_processes(engine)
# Wait some time for table creation to proceed
time.sleep(4)
if p.is_alive():
if retries <= 0:
raise RuntimeError('Max retries exceeded.')
logger.warning('Table creation failed, retrying.')
return create_tables_force(engine, delay, retries-1)
def collect(self, targetdir, timestamp, delta):
from logging import root
from multiprocessing import Process
from os import getpid
from .. import multiprocessing_logger
# We want to copy the files in a child process, so in case the filesystem is stuck, we won't get stuck too
kwargs = dict(targetdir=targetdir, timestamp=timestamp, delta=delta)
try:
[logfile_path] = [
handler.target.baseFilename for handler in root.handlers if self._is_my_kind_of_logging_handler(handler)
] or [None]
except ValueError:
logfile_path = None
subprocess = Process(
target=multiprocessing_logger,
args=(logfile_path, getpid(), Windows_Event_Logs.collect_process),
kwargs=kwargs,
)
subprocess.start()
subprocess.join(self.timeout_in_seconds)
if subprocess.is_alive():
msg = "Did not finish collecting {!r} within the {} seconds timeout_in_seconds"
logger.error(msg.format(self, self.timeout_in_seconds))
subprocess.terminate()
if subprocess.is_alive():
logger.info("Subprocess {!r} terminated".format(subprocess))
else:
logger.error("Subprocess {!r} is stuck".format(subprocess))
raise TimeoutError()
elif subprocess.exitcode:
logger.error("Subprocess {!r} returned non-zero exit code".format(subprocess))
raise RuntimeError(subprocess.exitcode)
class LiveviewServerProtocol(WebSocketServerProtocol):
def __init__(self, endpoint_url):
print("LiveviewServerProtocol init.")
self.endpoint_url = endpoint_url
def onConnect(self, request):
print("Client connecting: {0}".format(request.peer))
self.queue = Queue()
self.process = Process(target=liveview_main, args=(self.endpoint_url, self.queue))
self.process.start()
# @asyncio.coroutine
def onOpen(self):
while True:
self.sendMessage("UNCHI".encode("utf-8"), False)
time.sleep(0.001)
print("WebSocket connection open.")
if self.process and self.process.is_alive():
while True:
if not self.queue.empty():
payload = self.queue.get()
print(payload)
self.sendMessage(payload)
# yield from asyncio.sleep(0.001)
def onClose(self, wasClean, code, reason):
print("WebSocket connection closed: {0}".format(reason))
if self.process and self.process.is_alive():
self.process.terminate()
# this is hack that enables dynamic protocol initialization
def __call__(self):
return self
def run(self, output_prefix, run_dict, cmd_dict):
self.set_params(run_dict)
self.cmd_dict = cmd_dict
timeout = None
try:
timeout = self.timeout
except:
pass
sleeptime = 5
p = Process(target=self.run_)
p.start()
if not timeout:
p.join()
status = 'normal'
else:
total_sleep = 0
while total_sleep < timeout:
time.sleep(sleeptime)
total_sleep = total_sleep + sleeptime
if not p.is_alive():
p.join()
status = 'normal'
break
if p.is_alive():
p.terminate()
status = 'timeout'
else:
p.join()
status = 'normal'
return status
class UDPPipe(Pipe):
class PipeHandler(SocketServer.BaseRequestHandler):
def handle(self):
data = self.request[0]
print("Got data from " + str(self.client_address) + ": " + str(data))
socket = self.request[1]
socket.sendto(data, (self.server.reader_ip, self.server.reader_port))
def __init__(self, host, port):
self.server = SocketServer.UDPServer((host, port), UDPPipe.PipeHandler)
self.server.writer_ip = None
self.server.reader_ip = None
self.server.reader_port = None
self.server_proc = Process(target=self.server.serve_forever)
@property
def writer_ip(self):
return self.server.writer_ip
@writer_ip.setter
def writer_ip(self, ip):
if self.server_proc.is_alive():
raise Exception('Can not modify Pipe writer_ip when server is working')
self.server.writer_ip = ip
@property
def reader_ip(self):
return self.server.reader_ip
@reader_ip.setter
def reader_ip(self, ip):
if self.server_proc.is_alive():
raise Exception('Can not modify Pipe reader_ip when server is working')
self.server.reader_ip = ip
@property
def reader_port(self):
return self.server.reader_port
@reader_port.setter
def reader_port(self, port):
if self.server_proc.is_alive():
raise Exception("Can not modify Pipe reader_port when server is running")
self.server.reader_port = port
def run(self):
if (self.writer_ip is None) or (self.reader_ip is None):
raise Exception('You must specify writer and reader ip before running Pipe')
self.server_proc.start()
def stop(self):
self.server.shutdown()
self.server.server_close()
self.server_proc.join()
def test_kills_process(self):
p = Process(target=time.sleep, args=(100,))
p.start()
self.assertTrue(p.is_alive())
kill_tree(p.pid)
p.join(1)
self.assertFalse(p.is_alive())
def data_handler(spigot_data: SpigotData, lock: threading.Lock):
"""
Main thread for reading output from spigot IO worker.
Also interprets data.
spigot_data - shared SpigotData object
"""
client, child = Pipe()
t = Process(target=r_w_worker, args=(child, spigot_data.close_event))
t.start()
spigot_data.game.status = SpigotState.RUNNING
running = True
while running:
while t.is_alive():
if client.poll(0.3):
buf = client.recv()
parse_event(spigot_data, buf)
spigot_data.add_message(buf)
logging.debug('<<OUTPUT>> {}'.format(buf))
while not spigot_data.commands.empty():
command = spigot_data.commands.get()
client.send(command)
logging.debug('<<COMMAND>> {}'.format(command))
# After java process is dead
spigot_data.status = SpigotState.STOPPED
spigot_data.add_message(info_message("""The server has stopped. Type 'start' to start. """
"""Type 'quit' to close Spigot Monitor""")) # PEP8ers gonna hate
spigot_data.game.players = {} # No players are available on a stopped server...
while True and not AUTO_RESTART:
command = spigot_data.commands.get().strip() # strip because commands have newline appended
if command.lower() == 'start':
t = Process(target=r_w_worker, args=(child, spigot_data.close_event))
t.start()
logging.debug('Thread created.')
break
elif command.lower() == 'quit' or command.lower() == 'stop':
message = info_message("KTHXBAI")
spigot_data.add_message(message)
logging.debug('Quitting program.')
break
if AUTO_RESTART:
t = Process(target=r_w_worker, args=(child, spigot_data.close_event))
t.start()
logging.debug('Thread created.')
if not t.is_alive(): # thread hasn't started again
running = False
print('exiting data_handler loop')
class AlppacaIntegrationTest(object):
def __init__(self, config):
self.config = config
self.mock_job = Process(target=self.run_api_server_mock)
self.alppaca_job = Process(target=self.run_alppaca)
def __enter__(self):
self.mock_job.start()
# Ensure the mock IMS fully(!) up and running before starting Alppaca.
# Otherwise, Alppaca will see the failure and start its backoff
# behaviour.
time.sleep(0.5)
self.alppaca_job.start()
time.sleep(0.5)
return self
def __exit__(self, *args):
self.mock_job.terminate()
self.alppaca_job.terminate()
self.mock_job.join(10)
self.alppaca_job.join(10)
mock_alive = self.mock_job.is_alive()
if mock_alive:
os.kill(self.mock_job.pid, signal.SIGKILL)
alppaca_alive = self.alppaca_job.is_alive()
if alppaca_alive:
os.kill(self.alppaca_job.pid, signal.SIGKILL)
if mock_alive or alppaca_alive:
raise Exception("Processe(s) that ignored SIGTERM: "
"API mock server: %s Alppaca: %s" % (
mock_alive, alppaca_alive))
def run_alppaca(self):
daemon = AlppacaDaemon(pid_file="not used")
daemon.config = self.config
daemon.setup_logging()
daemon.run()
def run_api_server_mock(self):
MockIms().run()
def test_alppaca_returns_given_role(self):
url = 'http://{host}:{port}/latest/meta-data/iam/security-credentials/'.format(
host=self.config['bind_ip'], port=self.config['bind_port'])
response = requests.get(url)
assert response.status_code == 200, \
"Response status code should be 200, was: '{0}'".format(response.status_code)
assert(response.text == 'test_role'), \
"Response text should be 'test_role', was: '{0}'".format(response.text)
def execute(self):
self.test_alppaca_returns_given_role()
def MakeTiles():
### STAGE 1 ###
for kapPath in Resources.lstBsbFiles:
if pleaseContinue:
#we need to wait for this process to finish before moving on
kapToVrtProc = Process(target=KapToVrt, args=(kapPath,))
kapToVrtProc.start()
while kapToVrtProc.is_alive():
sleep(.1)
vrtPath = kapPath[0:-4] + ".vrt"
#print vrtPath
if os.path.isfile(vrtPath):
zxyFullPath = Resources.getTempDir() + "/" + kapPath.split("/")[-1][0:-4] + ".zxy/"
tileError = True
try:
vrtToTilesProc = Process(target=VrtToTiles, args=(Resources.getTempDir(), vrtPath, FindZoom.getKapZoom(kapPath),))
vrtToTilesProc.start()
while vrtToTilesProc.is_alive():
sleep(.1)
if os.path.isdir(zxyFullPath):
# if len(os.listdir(zxyFullPath)) > 0:
# tileError = False
for subDir in os.listdir(zxyFullPath):
if os.path.isdir(zxyFullPath + subDir):
tileError = False
except:
pass
print "VrtToTiles failed!"
if tileError:
print "ERROR gdal_tiler failed processing chart: ", kapPath.split("/")[-1]
Resources.lstBsbErrorFiles.append(kapPath.split("/")[-1][0:-4])
###
os.remove(vrtPath) #clean up vrt files
else:
###map2gdal failed because there is no vrt file...
print vrtPath
print "ERROR map2gdal failed processing chart: ", kapPath.split("/")[-1]
Resources.lstBsbErrorFiles.append(kapPath.split("/")[-1][0:-4])
# mp = Process(target=TileMap, args=(kapPath, parent.currentChart, resourcePipeB, signalPipeA))
# mp.start()
#Event is bound to Step4.nextStage(self, event)
wx.PostEvent(app, InThreadEvent(advance=False))
#posting this event signals to parent window that the task is finished
if pleaseContinue:
#Event is bound to Step4.nextStage(self, event)
wx.PostEvent(app, InThreadEvent(advance=True))
def test_does_not_fail(self):
p = Process(target=child_process)
p.start()
self.assertTrue(p.is_alive())
time.sleep(.1)
kill_tree(p.pid)
p.join(1)
self.assertFalse(p.is_alive())
def main(argv):
#specify host name and port number on the command line.
TCP_IP = sys.argv[1]
TCP_PORT = int(sys.argv[2])
BUFFER_SIZE = 5000
#--connect to server--#
#create sock stream and connect
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect((TCP_IP, TCP_PORT))
s.settimeout(.2)
name = getUsersHandle()
s.sendall(networkKeys.ACTIONS.name + name + "\n")
############################################################
#### Start curses process #################################
############################################################
cursesEnd, networkEnd = Pipe(duplex=True)
cursesProcess = Process(target=cursesIO.cursesEngine, args=(cursesEnd,))
cursesProcess.start()
while cursesProcess.is_alive():
#get messages from curses
if networkEnd.poll():
msg = networkEnd.recv()
log("(NET MSG-FROM-CURSES):" + str(type(msg)) + str(msg) + "\n")
#send curses message to server
s.sendall(msg + "\n")
#get update from server
try:
updateFromServer = s.recv(BUFFER_SIZE)
updateFromServer = updateFromServer.decode()
log("(NET MSG-TO-CURSES):" + updateFromServer + "\n")
if not updateFromServer:
networkEnd.send("/DISCONNECTED FROM SERVER\n")
break
#send network update to cureses
networkEnd.send(updateFromServer)
except socket.timeout:
pass
while cursesProcess.is_alive():
pass
############################################################
#### curses process has ended ##############################
############################################################
#send termination message s.send(MESSAGE.encode())
log("(NET): closing connection")
#close connection
s.close()
def run(self):
manager = Manager()
joystickState = manager.dict()
joystickStateNum = Value('d', 0.0)
acousticStateNum = Value('d', 0.0)
heartbeatValue = Array('i', [0]*3)
acousticProcess = Process(target=self.acousticPlayer, args=(joystickStateNum, acousticStateNum))
joystickReaderProcess = Process(target=self.joystickReader, args=(joystickState, joystickStateNum))
joystickUpdaterProcess = Process(target=self.joystickUpdater, args=(joystickState, joystickStateNum, acousticStateNum, heartbeatValue))
try:
# Start the processes
joystickReaderProcess.start()
joystickUpdaterProcess.start()
acousticProcess.start()
# control heartbeat
maxBrightness = 50
increment = 1
increasing = True
brightness = 0
while True:
# If all processes are running, show blue/green heartbeat
if acousticProcess.is_alive() and joystickReaderProcess.is_alive() and joystickUpdaterProcess.is_alive():
heartbeatValue[0] = 0
heartbeatValue[1] = maxBrightness-brightness
heartbeatValue[2] = brightness
# Otherwise show red/orange heartbeat
else:
heartbeatValue[0] = 25
heartbeatValue[1] = 0
heartbeatValue[2] = brightness/2
# if joystick updater is running, then it will set the LEDs
if not joystickUpdaterProcess.is_alive():
self._leds.setPixel(0, *heartbeatValue)
self._leds.show()
brightness += increment * (1 if increasing else -1)
if brightness > maxBrightness-2:
increasing = False
if brightness < 2:
increasing = True
sleep(0.05)
except:
pass
# Something went wrong: kill the process
acousticProcess.terminate()
joystickReaderProcess.terminate()
joystickUpdaterProcess.terminate()
# show a red LED then exit
self._leds.clearStrip()
self._leds.setPixel(0, 25, 0, 0)
self._leds.show()
class Breathe(object):
def __init__(self):
GPIO.setmode(GPIO.BCM) # We can also choose a BOARD numbering schemes.
GPIO.setup(21, GPIO.OUT) # set GPIO 21 as output
self.light = GPIO.PWM(21, 100) # create object for PWM on port 21 at 100 Hertz
self.p = Process(target=calm, args=(self.light,))
self.state = breathe_state(('CALM', 'ERRATIC', 'STOP'))
self.restart_state = self.state.CALM
def shutdown(self):
if (self.p.is_alive()):
self.p.terminate()
self.light.stop()
GPIO.cleanup()
print("breathing shut down.")
def restart(self):
print("restarting breathing!")
if (self.p.is_alive()):
self.p.terminate()
self.light.stop()
GPIO.cleanup()
GPIO.setmode(GPIO.BCM)
GPIO.setup(21, GPIO.OUT)
self.light = GPIO.PWM(21, 100)
self.light.start(0)
if (self.restart_state == self.state.CALM):
self.p = Process(target=calm, args=(self.light,))
self.p.start()
elif (self.restart_state == self.state.ERRATIC):
self.p = Process(target=erratic, args=(self.light,))
self.p.start()
else:
self.shutdown()
def calm(self):
if (self.p.is_alive()):
self.p.terminate()
self.light.stop()
self.p = Process(target=calm, args=(self.light,))
self.p.start()
print("calm breathing")
def erratic(self):
if (self.p.is_alive()):
self.p.terminate()
self.light.stop()
self.p = Process(target=erratic, args=(self.light,))
self.p.start()
print("erratic breathing")
def set(self, state):
self.restart_state = state
print("state has been set:", self.restart_state)
def main(main_proc: object, main_cpipe: object):
""" Listen on main_cpipe for signals. Depending on what signal is recieved
it will start new child processes.
"""
window_dict = {}
while main_proc.is_alive():
try:
signal, data = main_cpipe.recv()
except KeyboardInterrupt:
break
if signal == 'quit':
break
if signal == 'refresh':
# Make sure all children exit.
logging.info('\n'.join([f'PID: {t.pid} of {t}' for t in active_children()]))
for pid, proc in window_dict.items():
logging.info(f'PROCESS: {pid}')
if signal == 'new-proc':
proc = Process(target=run_browser, args=(data,))
proc.start()
main_cpipe.send(('proc-pid', proc.pid))
logging.info(f"MAIN_LOOP NEW_PROC: {data}")
window_dict[proc.pid] = proc
logging.info(f"child pid: {proc.pid}")
logging.info(f'window_dict: {window_dict}')
elif signal == 'terminate':
proc = window_dict.pop(data, None)
if proc:
logging.info(f'Joining pid: {data}')
proc.join(1)
if proc.is_alive():
logging.info(f"Terminating: {proc}")
proc.terminate()
logging.info("Quitting")
logging.info(window_dict)
for pid, proc in window_dict.items():
logging.info(f"Joining: {proc}")
proc.join(1)
if proc.is_alive():
logging.info(f"Terminating: {proc}")
proc.terminate()
# Make sure all children exit.
active_children()
return
def pfam_scan_multi(options):
'''
Run pfam_scan to recover the domains/protein families associated to
each sequence.
'''
print "# Running PfamScan over the input sequences...\n"
handle = open(options.outdir+"query.fasta", "r")
n_seq = 0
for line in handle:
if line[0] == ">":
n_seq = n_seq+1
handle.close()
if options.force and os.path.exists(options.outdir+"query_temp.pfam"):
os.remove(options.outdir+"query_temp.pfam")
if n_seq < 100 or options.threads == 1:
os.system("perl "+os.path.abspath(options.stdir).replace(" ","\ ")+"" \
"/pfam_scan.pl -e_dom "+str(options.pcut)+" -e_seq " \
""+str(options.pcut)+" -cpu "+str(options.threads)+" -fasta " \
""+os.path.abspath(options.outdir).replace(" ","\ ")+"/query.fasta"\
" -outfile "+os.path.abspath(options.outdir).replace(" ","\ ")+"" \
"/query_temp0.pfam -d " \
""+os.path.abspath(options.dbdir).replace(" ","\ "))
return 1
elif n_seq >= 100 or options.threads > 1:
i, n_threads = split_query_fasta(options, n_seq)
global q
num_files = i+1
if options.force:
for j in range(num_files):
if os.path.exists(options.outdir+"query_temp"+str(j)+".pfam"):
os.remove(options.outdir+"query_temp"+str(j)+".pfam")
for j in range(num_files):
q.put(j)
sleep(options.threads*0.05)
for j in range(n_threads):
p = Process(target=pfam_scan_mp, name='%i' % (j+1),
args = (options,))
p.start()
sleep(options.threads*0.05)
q.join()
sleep(options.threads*0.05)
if p.is_alive() and q.empty():
sleep(options.threads*0.2)
if p.is_alive() and q.empty():
p.terminate()
return num_files
def calculator_1_process():
a = Process(target=prime_calculator, args=(1, 10000))
start_time = time.time()
a.start()
a_boolen = a.is_alive()
while a_boolen == True:
a_boolen = a.is_alive()
print("--- 1 process took %s seconds --- \n" % (time.time() - start_time))
def __tcp_server_routine(server, verbosity=False):
conn, addr = server.accept()
p = Process(target=__tcp_server_process, args=(conn, addr, verbosity))
# ps -ax | grep main.py
p.start()
# conn.close()
print p, p.is_alive()
time.sleep(2)
print p, p.is_alive()
return True
def access_email_inbox(self):
print("access email functions call begin...")
p = Process(target=self.my_sub_process)
p.start()
print ("I am parent %d" % (os.getpid()))
print("access email functions call end...")
time.sleep(6)
if p.is_alive():
p.terminate()
else:
pass
print ('child process status: %s' % (p.is_alive()))
def calculator_2_process():
a = Process(target=prime_calculator, args=(1, 5000))
b = Process(target=prime_calculator, args=(5000, 10000))
start_time = time.time()
a.start()
b.start()
a_boolen, b_boolen = a.is_alive(), b.is_alive()
while a_boolen == True or b_boolen == True:
a_boolen, b_boolen = a.is_alive(), b.is_alive()
print("--- 2 process took %s seconds --- \n" % (time.time() - start_time))
def bootstrap(self):
print_head()
self.verify_environment()
self.load_config()
self.bootstrap_storage_mixin()
self.bootstrap_plugins()
self.verify_plugin_settings()
puts("Bootstrapping complete.")
puts("\nStarting core processes:")
# Scheduler
scheduler_thread = Process(target=self.bootstrap_scheduler)
# scheduler_thread.daemon = True
# Bottle
bottle_thread = Process(target=self.bootstrap_bottle)
# bottle_thread.daemon = True
# XMPP Listener
xmpp_thread = Process(target=self.bootstrap_xmpp)
# xmpp_thread.daemon = True
with indent(2):
try:
# Start up threads.
xmpp_thread.start()
scheduler_thread.start()
bottle_thread.start()
errors = self.get_startup_errors()
if len(errors) > 0:
default_room = self.get_room_from_name_or_id(settings.DEFAULT_ROOM)["room_id"]
error_message = "FYI, I ran into some problems while starting up:"
for err in errors:
error_message += "\n%s\n" % err
self.send_room_message(default_room, error_message, color="yellow")
puts(colored.red(error_message))
while True:
time.sleep(100)
except (KeyboardInterrupt, SystemExit):
scheduler_thread.terminate()
bottle_thread.terminate()
xmpp_thread.terminate()
print '\n\nReceived keyboard interrupt, quitting threads.',
while (scheduler_thread.is_alive() or
bottle_thread.is_alive() or
xmpp_thread.is_alive()):
sys.stdout.write(".")
sys.stdout.flush()
time.sleep(0.5)
def force_create_graph_tables(engine, delay, retries):
print 'Running table creator named', name
p = Process(target=create_graph_tables, args=(engine, delay*2))
p.start()
time.sleep(1)
if not p.is_alive():
print('Process not blocked.')
return p.join()
else:
print('Process is blocked, waiting {} seconds for unlock'
.format(delay))
time.sleep(delay)
# If p has ended, the block was cleared without intervention
if not p.is_alive():
print('Process unblocked without killing anything.')
return p.join()
# Lookup blocking processes
blocking = list(engine.execute(text(blocking_SQL), name=name))
# Check if any high importance process is blocking us
if is_blocked_by_no_kill(blocking):
if retries <= 0:
raise RuntimeError('Max retries exceeded')
print('Trying again in {} seconds ({} retries remaining)'.format(
delay, retries))
time.sleep(delay)
return force_create_graph_tables(engine, delay, retries-1)
# Kill blocking processes
for proc in blocking:
bd_appname, bd_pid, bing_appname, bing_pid, query = proc
# Skip other table_creators
if bing_appname.startswith(name_root):
continue
# Kill anything in the way, it was deemed of low importance
print('Killing blocking backend process: name({})\tpid({}): {}'
.format(bing_appname, bing_pid, query))
engine.execute(text('select pg_terminate_backend(:bing_pid);'),
bing_pid=bing_pid)
return p.join()
def load_link(browser, link):
''' Return true if load successful, false otherwise. '''
while True:
p = Process(target=browser_get, args=(browser, link))
p.start()
p.join(LOAD_TIME)
if p.is_alive():
p.terminate()
else:
break
while True:
wait_time = READY_TIME
start_time = time.time()
''' Wait for page to have completely loaded. '''
while True:
state = browser.execute_script('return document.readyState;')
if state == 'complete':
return True
if time.time() - start_time > wait_time:
logging.info("Document %s not ready after %ds", link, wait_time)
break
time.sleep(1)
wait_time = wait_time * READY_RATIO
if wait_time > MAX_READY_TIME * READY_RATIO:
logging.error("Skipping document %s. Was never ready.", link)
return False
else:
logging.info("Increasing wait time to %ds", wait_time)
class LightBluePebble(object):
""" a wrapper for LightBlue that provides Serial-style read, write and close"""
def __init__(self, id, should_pair, debug_protocol=False, connection_process_timeout=60):
self.mac_address = id
self.debug_protocol = debug_protocol
self.should_pair = should_pair
manager = multiprocessing.Manager()
self.send_queue = manager.Queue()
self.rec_queue = manager.Queue()
self.bt_teardown = multiprocessing.Event()
self.bt_message_sent = multiprocessing.Event()
self.bt_connected = multiprocessing.Event()
self.bt_socket_proc = Process(target=self.run)
self.bt_socket_proc.daemon = True
self.bt_socket_proc.start()
# wait for a successful connection from child process before returning to main process
self.bt_connected.wait(connection_process_timeout)
if not self.bt_connected.is_set():
raise LightBluePebbleError(id, "Connection timed out, LightBlueProcess was provided %d seconds to complete connecting" % connection_process_timeout)
def write(self, message):
""" send a message to the LightBlue processs"""
try:
self.send_queue.put(message)
self.bt_message_sent.wait()
except:
self.bt_teardown.set()
if self.debug_protocol:
log.debug("LightBlue process has shutdown (queue write)")
def read(self):
""" read a pebble message from the LightBlue processs"""
try:
tup = self.rec_queue.get()
return tup
except Queue.Empty:
return (None, None, None, '')
except:
self.bt_teardown.set()
if self.debug_protocol:
log.debug("LightBlue process has shutdown (queue read)")
return (None, None, None, '')
def close(self):
""" close the LightBlue connection process"""
self.bt_teardown.set()
def is_alive(self):
return self.bt_socket_proc.is_alive()
def run(self):
""" create bluetooth process paired to mac_address, must be run as a process"""
from lightblue import pair, socket as lb_socket, finddevices, selectdevice
def autodetect(self):
list_of_pebbles = list()
if self.mac_address is not None and len(self.mac_address) is 4:
# we have the friendly name, let's get the full mac address
log.warn("Going to get full address for device %s, ensure device is broadcasting." % self.mac_address)
# scan for active devices
devices = finddevices(timeout=8)
for device in devices:
if re.search(r'Pebble ' + self.mac_address, device[1], re.IGNORECASE):
log.debug("Found Pebble: %s @ %s" % (device[1], device[0]))
list_of_pebbles.append(device)
if len(list_of_pebbles) is 1:
return list_of_pebbles[0][0]
else:
raise LightBluePebbleError(self.mac_address, "Failed to find Pebble")
else:
# no pebble id was provided... give them the GUI selector
try:
return selectdevice()[0]
except TypeError:
log.warn("failed to select a device in GUI selector")
self.mac_address = None
# notify that the process has started
log.debug("LightBlue process has started on pid %d" % os.getpid())
# do we need to autodetect?
if self.mac_address is None or len(self.mac_address) is 4:
self.mac_address = autodetect(self)
# create the bluetooth socket from the mac address
if self.should_pair and self.mac_address is not None:
pair(self.mac_address)
try:
self._bts = lb_socket()
self._bts.connect((self.mac_address, 1)) # pebble uses RFCOMM port 1
self._bts.setblocking(False)
except:
raise LightBluePebbleError(self.mac_address, "Failed to connect to Pebble")
# give them the mac address for using in faster connections
log.debug("Connection established to " + self.mac_address)
# Tell our parent that we have a pebble connected now
self.bt_connected.set()
send_data = e = None
while not self.bt_teardown.is_set():
# send anything in the send queue
try:
send_data = self.send_queue.get_nowait()
self._bts.send(send_data)
if self.debug_protocol:
log.debug("LightBlue Send: %r" % send_data)
self.bt_message_sent.set()
except Queue.Empty:
pass
except (IOError, EOFError):
self.bt_teardown.set()
e = "Queue Error while sending data"
# if anything is received relay it back
rec_data = None
try:
rec_data = self._bts.recv(4)
except (socket.timeout, socket.error):
# Exception raised from timing out on nonblocking
pass
if (rec_data is not None) and (len(rec_data) == 4):
# check the Stream Multiplexing Layer message and get the length of the data to read
size, endpoint = unpack("!HH", rec_data)
resp = ''
while len(resp) < size:
try:
resp += self._bts.recv(size-len(resp))
except (socket.timeout, socket.error):
# Exception raised from timing out on nonblocking
# TODO: Should probably have some kind of timeout here
pass
try:
if self.debug_protocol:
log.debug("{}: {} {} ".format(endpoint, resp, rec_data))
self.rec_queue.put(("watch", endpoint, resp, rec_data))
except (IOError, EOFError):
self.BT_TEARDOWN.set()
e = "Queue Error while recieving data"
pass
if self.debug_protocol:
log.debug("LightBlue Read: %r " % resp)
# just let it die silent whenever the parent dies and it throws an EOFERROR
if e is not None and self.debug_protocol:
raise LightBluePebbleError(self.mac_address, "LightBlue polling loop closed due to " + e)
#进程函数的使用
from multiprocessing import Process
from time import sleep
a = 1
def worker(sec, msg):
#当worker作为子进程运行时,对全局量a 的修改只会
#影响在子进程中a的值 ,对父进程没有影响
global a
a = 1000
for i in range(3):
sleep(sec)
print("the worker msg", msg)
print(a)
p = Process(name = 'worker',\
target = worker,args = (2,),\
kwargs = {'msg':'You are a big man'})
p.start()
#进程名称
print("进程名称:", p.name)
print("进程PID:", p.pid)
print('进程状态:', p.is_alive())
p.join()
print('parent:', a)
class Device(object):
JOIN_TIMEOUT = 1.0
SYNC_TX_CHUNK_SIZE = 0
CONTINUOUS_TX_CHUNK_SIZE = 0
DATA_TYPE = np.float32
class Command(Enum):
STOP = 0
SET_FREQUENCY = 1
SET_SAMPLE_RATE = 2
SET_BANDWIDTH = 3
SET_RF_GAIN = 4
SET_IF_GAIN = 5
SET_BB_GAIN = 6
SET_DIRECT_SAMPLING_MODE = 7
SET_FREQUENCY_CORRECTION = 8
SET_CHANNEL_INDEX = 9
SET_ANTENNA_INDEX = 10
ASYNCHRONOUS = False
DEVICE_LIB = None
DEVICE_METHODS = {
Command.SET_FREQUENCY.name: "set_center_freq",
Command.SET_SAMPLE_RATE.name: "set_sample_rate",
Command.SET_BANDWIDTH.name: "set_bandwidth",
Command.SET_RF_GAIN.name: "set_rf_gain",
Command.SET_IF_GAIN.name: {"rx": "set_if_rx_gain", "tx": "set_if_tx_gain"},
Command.SET_BB_GAIN.name: {"rx": "set_baseband_gain"}
}
@classmethod
def get_device_list(cls):
return []
@classmethod
def process_command(cls, command, ctrl_connection, is_tx: bool):
is_rx = not is_tx
if command == cls.Command.STOP.name:
return cls.Command.STOP.name
tag, value = command
try:
if isinstance(cls.DEVICE_METHODS[tag], str):
method_name = cls.DEVICE_METHODS[tag]
elif isinstance(cls.DEVICE_METHODS[tag], dict):
method_name = cls.DEVICE_METHODS[tag]["rx" if is_rx else "tx"]
else:
method_name = None
except KeyError:
method_name = None
if method_name:
try:
try:
check_method_name = cls.DEVICE_METHODS[tag+"_get_allowed_values"]
allowed_values = getattr(cls.DEVICE_LIB, check_method_name)()
next_allowed = min(allowed_values, key=lambda x: abs(x-value))
if value != next_allowed:
ctrl_connection.send("{}: {} not in range of supported values. Assuming {}".format(
tag, value, next_allowed
))
value = next_allowed
except (KeyError, AttributeError):
pass
ret = getattr(cls.DEVICE_LIB, method_name)(value)
ctrl_connection.send("{0} to {1}:{2}".format(tag, value, ret))
except AttributeError as e:
logger.warning(str(e))
@classmethod
def setup_device(cls, ctrl_connection: Connection, device_identifier):
raise NotImplementedError("Overwrite this method in subclass!")
@classmethod
def init_device(cls, ctrl_connection: Connection, is_tx: bool, parameters: OrderedDict) -> bool:
if cls.setup_device(ctrl_connection, device_identifier=parameters["identifier"]):
for parameter, value in parameters.items():
cls.process_command((parameter, value), ctrl_connection, is_tx)
return True
else:
return False
@classmethod
def adapt_num_read_samples_to_sample_rate(cls, sample_rate: float):
raise NotImplementedError("Overwrite this method in subclass!")
@classmethod
def shutdown_device(cls, ctrl_connection, is_tx: bool):
raise NotImplementedError("Overwrite this method in subclass!")
@classmethod
def enter_async_receive_mode(cls, data_connection: Connection, ctrl_connection: Connection) -> int:
raise NotImplementedError("Overwrite this method in subclass!")
@classmethod
def prepare_sync_receive(cls, ctrl_connection: Connection):
raise NotImplementedError("Overwrite this method in subclass!")
@classmethod
def receive_sync(cls, data_conn: Connection):
raise NotImplementedError("Overwrite this method in subclass!")
@classmethod
def enter_async_send_mode(cls, callback: object):
raise NotImplementedError("Overwrite this method in subclass!")
@classmethod
def prepare_sync_send(cls, ctrl_connection: Connection):
raise NotImplementedError("Overwrite this method in subclass!")
@classmethod
def send_sync(cls, data):
raise NotImplementedError("Overwrite this method in subclass!")
@classmethod
def device_receive(cls, data_connection: Connection, ctrl_connection: Connection, dev_parameters: OrderedDict):
if not cls.init_device(ctrl_connection, is_tx=False, parameters=dev_parameters):
ctrl_connection.send("failed to start rx mode")
return False
try:
cls.adapt_num_read_samples_to_sample_rate(dev_parameters[cls.Command.SET_SAMPLE_RATE.name])
except NotImplementedError:
# Many SDRs like HackRF or AirSpy do not need to calculate SYNC_RX_CHUNK_SIZE
# as default values are either fine or given by the hardware
pass
if cls.ASYNCHRONOUS:
ret = cls.enter_async_receive_mode(data_connection, ctrl_connection)
else:
ret = cls.prepare_sync_receive(ctrl_connection)
if ret != 0:
ctrl_connection.send("failed to start rx mode")
return False
exit_requested = False
ctrl_connection.send("successfully started rx mode")
while not exit_requested:
if cls.ASYNCHRONOUS:
try:
time.sleep(0.25)
except KeyboardInterrupt:
pass
else:
cls.receive_sync(data_connection)
while ctrl_connection.poll():
result = cls.process_command(ctrl_connection.recv(), ctrl_connection, is_tx=False)
if result == cls.Command.STOP.name:
exit_requested = True
break
cls.shutdown_device(ctrl_connection, is_tx=False)
data_connection.close()
ctrl_connection.close()
@classmethod
def device_send(cls, ctrl_connection: Connection, send_config: SendConfig, dev_parameters: OrderedDict):
if not cls.init_device(ctrl_connection, is_tx=True, parameters=dev_parameters):
ctrl_connection.send("failed to start tx mode")
return False
if cls.ASYNCHRONOUS:
ret = cls.enter_async_send_mode(send_config.get_data_to_send)
else:
ret = cls.prepare_sync_send(ctrl_connection)
if ret != 0:
ctrl_connection.send("failed to start tx mode")
return False
exit_requested = False
buffer_size = cls.CONTINUOUS_TX_CHUNK_SIZE if send_config.continuous else cls.SYNC_TX_CHUNK_SIZE
if not cls.ASYNCHRONOUS and buffer_size == 0:
logger.warning("Send buffer size is zero!")
ctrl_connection.send("successfully started tx mode")
while not exit_requested and not send_config.sending_is_finished():
if cls.ASYNCHRONOUS:
try:
time.sleep(0.5)
except KeyboardInterrupt:
pass
else:
cls.send_sync(send_config.get_data_to_send(buffer_size))
while ctrl_connection.poll():
result = cls.process_command(ctrl_connection.recv(), ctrl_connection, is_tx=True)
if result == cls.Command.STOP.name:
exit_requested = True
break
if not cls.ASYNCHRONOUS:
# Some Sync send calls (e.g. USRP) are not blocking, so we wait a bit here to ensure
# that the send buffer on the SDR is cleared
time.sleep(0.75)
if exit_requested:
logger.debug("{}: exit requested. Stopping sending".format(cls.__class__.__name__))
if send_config.sending_is_finished():
logger.debug("{}: sending is finished.".format(cls.__class__.__name__))
cls.shutdown_device(ctrl_connection, is_tx=True)
ctrl_connection.close()
def __init__(self, center_freq, sample_rate, bandwidth, gain, if_gain=1, baseband_gain=1,
resume_on_full_receive_buffer=False):
super().__init__()
self.error_not_open = -4242
self.__bandwidth = bandwidth
self.__frequency = center_freq
self.__gain = gain # = rf_gain
self.__if_gain = if_gain
self.__baseband_gain = baseband_gain
self.__sample_rate = sample_rate
self.__channel_index = 0
self.__antenna_index = 0
self.__freq_correction = 0
self.__direct_sampling_mode = 0
self.bandwidth_is_adjustable = True
self.is_in_spectrum_mode = False
self.sending_is_continuous = False
self.continuous_send_ring_buffer = None
self.num_samples_to_send = None # None = get automatically. This value needs to be known in continuous send mode
self._current_sent_sample = Value("L", 0)
self._current_sending_repeat = Value("L", 0)
self.success = 0
self.error_codes = {}
self.device_messages = []
self.receive_process_function = self.device_receive
self.send_process_function = self.device_send
self.parent_data_conn, self.child_data_conn = Pipe(duplex=False)
self.parent_ctrl_conn, self.child_ctrl_conn = Pipe()
self.send_buffer = None
self.send_buffer_reader = None
self.device_serial = None
self.device_number = 0
self.samples_to_send = np.array([], dtype=self.DATA_TYPE)
self.sending_repeats = 1 # How often shall the sending sequence be repeated? 0 = forever
self.resume_on_full_receive_buffer = resume_on_full_receive_buffer # for Spectrum Analyzer or Protocol Sniffing
self.current_recv_index = 0
self.is_receiving = False
self.is_transmitting = False
self.device_ip = "192.168.10.2" # For USRP and RTLSDRTCP
self.receive_buffer = None
self.spectrum_x = None
self.spectrum_y = None
self.apply_dc_correction = False
def _start_read_rcv_buffer_thread(self):
self.read_recv_buffer_thread = threading.Thread(target=self.read_receiving_queue)
self.read_recv_buffer_thread.daemon = True
self.read_recv_buffer_thread.start()
def _start_read_message_thread(self):
self.read_dev_msg_thread = threading.Thread(target=self.read_device_messages)
self.read_dev_msg_thread.daemon = True
self.read_dev_msg_thread.start()
@property
def has_multi_device_support(self):
return False
@property
def current_sent_sample(self):
return self._current_sent_sample.value // 2
@current_sent_sample.setter
def current_sent_sample(self, value: int):
self._current_sent_sample.value = value * 2
@property
def current_sending_repeat(self):
return self._current_sending_repeat.value
@current_sending_repeat.setter
def current_sending_repeat(self, value: int):
self._current_sending_repeat.value = value
@property
def device_parameters(self) -> OrderedDict:
return OrderedDict([(self.Command.SET_FREQUENCY.name, self.frequency),
(self.Command.SET_SAMPLE_RATE.name, self.sample_rate),
(self.Command.SET_BANDWIDTH.name, self.bandwidth),
(self.Command.SET_RF_GAIN.name, self.gain),
(self.Command.SET_IF_GAIN.name, self.if_gain),
(self.Command.SET_BB_GAIN.name, self.baseband_gain),
("identifier", self.device_serial)])
@property
def send_config(self) -> SendConfig:
if self.num_samples_to_send is None:
total_samples = len(self.send_buffer)
else:
total_samples = 2 * self.num_samples_to_send
return SendConfig(self.send_buffer, self._current_sent_sample, self._current_sending_repeat,
total_samples, self.sending_repeats, continuous=self.sending_is_continuous,
iq_to_bytes_method=self.iq_to_bytes,
continuous_send_ring_buffer=self.continuous_send_ring_buffer)
@property
def receive_process_arguments(self):
return self.child_data_conn, self.child_ctrl_conn, self.device_parameters
@property
def send_process_arguments(self):
return self.child_ctrl_conn, self.send_config, self.device_parameters
def init_recv_buffer(self):
if self.receive_buffer is None:
num_samples = SettingsProxy.get_receive_buffer_size(self.resume_on_full_receive_buffer,
self.is_in_spectrum_mode)
self.receive_buffer = IQArray(None, dtype=self.DATA_TYPE, n=int(num_samples))
def log_retcode(self, retcode: int, action: str, msg=""):
msg = str(msg)
error_code_msg = self.error_codes[retcode] if retcode in self.error_codes else "Error Code: " + str(retcode)
if retcode == self.success:
if msg:
formatted_message = "{0}-{1} ({2}): Success".format(type(self).__name__, action, msg)
else:
formatted_message = "{0}-{1}: Success".format(type(self).__name__, action)
logger.info(formatted_message)
else:
if msg:
formatted_message = "{0}-{1} ({4}): {2} ({3})".format(type(self).__name__, action, error_code_msg,
retcode, msg)
else:
formatted_message = "{0}-{1}: {2} ({3})".format(type(self).__name__, action, error_code_msg, retcode)
logger.error(formatted_message)
self.device_messages.append(formatted_message)
@property
def received_data(self):
return self.receive_buffer[:self.current_recv_index]
@property
def sent_data(self):
return self.samples_to_send[:self.current_sent_sample]
@property
def sending_finished(self):
return self.current_sent_sample == len(self.samples_to_send)
@property
def bandwidth(self):
return self.__bandwidth
@bandwidth.setter
def bandwidth(self, value):
if not self.bandwidth_is_adjustable:
return
if value != self.__bandwidth:
self.__bandwidth = value
self.set_device_bandwidth(value)
def set_device_bandwidth(self, bw):
try:
self.parent_ctrl_conn.send((self.Command.SET_BANDWIDTH.name, int(bw)))
except (BrokenPipeError, OSError):
pass
@property
def frequency(self):
return self.__frequency
@frequency.setter
def frequency(self, value):
if value != self.__frequency:
self.__frequency = value
self.set_device_frequency(value)
def set_device_frequency(self, value):
try:
self.parent_ctrl_conn.send((self.Command.SET_FREQUENCY.name, int(value)))
except (BrokenPipeError, OSError):
pass
@property
def gain(self):
return self.__gain
@gain.setter
def gain(self, value):
if value != self.__gain:
self.__gain = value
self.set_device_gain(value)
def set_device_gain(self, gain):
try:
# Do not cast gain to int here, as it may be float e.g. for normalized USRP gain or LimeSDR gain
self.parent_ctrl_conn.send((self.Command.SET_RF_GAIN.name, gain))
except (BrokenPipeError, OSError):
pass
@property
def if_gain(self):
return self.__if_gain
@if_gain.setter
def if_gain(self, value):
if value != self.__if_gain:
self.__if_gain = value
self.set_device_if_gain(value)
def set_device_if_gain(self, if_gain):
try:
# Do not cast gain to int here, as it may be float e.g. for normalized USRP gain or LimeSDR gain
self.parent_ctrl_conn.send((self.Command.SET_IF_GAIN.name, if_gain))
except (BrokenPipeError, OSError):
pass
@property
def baseband_gain(self):
return self.__baseband_gain
@baseband_gain.setter
def baseband_gain(self, value):
if value != self.__baseband_gain:
self.__baseband_gain = value
self.set_device_baseband_gain(value)
def set_device_baseband_gain(self, baseband_gain):
try:
# Do not cast gain to int here, as it may be float e.g. for normalized USRP gain or LimeSDR gain
self.parent_ctrl_conn.send((self.Command.SET_BB_GAIN.name, baseband_gain))
except (BrokenPipeError, OSError):
pass
@property
def sample_rate(self):
return self.__sample_rate
@sample_rate.setter
def sample_rate(self, value):
if value != self.__sample_rate:
self.__sample_rate = value
self.set_device_sample_rate(value)
def set_device_sample_rate(self, sample_rate):
try:
self.parent_ctrl_conn.send((self.Command.SET_SAMPLE_RATE.name, int(sample_rate)))
except (BrokenPipeError, OSError):
pass
@property
def channel_index(self) -> int:
return self.__channel_index
@channel_index.setter
def channel_index(self, value: int):
if value != self.__channel_index:
self.__channel_index = value
self.set_device_channel_index(value)
def set_device_channel_index(self, value):
try:
self.parent_ctrl_conn.send((self.Command.SET_CHANNEL_INDEX.name, int(value)))
except (BrokenPipeError, OSError):
pass
@property
def antenna_index(self):
return self.__antenna_index
@antenna_index.setter
def antenna_index(self, value):
if value != self.__antenna_index:
self.__antenna_index = value
self.set_device_antenna_index(value)
def set_device_antenna_index(self, value):
try:
self.parent_ctrl_conn.send((self.Command.SET_ANTENNA_INDEX.name, int(value)))
except (BrokenPipeError, OSError):
pass
@property
def freq_correction(self):
return self.__freq_correction
@freq_correction.setter
def freq_correction(self, value):
if value != self.__freq_correction:
self.__freq_correction = value
self.set_device_freq_correction(value)
def set_device_freq_correction(self, value):
try:
self.parent_ctrl_conn.send((self.Command.SET_FREQUENCY_CORRECTION.name, int(value)))
except (BrokenPipeError, OSError):
pass
@property
def direct_sampling_mode(self):
return self.__direct_sampling_mode
@direct_sampling_mode.setter
def direct_sampling_mode(self, value):
if value != self.__direct_sampling_mode:
self.__direct_sampling_mode = value
self.set_device_direct_sampling_mode(value)
def set_device_direct_sampling_mode(self, value):
try:
self.parent_ctrl_conn.send((self.Command.SET_DIRECT_SAMPLING_MODE.name, int(value)))
except (BrokenPipeError, OSError):
pass
def start_rx_mode(self):
self.init_recv_buffer()
self.parent_data_conn, self.child_data_conn = Pipe(duplex=False)
self.parent_ctrl_conn, self.child_ctrl_conn = Pipe()
self.is_receiving = True
logger.info("{0}: Starting RX Mode".format(self.__class__.__name__))
self.receive_process = Process(target=self.receive_process_function,
args=self.receive_process_arguments)
self.receive_process.daemon = True
self._start_read_rcv_buffer_thread()
self._start_read_message_thread()
try:
self.receive_process.start()
except OSError as e:
logger.error(repr(e))
self.device_messages.append(repr(e))
def stop_rx_mode(self, msg):
try:
self.parent_ctrl_conn.send(self.Command.STOP.name)
except (BrokenPipeError, OSError) as e:
logger.debug("Closing parent control connection: " + str(e))
logger.info("{0}: Stopping RX Mode: {1}".format(self.__class__.__name__, msg))
if hasattr(self, "receive_process") and self.receive_process.is_alive():
self.receive_process.join(self.JOIN_TIMEOUT)
if self.receive_process.is_alive():
logger.warning("{0}: Receive process is still alive, terminating it".format(self.__class__.__name__))
self.receive_process.terminate()
self.receive_process.join()
self.is_receiving = False
for connection in (self.parent_ctrl_conn, self.parent_data_conn, self.child_ctrl_conn, self.child_data_conn):
try:
connection.close()
except OSError as e:
logger.exception(e)
def start_tx_mode(self, samples_to_send: np.ndarray = None, repeats=None, resume=False):
self.is_transmitting = True
self.parent_ctrl_conn, self.child_ctrl_conn = Pipe()
self.init_send_parameters(samples_to_send, repeats, resume=resume)
logger.info("{0}: Starting TX Mode".format(self.__class__.__name__))
self.transmit_process = Process(target=self.send_process_function,
args=self.send_process_arguments)
self.transmit_process.daemon = True
self._start_read_message_thread()
self.transmit_process.start()
def stop_tx_mode(self, msg):
try:
self.parent_ctrl_conn.send(self.Command.STOP.name)
except (BrokenPipeError, OSError) as e:
logger.debug("Closing parent control connection: " + str(e))
logger.info("{0}: Stopping TX Mode: {1}".format(self.__class__.__name__, msg))
if hasattr(self, "transmit_process") and self.transmit_process.is_alive():
self.transmit_process.join(self.JOIN_TIMEOUT)
if self.transmit_process.is_alive():
logger.warning("{0}: Transmit process is still alive, terminating it".format(self.__class__.__name__))
self.transmit_process.terminate()
self.transmit_process.join()
self.is_transmitting = False
try:
self.parent_ctrl_conn.close()
except OSError as e:
logger.exception(e)
try:
self.child_ctrl_conn.close()
except OSError as e:
logger.exception(e)
@staticmethod
def bytes_to_iq(buffer) -> np.ndarray:
pass
@staticmethod
def iq_to_bytes(complex_samples: np.ndarray):
pass
def read_device_messages(self):
while self.is_receiving or self.is_transmitting:
try:
message = self.parent_ctrl_conn.recv()
try:
splitted = message.split(":")
action = ":".join(splitted[:-1])
return_code = splitted[-1]
self.log_retcode(int(return_code), action)
except ValueError:
self.device_messages.append("{0}: {1}".format(self.__class__.__name__, message))
except (EOFError, UnpicklingError, OSError, ConnectionResetError) as e:
logger.info("Exiting read device message thread due to " + str(e))
break
self.is_transmitting = False
self.is_receiving = False
logger.debug("Exiting read device errors thread")
def read_receiving_queue(self):
while self.is_receiving:
try:
byte_buffer = self.parent_data_conn.recv_bytes()
samples = self.bytes_to_iq(byte_buffer)
n_samples = len(samples)
if n_samples == 0:
continue
if self.apply_dc_correction:
samples = samples - np.mean(samples, axis=0)
except OSError as e:
logger.exception(e)
continue
except EOFError:
logger.info("EOF Error: Ending receive thread")
break
if self.current_recv_index + n_samples >= len(self.receive_buffer):
if self.resume_on_full_receive_buffer:
self.current_recv_index = 0
if n_samples >= len(self.receive_buffer):
n_samples = len(self.receive_buffer) - 1
else:
self.stop_rx_mode(
"Receiving buffer is full {0}/{1}".format(self.current_recv_index + n_samples,
len(self.receive_buffer)))
return
self.receive_buffer[self.current_recv_index:self.current_recv_index + n_samples] = samples[:n_samples]
self.current_recv_index += n_samples
logger.debug("Exiting read_receive_queue thread.")
def init_send_parameters(self, samples_to_send: IQArray = None, repeats: int = None, resume=False):
if samples_to_send is not None:
if isinstance(samples_to_send, IQArray):
samples_to_send = samples_to_send.convert_to(self.DATA_TYPE)
else:
samples_to_send = IQArray(samples_to_send).convert_to(self.DATA_TYPE)
self.samples_to_send = samples_to_send
self.send_buffer = None
if self.send_buffer is None:
if isinstance(self.samples_to_send, IQArray):
self.send_buffer = self.iq_to_bytes(self.samples_to_send.data)
else:
self.send_buffer = self.iq_to_bytes(self.samples_to_send)
elif not resume:
self.current_sending_repeat = 0
if repeats is not None:
self.sending_repeats = repeats
class Worker(object):
"""This class is used for poller and reactionner to work.
The worker is a process launch by theses process and read Message in a Queue
(self.s) (slave)
They launch the Check and then send the result in the Queue self.m (master)
they can die if they do not do anything (param timeout)
"""
_id = 0 # None
_process = None
_mortal = None
_idletime = None
_timeout = None
_control_q = None
def __init__(self,
_id,
slave_q,
returns_queue,
processes_by_worker,
mortal=True,
timeout=300,
max_plugins_output_length=8192,
target=None,
loaded_into='unknown',
http_daemon=None):
self._id = self.__class__._id
self.__class__._id += 1
self._mortal = mortal
self._idletime = 0
self._timeout = timeout
self.slave_q = None
self.processes_by_worker = processes_by_worker
self._control_q = Queue() # Private Control queue for the Worker
# By default, take our own code
if target is None:
target = self.work
self._process = Process(target=self._prework,
args=(target, slave_q, returns_queue,
self._control_q))
self.returns_queue = returns_queue
self.max_plugins_output_length = max_plugins_output_length
self.i_am_dying = False
# Keep a trace where the worker is launch from (poller or reactionner?)
self.loaded_into = loaded_into
if os.name != 'nt':
self.http_daemon = http_daemon
else: # windows forker do not like pickle http/lock
self.http_daemon = None
def _prework(self, real_work, *args):
"""Simply drop the BrokHandler before doing the real_work"""
for handler in list(logger.handlers):
if isinstance(handler, BrokHandler):
logger.info("Cleaning BrokHandler %r from logger.handlers..",
handler)
logger.removeHandler(handler)
real_work(*args)
def is_mortal(self):
"""
Accessor to _mortal attribute
:return: A boolean indicating if the worker is mortal or not.
:rtype: bool
"""
return self._mortal
def start(self):
"""
Start the worker. Wrapper for calling start method of the process attribute
:return: None
"""
self._process.start()
def terminate(self):
"""
Wrapper for calling terminate method of the process attribute
Also close queues (input and output) and terminate queues thread
:return: None
"""
# We can just terminate process, not threads
self._process.terminate()
# Is we are with a Manager() way
# there should be not such functions
if hasattr(self._control_q, 'close'):
self._control_q.close()
self._control_q.join_thread()
if hasattr(self.slave_q, 'close'):
self.slave_q.close()
self.slave_q.join_thread()
def join(self, timeout=None):
"""
Wrapper for calling join method of the process attribute
:param timeout: time to wait for the process to terminate
:type timeout: int
:return: None
"""
self._process.join(timeout)
def is_alive(self):
"""
Wrapper for calling is_alive method of the process attribute
:return: A boolean indicating if the process is alive
:rtype: bool
"""
return self._process.is_alive()
def is_killable(self):
"""
Determine whether a process is killable :
* process is mortal
* idletime > timeout
:return: a boolean indicating if it is killable
:rtype: bool
"""
return self._mortal and self._idletime > self._timeout
def add_idletime(self, time):
"""
Increment idletime
:param time: time to increment in seconds
:type time: int
:return: None
"""
self._idletime += time
def reset_idle(self):
"""
Reset idletime (set to 0)
:return: None
"""
self._idletime = 0
def send_message(self, msg):
"""
Wrapper for calling put method of the _control_q attribute
:param msg: the message to put in queue
:type msg: str
:return: None
"""
self._control_q.put(msg)
def set_zombie(self):
"""
Set the process as zombie (mortal to False)
:return:None
"""
self._mortal = False
def get_new_checks(self):
"""
Get new checks if less than nb_checks_max
If no new checks got and no check in queue, sleep for 1 sec
REF: doc/alignak-action-queues.png (3)
:return: None
"""
try:
while len(self.checks) < self.processes_by_worker:
# print "I", self._id, "wait for a message"
msg = self.slave_q.get(block=False)
if msg is not None:
self.checks.append(msg.get_data())
# print "I", self._id, "I've got a message!"
except Empty, exp:
if len(self.checks) == 0:
self._idletime += 1
time.sleep(1)
# Maybe the Queue() is not available, if so, just return
# get back to work :)
except IOError, exp:
return
class Controller(tk.Frame):
def __init__(self, parent, pipe_beacon):
tk.Frame.__init__(self, parent)
self.parent = parent
self.parent.resizable(width=False, height=False)
self.parent.iconbitmap(resource_path("assets/satellite.ico"))
self.parent.title("Dream2space Ground Station")
# Scan for serial ports
ports = self.scan_serial_ports()
self.ports = ports
# Pipe for beacon
self.pipe_beacon = pipe_beacon
# List of pending missions
self.pending_mission_list = []
# List of executing missions
self.current_mission_list = []
# Put all pages into container
self.container = tk.Frame(self.parent)
self.container.grid()
self.make_start_page()
# Poll and check for missions to execute
self.mission_execution_check()
# Initializing method to create Start Page
def make_start_page(self):
self.start = StartPage(self.container, self)
# Scan Serial ports in PC
def scan_serial_ports(self):
ports = []
if sys.platform.startswith('win'):
ports = ['COM%s' % (i + 1) for i in range(256)]
elif sys.platform.startswith('linux') or sys.platform.startswith('cygwin'):
# this excludes your current terminal "/dev/tty"
ports = glob.glob('/dev/tty[A-Za-z]*')
result = []
for port in ports:
try:
s = serial.Serial(port)
s.close()
result.append(port)
except (OSError, serial.SerialException):
continue
result.insert(0, " ")
# In testing, add dummy entries
if IS_TESTING:
result.append("COM14")
result.append("COM15")
return result
# Handle transition from Start Page to Main Page
# Upon pressing button to select Serial ports
def handle_transition(self):
# Extract ports selected
self.port_ttnc = self.start.get_ttnc_port()
self.port_payload = self.start.get_payload_port()
# Ports selected are wrong -> Prompt users to reselect
if self.port_ttnc == self.port_payload or self.port_ttnc == " " or self.port_ttnc == " ":
# Same ports selected
self.start.set_port_warning_message()
# Ports selected are correct -> Proceed to generate main page
else:
# Pass ttnc serial object via pipe to thread
self.pipe_beacon.send(self.port_ttnc)
# Erase Start Page
self.container.grid_forget()
self.container = tk.Frame(self.parent)
self.container.pack()
# Generate Container to store new page
self.main_page = MainPage(parent=self.container, controller=self, beacon_pipe=self.pipe_beacon,
width=app_param.APP_WIDTH, height=app_param.APP_HEIGHT)
# Handles Housekeeping Process after button pressed
def handle_hk_process_start(self):
if IS_TESTING:
self.housekeeping_process = Process(target=sample_hk_command_process, daemon=True) # Testing
else:
self.is_hk_process_success = False
self.prev_file_number = len(os.listdir(
app_param.HOUSEKEEPING_DATA_FOLDER_FILEPATH))
self.housekeeping_process = Process(target=process_get_HK_logs, daemon=True,
args=(self.pipe_beacon, self.port_ttnc, ))
self.housekeeping_process.start()
# Disable mission and housekeeping commands
self.main_page.show_disable_command_after_hk_command()
# Checks regularly if housekeeping process is complete
def hk_process_checking(self):
# If process still alive, continute to check
if self.housekeeping_process.is_alive():
self.after(100, self.hk_process_checking)
# If process ended, inform user
else:
is_success = False
if not IS_TESTING:
# Determine if telecommand obtaining is successful
curr_number_files = len(os.listdir(
app_param.HOUSEKEEPING_DATA_FOLDER_FILEPATH))
if curr_number_files > self.prev_file_number:
self.is_hk_process_success = True
self.prev_file_number = curr_number_files
else:
self.is_hk_process_success = True
# Housekeeping data parsing sucess - Open up explorer
if self.is_hk_process_success == True:
path = os.path.relpath(
app_param.HOUSEKEEPING_DATA_FOLDER_FILEPATH)
if sys.platform.startswith('win'):
os.startfile(path)
elif sys.platform.startswith('linux') or sys.platform.startswith('cygwin'):
subprocess.check_call(['xdg-open', '--', path])
# display success message
is_success = True
# Housekeeping data parsing failed
else:
# display fail message
is_success = False
# Update screens
self.main_page.show_enable_command_after_hk_command()
self.main_page.show_status_after_hk_command(is_success)
# Undo flag
self.is_hk_process_success = False
# Open mission window
def open_mission_downlink_command_window(self):
self.mission_window = MissionWindow(self.parent, self)
# Handle mission checking and scheduling after submited on mission window
def handle_mission_scheduling(self):
def validate_mission(mission_input, pending_mission_list, current_mission_list):
# print(mission_input)
# Validate if (1) mission time is after current time, (2) downlink time after mission time
is_mission_time_future = mission_input.mission_datetime > datetime.datetime.now()
is_downlink_after_mission = mission_input.downlink_datetime > mission_input.mission_datetime
num_mission = len(self.pending_mission_list)
# Generate a complete list of current and pending missions
merge_list = pending_mission_list+current_mission_list
# Validate if (1) new mission is not less than 15 sec before or after a mission
# (2) new downlink is not less than 10 mins before or after a downlink
is_new_mission_allowed = True
for mission in merge_list:
if abs(mission.mission_datetime - mission_input.mission_datetime).total_seconds() <= 15:
is_new_mission_allowed = False
break
if abs(mission.downlink_datetime - mission_input.downlink_datetime).total_seconds() <= 3 * 3 * 60:
is_new_mission_allowed = False
break
if is_new_mission_allowed and is_mission_time_future and is_downlink_after_mission and num_mission < 3:
return True
else:
return False
# Do input validation
mission = self.mission_window.get_user_mission_input()
is_valid_input = validate_mission(mission, self.pending_mission_list, self.current_mission_list)
if is_valid_input:
# Close top window
self.mission_window.handle_mission_success()
# Add into pending mission list
self.pending_mission_list.append(mission)
self.pending_mission_list.sort(key=lambda x: x.downlink_datetime) # Sort on earliest downlink datetime
# print(self.pending_mission_list)
# Send CCSDS mission command to Cubesat
if IS_TESTING:
self.mission_command_process = Process(target=sample_mission_command_process, daemon=True) # Testing
else:
self.mission_command_process = Process(target=process_send_mission_telecommand, daemon=True, args=(
mission, self.pipe_beacon, self.port_ttnc, )) # Testing
self.mission_command_process.start()
# Update screens
self.main_page.show_disable_command_after_mission_command()
self.main_page.update_pending_mission_table(self.pending_mission_list)
else:
# Input time is not valid
num_current_missions = len(self.pending_mission_list)
self.mission_window.display_error_message(num_current_missions)
def mission_execution_check(self):
# print(f"CHECK! {datetime.datetime.now()}")
num_mission = len(self.pending_mission_list)
# print(self.pending_mission_list)
if num_mission != 0:
# Check top most mission item
# Start collection process if within 2 minutes of downlink
earliest_mission = self.pending_mission_list[0]
upcoming_downlink_datetime = earliest_mission.downlink_datetime
if upcoming_downlink_datetime - datetime.datetime.now() < datetime.timedelta(seconds=120):
print(f"less than 2 minutes to mission!!")
self.current_mission_list.append(earliest_mission)
del self.pending_mission_list[0]
if IS_TESTING:
self.downlink_process = Process(target=sample_downlink_process, daemon=True) # Testing
else:
self.downlink_process = Process(
target=process_handle_downlink, daemon=True,
args=(self.port_payload, earliest_mission.get_mission_name(),
earliest_mission.get_mission_datetime_string(),
earliest_mission.get_downlink_datetime_string(),))
self.downlink_process.start()
# Render on the missions screens
self.main_page.update_pending_mission_table(self.pending_mission_list)
self.main_page.update_current_mission_table(self.current_mission_list)
try:
if len(self.current_mission_list) != 0 and not self.downlink_process.is_alive():
del self.current_mission_list[0]
self.main_page.update_current_mission_table(self.current_mission_list)
except AttributeError:
pass
self.after(app_param.APP_DOWNLINK_PROCESS_CHECK_INTERVAL, self.mission_execution_check)
# Respond to button pressed when User wishes to view completed Missions/Downlink
def view_completed_missions(self):
# If no mission created yet, not records found
# Show popup warning
if not os.path.exists(app_param.GROUND_STN_MISSION_LOG_FILEPATH):
messagebox.showerror(title="Dream2space Ground Station",
message="Mission records not found!\nTry to send a mission command.")
else:
if sys.platform.startswith('win'):
# Replace slash with backslash
os.startfile(app_param.GROUND_STN_MISSION_LOG_FILEPATH.replace("/", "\\"))
path = os.path.relpath(app_param.GROUND_STN_MISSION_FOLDER_PATH)
os.startfile(path)
else:
os.startfile(app_param.GROUND_STN_MISSION_LOG_FILEPATH)
if cfg.getSetting('enable_plexgdm') == 'True':
if PlexGDM.Run() > 0:
param['IP_PMS'] = PlexGDM.getIP_PMS()
param['Port_PMS'] = PlexGDM.getPort_PMS()
param['Addr_PMS'] = param['IP_PMS'] + ':' + param['Port_PMS']
dprint('PlexConnect', 0, "PMS: {0}", param['Addr_PMS'])
if cfg.getSetting('enable_dnsserver') == 'True':
p_DNSServer = Process(target=DNSServer.Run,
args=(pipe_DNSServer[1], param))
p_DNSServer.start()
time.sleep(0.1)
if not p_DNSServer.is_alive():
dprint('PlexConnect', 0, "DNSServer not alive. Shutting down.")
sys.exit(1)
p_WebServer = Process(target=WebServer.Run,
args=(pipe_WebServer[1], param))
p_WebServer.start()
time.sleep(0.1)
if not p_WebServer.is_alive():
dprint('PlexConnect', 0, "WebServer not alive. Shutting down.")
if cfg.getSetting('enable_dnsserver') == 'True':
pipe_DNSServer[0].send('shutdown')
p_DNSServer.join()
sys.exit(1)
#for i in range(top_k_results):
# print(labels[top_k_indices[i]], predictions[top_k_indices[i]] / 255.0)
if detections is not None:
label = '%s: %d%%' % (labels[predicted_label],
int((detections[predicted_label] / 255.0) *
100))
cv2.putText(frame, label, (20, 20), cv2.FONT_HERSHEY_SIMPLEX, 0.7,
(0, 0, 0), 2)
frame = cv2.resize(frame, (500, 500))
cv2.imshow('frame', frame)
if cv2.waitKey(1) == ord('q'):
break
# update the FPS counter
fps.update()
else:
cv2.destroyAllWindows()
break
# stop the timer and display FPS information
fps.stop()
time.sleep(1)
print(p, p.is_alive())
print("[INFO] elapsed time: {:.2f}".format(fps.elapsed()))
print("[INFO] approx. FPS: {:.2f}".format(fps.fps()))
print("process id: ", os.getpid())
print("parent process:", os.getppid())
#p.terminate()
p.kill()
#p.join()
video.release()
# time.sleep(1)
"""
1.先创建主进程：
A:创建子进程
B：启动子进程
主进程不能早死于子进程：
"""
"""
子进程中打印数字：1-1000
主进程中打印字母：A
获取进程的pid
"""
def put_num():
for i in range(1, 1000):
print(i, end=" ")
if __name__ == '__main__':
print("A")
print(os.getpid())
p = Process(target=put_num)
p.start()
if p.is_alive():
print("B")
print(current_process())
# p.join()
class Worker:
"""This class is used for poller and reactionner to work.
The worker is a process launch by theses process and read Message in a Queue
(self.s) (slave)
They launch the Check and then send the result in the Queue self.m (master)
they can die if they do not do anything (param timeout)
"""
id = 0 # None
_process = None
_mortal = None
_idletime = None
_timeout = None
_c = None
def __init__(self, id, s, returns_queue, processes_by_worker, mortal=True, timeout=300, max_plugins_output_length=8192, target=None, loaded_into='unknown', http_daemon=None):
self.id = self.__class__.id
self.__class__.id += 1
self._mortal = mortal
self._idletime = 0
self._timeout = timeout
self.s = None
self.processes_by_worker = processes_by_worker
self._c = Queue() # Private Control queue for the Worker
# By default, take our own code
if target is None:
target = self.work
self._process = Process(target=target, args=(s, returns_queue, self._c))
self.returns_queue = returns_queue
self.max_plugins_output_length = max_plugins_output_length
self.i_am_dying = False
# Keep a trace where the worker is launch from (poller or reactionner?)
self.loaded_into = loaded_into
self.http_daemon = http_daemon
def is_mortal(self):
return self._mortal
def start(self):
self._process.start()
# Kill the background process
# AND close correctly the queues (input and output)
# each queue got a thread, so close it too....
def terminate(self):
# We can just terminate process, not threads
if not is_android:
self._process.terminate()
# Is we are with a Manager() way
# there should be not such functions
if hasattr(self._c, 'close'):
self._c.close()
self._c.join_thread()
if hasattr(self.s, 'close'):
self.s.close()
self.s.join_thread()
def join(self, timeout=None):
self._process.join(timeout)
def is_alive(self):
return self._process.is_alive()
def is_killable(self):
return self._mortal and self._idletime > self._timeout
def add_idletime(self, time):
self._idletime = self._idletime + time
def reset_idle(self):
self._idletime = 0
def send_message(self, msg):
self._c.put(msg)
# A zombie is immortal, so kill not be kill anymore
def set_zombie(self):
self._mortal = False
# Get new checks if less than nb_checks_max
# If no new checks got and no check in queue,
# sleep for 1 sec
# REF: doc/shinken-action-queues.png (3)
def get_new_checks(self):
try:
while(len(self.checks) < self.processes_by_worker):
#print "I", self.id, "wait for a message"
msg = self.s.get(block=False)
if msg is not None:
self.checks.append(msg.get_data())
#print "I", self.id, "I've got a message!"
except Empty, exp:
if len(self.checks) == 0:
self._idletime = self._idletime + 1
time.sleep(1)
# Maybe the Queue() is not available, if so, just return
# get back to work :)
except IOError, exp:
return
class camera:
def __init__(self, flags):
# flags[0] = quit
# flags[1] = add user
# flags[2] = camera on or off
self.flags = flags
self.to_scan = 0
self.db_update = 0
self.face_thread = None
self.db_thread = None
self.db = database.Database('python', 'Hinoob22')
self.recognizer = recognizer.Recognizer()
self.faces = set()
self.faceCascade = cv2.CascadeClassifier(
"haarcascade_frontalface_default.xml")
#used for testing
def display_camera_full(self, dev, data, timestamp):
data = cv2.resize(data, (1900, 1050))
cv2.imshow('RGB', frame_convert2.video_cv(data))
if cv2.waitKey(1) == 27:
self.flags[0] = False
#used for testing
def display_camera_half(self, dev, data, timestamp):
data = cv2.resize(data, (1280, 720))
cv2.imshow('RGB', frame_convert2.video_cv(data))
if cv2.waitKey(1) == 27:
self.flags[0] = False
#used for testing
def display_camera_fourth(self, dev, data, timestamp):
if self.to_scan == 120:
print('flags[1]:' + str(self.flags[1]))
if self.flags[1] != False:
print('adding new user')
rgb = data[:, :, ::-1]
self.add_new_user(rgb, self.recognizer)
else:
if self.face_thread != None and self.face_thread.is_alive():
self.face_thread.terminate()
rgb = data[:, :, ::-1]
self.face_thread = Process(target=self.face_detect,
args=(rgb, self.recognizer))
self.face_thread.start()
self.to_scan = 0
cv2.imshow('RGB', frame_convert2.video_cv(data))
if cv2.waitKey(1) == 27:
self.flags[0] = False
self.to_scan += 1
#main function
def display_camera_test(self, dev, data, timestamp):
if self.to_scan == 60:
print('flags' + str(self.flags))
if self.flags[1] != False:
rgb = data[:, :, ::-1]
self.add_new_user(data, rgb, self.recognizer)
else:
if self.face_thread != None and self.face_thread.is_alive():
while True:
if not self.face_thread.is_alive():
break
rgb = data[:, :, ::-1]
self.face_thread = threading.Thread(target=self.face_detect,
args=(data, rgb,
self.recognizer,
self.faces))
self.face_thread.start()
self.to_scan = 0
if self.db_update == 480:
print(self.faces)
if self.db_thread != None and self.db_thread.is_alive():
self.db_thread.terminate()
if self.face_thread != None and self.face_thread.is_alive():
while True:
if not self.face_thread.is_alive():
break
self.db_thread = Process(target=self.update_db,
args=(self.faces, ))
self.db_thread.start()
self.db_update = 0
self.faces = set()
if self.flags[2] == True:
data = cv2.flip(data, 1)
data = cv2.resize(data, (1900, 1050))
cv2.imshow('RGB', frame_convert2.video_cv(data))
else:
data = cv2.resize(data, (1, 1))
cv2.imshow('RGB', frame_convert2.video_cv(data))
if cv2.waitKey(10) == 27:
self.flags[0] = False
self.to_scan += 1
self.db_update += 1
# freenect body
def body(self, *args):
if self.flags[0]:
cv2.destroyWindow('RGB')
self.flags = False
raise freenect.Kill
# properly initializes cv2 and runs freenect
def open_camera(self):
cv2.startWindowThread()
cv2.namedWindow('RGB')
cv2.moveWindow('RGB', 0, 0)
cv2.startWindowThread()
print('starting freenect')
freenect.runloop(video=self.display_camera_test, body=self.body)
# takes a bgr image from the kinect and uses the recognizer to detect and recognize faces.
def face_detect(self, bgr, image, recognizer, faces):
gray = cv2.cvtColor(bgr, cv2.COLOR_BGR2GRAY)
face_locs = self.faceCascade.detectMultiScale(gray,
scaleFactor=1.1,
minNeighbors=5,
minSize=(30, 30))
locations = [[y, x + w, y + h, x] for (x, y, w, h) in face_locs]
results = None
if len(locations) > 0:
try:
results = recognizer.recognize_face_loc(image, locations)
except:
print('exception occured in face_detect for camera')
print(results)
if results and 'obama' in results:
print('obama detected')
#playsound('./obama.mp3')
#self.engine.say('A P T obama obama obama obama obama obama obama')
#self.engine.runAndWait()
if results:
for k in results:
faces.add(k)
# detects and adds a new user face encoding to our dataset of faces
def add_new_user(self, bgr, image, recognizer):
gray = cv2.cvtColor(bgr, cv2.COLOR_BGR2GRAY)
face_locs = self.faceCascade.detectMultiScale(gray,
scaleFactor=1.1,
minNeighbors=5,
minSize=(30, 30))
locations = [[y, x + w, y + h, x] for (x, y, w, h) in face_locs]
if len(locations) > 0:
if (recognizer.detect_and_add(self.flags[1], image,
locations) == True):
print('user found changing add user flag to false')
print('user: '******'updating db')
if len(names) > 0:
db = database.Database('python', 'Hinoob22')
for k in names:
print('adding: ' + k)
db.add_time_now(k)
db.close()
class MainWindow(QMainWindow):
UseDock=False
ShowBBMatchTableView=False
def __init__(self,database_name):
super(MainWindow,self).__init__()
self.setWindowTitle("DarunGrim 4")
self.setWindowIcon(QIcon('DarunGrim.png'))
self.PerformDiffProcess=None
self.DatabaseName=database_name
self.LogDialog=LogTextBoxDialog()
self.LogDialog.resize(800,600)
if RedirectStdOutErr:
self.PHOut=PrintHook(True,func=self.onTextBoxDataReady)
self.PHOut.Start()
self.PHErr=PrintHook(False,func=self.onTextBoxDataReady)
self.PHErr.Start()
self.NonMaxGeometry=None
self.DarunGrimEngine=DarunGrimEngine.DarunGrim(start_ida_listener=True)
self.readSettings()
# Menu
self.createActions()
self.createMenus()
#Use dock? not yet
if not self.UseDock:
bottom_splitter=QSplitter()
self.GraphSplitter=QSplitter()
# Functions
self.FunctionMatchTableView=QTableView()
vheader=QHeaderView(Qt.Orientation.Vertical)
vheader.setResizeMode(QHeaderView.ResizeToContents)
self.FunctionMatchTableView.setVerticalHeader(vheader)
self.FunctionMatchTableView.horizontalHeader().setResizeMode(QHeaderView.Stretch)
self.FunctionMatchTableView.setSortingEnabled(True)
self.FunctionMatchTableView.setSelectionBehavior(QAbstractItemView.SelectRows)
if self.ShowBBMatchTableView:
self.BBMatchTableView=QTableView()
vheader=QHeaderView(Qt.Orientation.Vertical)
vheader.setResizeMode(QHeaderView.ResizeToContents)
self.BBMatchTableView.setVerticalHeader(vheader)
self.BBMatchTableView.horizontalHeader().setResizeMode(QHeaderView.Stretch)
self.BBMatchTableView.setSortingEnabled(True)
self.BBMatchTableView.setSelectionBehavior(QAbstractItemView.SelectRows)
if self.UseDock:
dock=QDockWidget("Functions",self)
dock.setObjectName("Functions")
dock.setAllowedAreas(Qt.LeftDockWidgetArea|Qt.RightDockWidgetArea)
dock.setWidget(self.FunctionMatchTableView)
self.addDockWidget(Qt.BottomDockWidgetArea,dock)
else:
bottom_splitter.addWidget(self.FunctionMatchTableView)
# Blocks
self.BlockTableModel=BlockTable(self,database_name)
self.BlockTableView=QTableView()
vheader=QHeaderView(Qt.Orientation.Vertical)
vheader.setResizeMode(QHeaderView.ResizeToContents)
self.BlockTableView.setVerticalHeader(vheader)
self.BlockTableView.horizontalHeader().setResizeMode(QHeaderView.Stretch)
self.BlockTableView.setSortingEnabled(True)
self.BlockTableView.setModel(self.BlockTableModel)
self.BlockTableView.setSelectionBehavior(QAbstractItemView.SelectRows)
if self.UseDock:
dock=QDockWidget("Blocks",self)
dock.setObjectName("Blocks")
dock.setAllowedAreas(Qt.LeftDockWidgetArea|Qt.RightDockWidgetArea)
dock.setWidget(self.BlockTableView)
self.addDockWidget(Qt.BottomDockWidgetArea,dock)
else:
bottom_splitter.addWidget(self.BlockTableView)
bottom_splitter.setStretchFactor(0,1)
bottom_splitter.setStretchFactor(1,0)
# Function Graph
self.OrigFunctionGraph=MyGraphicsView()
self.OrigFunctionGraph.setRenderHints(QPainter.Antialiasing)
if self.UseDock:
dock=QDockWidget("Orig",self)
dock.setObjectName("Orig")
dock.setAllowedAreas(Qt.LeftDockWidgetArea|Qt.RightDockWidgetArea)
dock.setWidget(view)
self.addDockWidget(Qt.TopDockWidgetArea,dock)
else:
self.GraphSplitter.addWidget(self.OrigFunctionGraph)
# Function Graph
self.PatchedFunctionGraph=MyGraphicsView()
self.PatchedFunctionGraph.setRenderHints(QPainter.Antialiasing)
if self.UseDock:
dock=QDockWidget("Patched",self)
dock.setObjectName("Patched")
dock.setAllowedAreas(Qt.LeftDockWidgetArea|Qt.RightDockWidgetArea)
dock.setWidget(view)
self.addDockWidget(Qt.TopDockWidgetArea,dock)
else:
self.GraphSplitter.addWidget(self.PatchedFunctionGraph)
self.RefreshGraphViews()
if not self.UseDock:
virt_splitter=QSplitter()
virt_splitter.setOrientation(Qt.Vertical)
virt_splitter.addWidget(self.GraphSplitter)
if self.ShowBBMatchTableView:
tab_widget=QTabWidget()
tab_widget.addTab(bottom_splitter,"Functions..")
tab_widget.addTab(self.BBMatchTableView,"Basic blocks...")
virt_splitter.addWidget(tab_widget)
else:
virt_splitter.addWidget(bottom_splitter)
virt_splitter.setStretchFactor(0,1)
virt_splitter.setStretchFactor(1,0)
main_widget=QWidget()
vlayout=QVBoxLayout()
vlayout.addWidget(virt_splitter)
main_widget.setLayout(vlayout)
self.setCentralWidget(main_widget)
self.show()
self.clearAreas()
if database_name:
self.OpenDatabase(database_name)
self.restoreUI()
def RefreshGraphViews(self):
if self.ShowGraphs==True:
self.OrigFunctionGraph.show()
self.PatchedFunctionGraph.show()
self.GraphSplitter.show()
else:
self.OrigFunctionGraph.hide()
self.PatchedFunctionGraph.hide()
self.GraphSplitter.hide()
def clearAreas(self):
self.OrigFunctionGraph.clear()
self.PatchedFunctionGraph.clear()
self.FunctionMatchTable=FunctionMatchTable(self)
self.FunctionMatchTableView.setModel(self.FunctionMatchTable)
if self.ShowBBMatchTableView:
self.BBMatchTable=BBMatchTable(self)
self.BBMatchTableView.setModel(self.BBMatchTable)
self.BlockTableModel=BlockTable(self)
self.BlockTableView.setModel(self.BlockTableModel)
def manageFileStore(self):
dialog=FileStoreBrowserDialog(database_name=self.FileStoreDatabase, darungrim_storage_dir=self.FileStoreDir)
dialog.exec_()
def newFromFileStore(self):
dialog=NewDiffingFromFileStoreDialog(database_name=self.FileStoreDatabase, darungrim_storage_dir=self.FileStoreDir)
if dialog.exec_():
result_filename='%s-%s.dgf' % (dialog.OrigFileSHA1, dialog.PatchedFileSHA1)
log_filename='%s-%s.log' % (dialog.OrigFileSHA1, dialog.PatchedFileSHA1)
self.StartPerformDiff(dialog.OrigFilename,
dialog.PatchedFilename,
os.path.join(self.DataFilesDir, result_filename),
os.path.join(self.DataFilesDir, log_filename),
debug=False
)
file_store_database=FileStoreDatabase.Database(self.FileStoreDatabase)
file_store_database.AddSession(dialog.name_line.text(), dialog.description_line.text(), dialog.OrigFileID, dialog.PatchedFileID, result_filename)
def openFromFileStore(self):
dialog=SessionsDialog(database_name=self.FileStoreDatabase)
if dialog.exec_():
self.OpenDatabase(os.path.join(self.DataFilesDir, dialog.GetFilename()))
self.setWindowTitle("DarunGrim 4 %s" % dialog.GetDescription())
def new(self):
dialog=NewDiffingDialog()
if dialog.exec_():
src_filename = str(dialog.Filenames['Orig'])
target_filename = str(dialog.Filenames['Patched'])
result_filename = str(dialog.Filenames['Result'])
log_filename=result_filename+'.log'
is_src_target_storage = False
if src_filename.lower()[-4:]=='.dgf' and target_filename.lower()[-4:]=='.dgf':
is_src_target_storage=True
self.StartPerformDiff(
src_filename,
target_filename,
result_filename,
log_filename,
is_src_target_storage=is_src_target_storage,
)
def reanalyze(self):
database = DarunGrimDatabase.Database(self.DatabaseName)
[src_filename,target_filename] = database.GetDGFFileLocations()
database.Close()
del database
result_filename=''
if self.DatabaseName[-4:].lower()=='.dgf':
prefix=self.DatabaseName[0:-4]
else:
prefix=self.DatabaseName
i=0
while True:
result_filename=prefix+'-%d.dgf' % i
if not os.path.isfile(result_filename):
break
i+=1
log_filename=result_filename + '.log'
self.StartPerformDiff(src_filename,
target_filename,
str(self.DatabaseName),
log_filename=log_filename,
is_src_target_storage=True,
debug=False)
def onTextBoxDataReady(self,data):
if not self.LogDialog.isVisible():
self.LogDialog.show()
self.LogDialog.addText(data)
def onDiffLogReady(self,data):
if not self.LogDialog.isVisible():
self.LogDialog.show()
self.LogDialog.addText(data)
def PerformDiffCancelled(self):
if self.PerformDiffProcess!=None:
self.PerformDiffProcess.terminate()
self.PerformDiffProcessCancelled=True
def StartPerformDiff(self,src_filename,target_filename,result_filename,log_filename='',is_src_target_storage=False, debug=False):
print "Start Diffing Process: %s vs %s -> %s" % (src_filename,target_filename,result_filename)
self.clearAreas()
if os.path.isfile(log_filename):
os.unlink(log_filename)
try:
os.makedirs(os.path.dirname(result_filename))
except:
pass
src_ida_log_filename=result_filename+'.src.log'
target_ida_log_filename=result_filename+'.target.log'
q=None
debug=False
if debug:
self.PerformDiffProcess=None
PerformDiffThread(src_filename,target_filename,result_filename,log_level=self.LogLevel,dbg_storage_dir=self.DataFilesDir,is_src_target_storage=is_src_target_storage,src_ida_log_filename = src_ida_log_filename, target_ida_log_filename = target_ida_log_filename, ida_path=self.IDAPath, ida64_path=self.IDA64Path, q=q)
else:
q=Queue()
self.PerformDiffProcess=Process(target=PerformDiffThread,args=(src_filename,target_filename,result_filename,log_filename,self.LogLevel,self.DataFilesDir,is_src_target_storage,src_ida_log_filename,target_ida_log_filename,self.IDAPath,self.IDA64Path,q))
self.PerformDiffProcess.start()
self.PerformDiffProcessCancelled=False
if self.PerformDiffProcess!=None:
qlog_thread=QueReadThread(q)
self.LogDialog.SetCancelCallback(self.PerformDiffCancelled)
self.LogDialog.DisableClose()
self.LogDialog.show()
qlog_thread.data_read.connect(self.onDiffLogReady)
qlog_thread.start()
log_threads=[]
for filename in [log_filename,src_ida_log_filename,target_ida_log_filename]:
log_thread=LogThread(filename)
log_thread.data_read.connect(self.onDiffLogReady)
log_thread.start()
log_threads.append(log_thread)
while True:
time.sleep(0.01)
if not self.PerformDiffProcess.is_alive():
break
qApp.processEvents()
for log_thread in log_threads:
log_thread.end()
qlog_thread.end()
self.LogDialog.EnableClose()
if not self.PerformDiffProcessCancelled:
self.LogDialog.addText("Diffing process finished.")
else:
self.LogDialog.addText("Diffing process cancelled.")
self.LogDialog.SetCancelCallback(None)
self.PerformDiffProcess=None
if not self.PerformDiffProcessCancelled:
self.OpenDatabase(result_filename)
def open(self):
(filename,filter)=QFileDialog.getOpenFileName(self,"Open...")
if filename:
self.clearAreas()
self.OpenDatabase(filename)
def OpenFolder(self,folder):
try:
subprocess.check_call(['explorer', folder])
except:
pass
def openOriginalFilesLocation(self):
database = DarunGrimDatabase.Database(self.DatabaseName)
[src_filename,target_filename]=database.GetFilesLocation()
self.OpenFolder(os.path.dirname(src_filename))
def openPatchedFilesLocation(self):
database = DarunGrimDatabase.Database(self.DatabaseName)
[src_filename,target_filename]=database.GetFilesLocation()
self.OpenFolder(os.path.dirname(target_filename))
def OpenIDA(self,filename):
ida_filename=filename
if filename[-4:].lower()!='.idb' and filename[-4:].lower()!='.i64':
for path in [filename[0:-4] + '.idb', filename[0:-4] + '.i64']:
if os.path.isfile(path):
ida_filename=path
break
self.DarunGrimEngine.OpenIDA(ida_filename)
def synchronizeIDA(self):
if self.DatabaseName:
database = DarunGrimDatabase.Database(self.DatabaseName)
[src_filename,target_filename]=database.GetFilesLocation()
self.DarunGrimEngine.SetSourceController(src_filename)
self.DarunGrimEngine.SetTargetController(target_filename)
self.OpenIDA(src_filename)
self.OpenIDA(target_filename)
def captureWindow(self):
(filename,filter)=QFileDialog.getSaveFileName(self,'Save file', filter="*.png")
if filename:
pixmap=QPixmap.grabWidget(super(QMainWindow,self))
pixmap.save(filename,"png")
def saveOrigGraph(self):
(filename,filter)=QFileDialog.getSaveFileName(self,'Save file', filter="*.png")
if filename:
self.OrigFunctionGraph.SaveImg(filename)
def savePatchedGraph(self):
(filename,filter)=QFileDialog.getSaveFileName(self,'Save file', filter="*.png")
if filename:
self.PatchedFunctionGraph.SaveImg(filename)
def showLogs(self):
self.LogDialog.show()
def toggleShowGraphs(self):
if self.ShowGraphs==True:
self.ShowGraphs=False
else:
self.ShowGraphs=True
self.RefreshGraphViews()
def toggleSyncrhonizeIDAUponOpening(self):
if self.SyncrhonizeIDAUponOpening==True:
self.SyncrhonizeIDAUponOpening=False
else:
self.SyncrhonizeIDAUponOpening=True
def showConfiguration(self):
dialog=ConfigurationDialog( file_store_dir=self.FileStoreDir,
data_files_dir=self.DataFilesDir,
ida_path=self.IDAPath,
ida64_path=self.IDA64Path,
log_level=self.LogLevel
)
if dialog.exec_():
self.FileStoreDir=dialog.file_store_dir_line.text()
self.DataFilesDir=dialog.data_files_dir_line.text()
self.FileStoreDatabase=os.path.join(self.DataFilesDir,'index.db')
self.IDAPath=dialog.ida_path_line.text()
self.IDA64Path=dialog.ida64_path_line.text()
self.DarunGrimEngine.SetIDAPath(self.IDAPath)
self.DarunGrimEngine.SetIDAPath(self.IDA64Path,True)
self.LogLevel=int(dialog.log_level_line.text())
def serverInfo(self):
dialog=ServerInfoDialog(port=self.DarunGrimEngine.ListeningPort)
dialog.exec_()
def toggleStaysOnTop(self):
if self.StaysOnTop==True:
self.StaysOnTop=False
self.hide()
self.setWindowFlags(self.windowFlags()& ~Qt.WindowStaysOnTopHint)
self.show()
else:
self.StaysOnTop=True
self.hide()
self.setWindowFlags(self.windowFlags()|Qt.WindowStaysOnTopHint)
self.show()
def intallIDAPlugin(self):
(ret1,message1)=self.DarunGrimEngine.InstallIDAPlugin('DarunGrimPlugin.plw')
(ret2,message2)=self.DarunGrimEngine.InstallIDAPlugin('DarunGrimPlugin.p64')
if not ret1 or not ret2:
msg_box=QMessageBox()
if message1!=message2:
message1 += '\n' + message2
msg_box.setText('Try to run the program with an Administrator privilege\n' + message1)
msg_box.exec_()
return False
else:
msg_box=QMessageBox()
msg_box.setText('Installation successful\n'+message1 + '\n' + message2)
msg_box.exec_()
return True
def createActions(self):
self.newAct = QAction("New Diffing...",
self,
shortcut=QKeySequence.New,
statusTip="Create new diffing output",
triggered=self.new
)
self.openAct = QAction("Open...",
self,
shortcut=QKeySequence.Open,
statusTip="Open a dgf database",
triggered=self.open
)
self.manageFileStoreAct = QAction("Manage FileStore...",
self,
statusTip="Manage FileStore",
triggered=self.manageFileStore
)
self.newFromFileStoreAct = QAction("New Diffing (FileStore)...",
self,
statusTip="Create new diffing output",
triggered=self.newFromFileStore
)
self.openFromFileStoreAct = QAction("Open Diffing (FileStore)...",
self,
statusTip="Open diffing output",
triggered=self.openFromFileStore
)
self.reanalyzeAct = QAction("Reanalyze...",
self,
statusTip="Reanalyze current files",
triggered=self.reanalyze
)
self.synchornizeIDAAct= QAction("Synchornize IDA",
self,
statusTip="Synchronize IDA",
triggered=self.synchronizeIDA
)
self.openOriginalFilesLocationAct = QAction("Open Orininal Files Location",
self,
statusTip="Open original file location",
triggered=self.openOriginalFilesLocation
)
self.openPatchedFilesLocationAct = QAction("Open Patched Files Location",
self,
statusTip="Open patched file location",
triggered=self.openPatchedFilesLocation
)
self.captureWindowAct = QAction("Capture...",
self,
statusTip="Save patched graph",
triggered=self.captureWindow
)
self.saveOrigGraphAct = QAction("Save orig graph...",
self,
statusTip="Save original graph",
triggered=self.saveOrigGraph
)
self.savePatchedGraphAct = QAction("Save patched graph...",
self,
statusTip="Save patched graph",
triggered=self.savePatchedGraph
)
self.showLogsAct = QAction("Show logs...",
self,
statusTip="Show logs",
triggered=self.showLogs
)
self.showGraphsAct = QAction("Show graphs...",
self,
statusTip="Show graphs",
triggered=self.toggleShowGraphs,
checkable=True
)
self.showGraphsAct.setChecked(self.ShowGraphs)
self.syncrhonizeIDAUponOpeningAct = QAction("Synchronize IDA upon opening...",
self,
statusTip="Synchronize IDA upon opening",
triggered=self.toggleSyncrhonizeIDAUponOpening,
checkable=True
)
self.syncrhonizeIDAUponOpeningAct.setChecked(self.SyncrhonizeIDAUponOpening)
self.configurationAct = QAction("Configuration...",
self,
statusTip="Configuration",
triggered=self.showConfiguration
)
self.serverInfoAct = QAction("Server...",
self,
statusTip="Server Info",
triggered=self.serverInfo
)
self.staysOnTopAct = QAction("Statys on top...",
self,
statusTip="Server Info",
triggered=self.toggleStaysOnTop,
checkable=True
)
self.staysOnTopAct.setChecked(self.StaysOnTop)
self.intallIDAPluginAct = QAction("Install IDA Plugin...",
self,
statusTip="Install IDA Plugin...",
triggered=self.intallIDAPlugin
)
def createMenus(self):
self.fileMenu = self.menuBar().addMenu("&File")
self.fileMenu.addAction(self.newAct)
self.fileMenu.addAction(self.openAct)
self.fileMenu.addAction(self.manageFileStoreAct)
self.fileMenu.addAction(self.newFromFileStoreAct)
self.fileMenu.addAction(self.openFromFileStoreAct)
self.fileMenu.addAction(self.reanalyzeAct)
self.analysisMenu = self.menuBar().addMenu("&Analysis")
self.analysisMenu.addAction(self.synchornizeIDAAct)
self.analysisMenu.addAction(self.openOriginalFilesLocationAct)
self.analysisMenu.addAction(self.openPatchedFilesLocationAct)
self.analysisMenu.addAction(self.captureWindowAct)
self.analysisMenu.addAction(self.saveOrigGraphAct)
self.analysisMenu.addAction(self.savePatchedGraphAct)
self.analysisMenu.addAction(self.showLogsAct)
self.optionsMenu = self.menuBar().addMenu("&Options")
self.optionsMenu.addAction(self.showGraphsAct)
self.optionsMenu.addAction(self.syncrhonizeIDAUponOpeningAct)
self.optionsMenu.addAction(self.staysOnTopAct)
self.optionsMenu.addAction(self.configurationAct)
self.optionsMenu.addAction(self.serverInfoAct)
self.optionsMenu.addAction(self.intallIDAPluginAct)
def OpenDatabase(self,databasename):
self.DatabaseName=databasename
self.FunctionMatchTable=FunctionMatchTable(self,self.DatabaseName)
self.FunctionMatchTableView.setModel(self.FunctionMatchTable)
selection=self.FunctionMatchTableView.selectionModel()
if selection!=None:
selection.selectionChanged.connect(self.handleFunctionMatchTableChanged)
if self.ShowBBMatchTableView:
self.BBMatchTable=BBMatchTable(self,self.DatabaseName)
self.BBMatchTableView.setModel(self.BBMatchTable)
selection=self.BBMatchTableView.selectionModel()
if selection!=None:
selection.selectionChanged.connect(self.handleBBMatchTableChanged)
database = DarunGrimDatabase.Database(self.DatabaseName)
self.setWindowTitle("DarunGrim 4 - %s" % (database.GetDescription()))
if self.SyncrhonizeIDAUponOpening:
self.synchronizeIDA()
def ColorController(self, type, disasms, match_info):
for (address,[end_address,disasm]) in disasms.items():
if not match_info.has_key(address):
#Red block
self.DarunGrimEngine.ColorAddress(type, address, end_address+1, 0x0000FF)
elif match_info[address][1]!=100:
#Yellow block
self.DarunGrimEngine.ColorAddress(type, address, end_address+1, 0x00FFFF)
def handleFunctionMatchTableChanged(self,selected,dselected):
for item in selected:
for index in item.indexes():
[source_function_address, target_function_address] = self.FunctionMatchTable.GetFunctionAddresses(index.row())
self.BlockTableModel=BlockTable(self,self.DatabaseName,source_function_address, target_function_address)
self.BlockTableView.setModel(self.BlockTableModel)
selection=self.BlockTableView.selectionModel()
if selection!=None:
selection.selectionChanged.connect(self.handleBlockTableChanged)
database=DarunGrimDatabase.Database(self.DatabaseName)
(source_disasms, source_links) = database.GetFunctionDisasmLines("Source", source_function_address)
(target_disasms, target_links) = database.GetFunctionDisasmLines("Target", target_function_address)
source_match_info=self.BlockTableModel.GetSourceMatchInfo()
target_match_info=self.BlockTableModel.GetTargetMatchInfo()
#IDA Sync
self.ColorController(0, source_disasms, source_match_info )
self.ColorController(1, target_disasms, target_match_info )
self.DarunGrimEngine.JumpToAddresses(source_function_address, target_function_address)
if self.ShowGraphs:
# Draw graphs
self.OrigFunctionGraph.SetDatabaseName(self.DatabaseName)
self.OrigFunctionGraph.DrawFunctionGraph("Source", source_function_address, source_disasms, source_links, source_match_info)
self.OrigFunctionGraph.SetSelectBlockCallback(self.SelectedBlock)
self.OrigFunctionGraph.HilightAddress(source_function_address)
self.PatchedFunctionGraph.SetDatabaseName(self.DatabaseName)
self.PatchedFunctionGraph.DrawFunctionGraph("Target", target_function_address, target_disasms, target_links, target_match_info)
self.PatchedFunctionGraph.SetSelectBlockCallback(self.SelectedBlock)
self.PatchedFunctionGraph.HilightAddress(target_function_address)
break
def handleBBMatchTableChanged(self,selected,dselected):
pass
def handleBlockTableChanged(self,selected,dselected):
for item in selected:
for index in item.indexes():
[orig_address,patched_address]=self.BlockTableModel.GetBlockAddresses(index.row())
if self.ShowGraphs:
if orig_address!=0:
self.OrigFunctionGraph.HilightAddress(orig_address)
if patched_address!=0:
self.PatchedFunctionGraph.HilightAddress(patched_address)
self.DarunGrimEngine.JumpToAddresses(orig_address, patched_address)
break
def SelectedBlock(self,graph,address):
if graph==self.OrigFunctionGraph:
matched_address=self.BlockTableModel.GetMatchAddresses(0,address)
if matched_address!=None:
self.PatchedFunctionGraph.HilightAddress(matched_address)
self.DarunGrimEngine.JumpToAddresses(0, matched_address)
elif graph==self.PatchedFunctionGraph:
matched_address=self.BlockTableModel.GetMatchAddresses(1,address)
if matched_address!=None:
self.OrigFunctionGraph.HilightAddress(matched_address)
self.DarunGrimEngine.JumpToAddresses(matched_address, 0)
def changeEvent(self,event):
if event.type()==QEvent.WindowStateChange:
if (self.windowState()&Qt.WindowMinimized)==0 and \
(self.windowState()&Qt.WindowMaximized)==0 and \
(self.windowState()&Qt.WindowFullScreen)==0 and \
(self.windowState()&Qt.WindowActive)==0:
pass
def resizeEvent(self,event):
if not self.isMaximized():
self.NonMaxGeometry=self.saveGeometry()
def restoreUI(self):
settings=QSettings("DarunGrim LLC", "DarunGrim")
if settings.contains("geometry/non_max"):
self.NonMaxGeometry=settings.value("geometry/non_max")
self.restoreGeometry(self.NonMaxGeometry)
else:
self.resize(800,600)
self.NonMaxGeometry=self.saveGeometry()
if settings.contains("isMaximized"):
if settings.value("isMaximized")=="true":
self.setWindowState(self.windowState()|Qt.WindowMaximized)
self.restoreState(settings.value("windowState"))
self.FirstConfigured=False
if not settings.contains("General/FirstConfigured"):
self.showConfiguration()
if self.intallIDAPlugin():
self.FirstConfigured=True
else:
self.FirstConfigured=True
def readSettings(self):
settings=QSettings("DarunGrim LLC", "DarunGrim")
self.ShowGraphs=True
if settings.contains("General/ShowGraphs"):
if settings.value("General/ShowGraphs")=='true':
self.ShowGraphs=True
else:
self.ShowGraphs=False
self.SyncrhonizeIDAUponOpening=False
if settings.contains("General/SyncrhonizeIDAUponOpening"):
if settings.value("General/SyncrhonizeIDAUponOpening")=='true':
self.SyncrhonizeIDAUponOpening=True
else:
self.SyncrhonizeIDAUponOpening=False
self.StaysOnTop=False
if settings.contains("General/StaysOnTop"):
if settings.value("General/StaysOnTop")=='true':
self.StaysOnTop=True
else:
self.StaysOnTop=False
if self.StaysOnTop==True:
self.setWindowFlags(self.windowFlags()|Qt.WindowStaysOnTopHint)
else:
self.setWindowFlags(self.windowFlags()& ~Qt.WindowStaysOnTopHint)
self.FileStoreDir = os.path.join(os.getcwd(), "DarunGrimStore")
if settings.contains("General/FileStoreDir"):
self.FileStoreDir=settings.value("General/FileStoreDir")
if not os.path.isdir(self.FileStoreDir):
try:
os.makedirs(self.FileStoreDir)
except:
import traceback
traceback.print_exc()
self.FileStoreDatabase='index.db'
if settings.contains("General/FileStoreDatabase"):
self.FileStoreDatabase=settings.value("General/FileStoreDatabase")
self.DataFilesDir=os.path.join(os.getcwd(), "DarunGrimData")
if settings.contains("General/DataFilesDir"):
self.DataFilesDir=settings.value("General/DataFilesDir")
if not os.path.isdir(self.DataFilesDir):
try:
os.makedirs(self.DataFilesDir)
except:
import traceback
traceback.print_exc()
self.IDAPath=''
if settings.contains("General/IDAPath"):
self.IDAPath=settings.value("General/IDAPath")
else:
files=self.DarunGrimEngine.LocateIDAExecutables()
if len(files)>0:
self.IDAPath=files[0][0]
self.DarunGrimEngine.SetIDAPath(self.IDAPath)
if not self.DarunGrimEngine.CheckIDAPlugin():
#print 'DarunGrim plugin is missing'
pass
self.IDA64Path=''
if settings.contains("General/IDA64Path"):
self.IDAPath=settings.value("General/IDA64Path")
else:
files=self.DarunGrimEngine.LocateIDAExecutables(is_64=True)
if len(files)>0:
self.IDA64Path=files[0][0]
self.DarunGrimEngine.SetIDAPath(self.IDA64Path,is_64=True)
self.LogLevel=10
if settings.contains("General/LogLevel"):
self.LogLevel=int(settings.value("General/LogLevel"))
def saveSettings(self):
settings = QSettings("DarunGrim LLC", "DarunGrim")
settings.setValue("General/ShowGraphs", self.ShowGraphs)
settings.setValue("General/SyncrhonizeIDAUponOpening", self.SyncrhonizeIDAUponOpening)
settings.setValue("General/StaysOnTop", self.StaysOnTop)
settings.setValue("General/FileStoreDir", self.FileStoreDir)
settings.setValue("General/FileStoreDatabase", self.FileStoreDatabase)
settings.setValue("General/DataFilesDir", self.DataFilesDir)
settings.setValue("General/LogLevel", self.LogLevel)
if self.FirstConfigured==True:
settings.setValue("General/FirstConfigured", self.FirstConfigured)
if self.NonMaxGeometry!=None:
settings.setValue("geometry/non_max", self.NonMaxGeometry)
settings.setValue("isMaximized", self.isMaximized())
settings.setValue("windowState", self.saveState())
def closeEvent(self, event):
self.PerformDiffCancelled()
self.saveSettings()
QMainWindow.closeEvent(self, event)
class AstroPrintPipeline(object):
def __init__(self, device, size, rotation, source, encoding, onFatalError):
self._logger = logging.getLogger(__name__)
self._parentConn, self._processConn = Pipe(True)
self._pendingReqs = {}
self._preservativePendingReqs = {}
self._lastReqId = 0
self._device = device
self._rotation = rotation
self._source = source.lower()
self._encoding = encoding.lower()
self._size = tuple([int(x) for x in size.split('x')])
self._sendCondition = Condition()
self._onFatalError = onFatalError
self._responseListener = ProcessResponseListener(
self._parentConn, self._onProcessResponse)
self._responseListener.start()
self._process = None
self._listening = False
def __del__(self):
self._logger.debug('Pipeline Process Controller removed')
def _kill(self):
if self._process:
try:
os.kill(self._process.pid, signal.SIGKILL)
self._process.join()
except OSError as e:
# error 3: means the pid is not valid, so the process has been killed
if e.errno != 3:
raise e
self._process = None
def startProcess(self):
if self._process:
# This should almost never happen (but it does)
# Make sure the previous process is killed before a new one is started
self._logger.warn(
"A previous process was still running, killing it")
self._kill()
onListeningEvent = Event()
errorState = Value(
'b',
False) #If True, it means the process had an error while starting
self._listening = False
self._process = Process(
target=startPipelineProcess,
args=(self._device, self._size, self._rotation, self._source,
self._encoding, onListeningEvent, errorState,
(self._parentConn, self._processConn),
settings().getInt(['camera', 'debug-level'])))
self._process.daemon = True
self._process.start()
if onListeningEvent.wait(20.0):
if errorState.value:
self._logger.error('Pipeline Failed to start.')
self._kill()
self._logger.debug('Pipeline Process killed.')
else:
self._logger.debug('Pipeline Process Started.')
self._listening = True
else:
self._logger.debug(
'Timeout while waiting for pipeline process to start')
self._kill()
def stopProcess(self):
if self._process:
if self._listening:
self._sendReqToProcess({'action': 'shutdown'})
self._process.join(
2.0) #Give it two seconds to exit and kill otherwise
if self._process.exitcode is None:
self._logger.warn(
'Process did not shutdown properly. Terminating...')
self._process.terminate()
self._process.join(
2.0
) # Give it another two secods to terminate, otherwise kill
if self._process and self._process.exitcode is None:
self._logger.warn(
'Process did not terminate properly. Sending KILL signal...'
)
self._kill()
self._logger.debug('Process terminated')
self._process = None
@property
def processRunning(self):
return self._process and self._process.is_alive()
def stop(self):
self._responseListener.stop()
self.stopProcess()
#It's possible that stop is called as a result of a response which is
#executed in the self._responseListener Thread. You can't join your own thread!
if current_thread() != self._responseListener:
self._responseListener.join()
self._responseListener = None
def startLocalVideo(self, onFrameTakenCallback):
def postprocesingLocalVideoFrame(resp):
if resp:
if isinstance(resp, dict) and 'error' in resp:
self._logger.error(
'Error during local video\'s frames capture: %s' %
resp['error'])
onFrameTakenCallback(None)
else:
from base64 import b64decode
try:
onFrameTakenCallback(b64decode(resp))
except TypeError as e:
self._logger.error(
'Invalid returned local video\'s frame. Received. Error: %s'
% e)
onFrameTakenCallback(None)
else:
onFrameTakenCallback(None)
self._sendPreservativeReq({'action': 'startLocalVideo'},
postprocesingLocalVideoFrame)
def stopLocalVideo(self, doneCallback=None):
self._sendReqToProcess({'action': 'stopLocalVideo'}, doneCallback)
def isLocalVideoPlaying(self, doneCallback):
self._sendReqToProcess({'action': 'isLocalVideoPlaying'}, doneCallback)
def startVideo(self, doneCallback=None):
self._sendReqToProcess({'action': 'startVideo'}, doneCallback)
def stopVideo(self, doneCallback=None):
self._sendReqToProcess({'action': 'stopVideo'}, doneCallback)
def isVideoPlaying(self, doneCallback):
self._sendReqToProcess({'action': 'isVideoPlaying'}, doneCallback)
def isAnyVideoPlaying(self, doneCallback):
self._sendReqToProcess({'action': 'isAnyVideoPlaying'}, doneCallback)
def takePhoto(self, doneCallback, text=None):
def postprocesing(resp):
if resp:
if isinstance(resp, dict) and 'error' in resp:
self._logger.error('Error during photo capture: %s' %
resp['error'])
doneCallback(None)
else:
from base64 import b64decode
try:
doneCallback(b64decode(resp))
except TypeError as e:
self._logger.error(
'Invalid returned photo. Received. Error: %s' % e)
doneCallback(None)
else:
doneCallback(None)
if text is not None:
self._sendReqToProcess(
{
'action': 'takePhoto',
'data': {
'text': text
}
}, postprocesing)
else:
self._sendReqToProcess({
'action': 'takePhoto',
'data': None
}, postprocesing)
def _onProcessResponse(self, id, data):
if id is 0: # this is a broadcast, likely an error. Inform all pending requests
self._logger.warn(
'Broadcasting error to ALL pending requests [ %s ]' %
repr(data))
if self._pendingReqs:
for cb in self._pendingReqs.values():
if cb:
cb(data)
if self._preservativePendingReqs:
for cb in self._preservativePendingReqs.values():
if cb:
cb(data)
if data and 'error' in data and data['error'] == 'fatal_error':
message = 'Fatal error occurred in video streaming (%s)' % data[
'details'] if 'details' in data else 'unkonwn'
#signaling for remote peers
manage_fatal_error_webrtc = blinkerSignal(
'manage_fatal_error_webrtc')
manage_fatal_error_webrtc.send(self, message=message)
#event for local peers
eventManager().fire(Events.GSTREAMER_EVENT,
{'message': message})
try:
self._logger.info(
"Trying to get list of formats supported by your camera..."
)
self._logger.info(
subprocess.Popen("v4l2-ctl --list-formats-ext -d %s" %
str(self._device),
shell=True,
stdout=subprocess.PIPE).stdout.read())
except:
self._logger.error("Unable to retrieve supported formats")
#shutdown the process
self._pendingReqs = {}
self._preservativePendingReqs = {}
self._onFatalError()
elif id in self._pendingReqs:
try:
callback = self._pendingReqs[id]
if callback:
callback(data)
del self._pendingReqs[id]
if self._logger.isEnabledFor(logging.DEBUG):
if sys.getsizeof(data) > 50:
dataStr = "%d bytes" % sys.getsizeof(data)
else:
dataStr = repr(data)
self._logger.debug('Response for %d handled [ %s ]' %
(id, dataStr))
except Exception:
self._logger.error("Problem executing callback response",
exc_info=True)
elif id in self._preservativePendingReqs:
try:
callback = self._preservativePendingReqs[id]
if callback:
callback(data)
if self._logger.isEnabledFor(logging.DEBUG):
if sys.getsizeof(data) > 50:
dataStr = "%d bytes" % sys.getsizeof(data)
else:
dataStr = repr(data)
self._logger.debug(
'Preservative response for %d handled [ %s ]' %
(id, dataStr))
except Exception:
self._logger.error("Problem executing callback response",
exc_info=True)
else:
self._logger.error(
"There's no pending [normal/preservative] request for response %s"
% id)
def _sendPreservativeReq(self, data, callback=None):
self.__addReq(data, callback, True)
def _sendReqToProcess(self, data, callback=None):
self.__addReq(data, callback, False)
def __addReq(self, data, callback, preservative):
if self.processRunning:
with self._sendCondition:
if self._listening:
self._lastReqId += 1
id = self._lastReqId
if preservative:
self._preservativePendingReqs[id] = callback
else:
self._pendingReqs[id] = callback
self._parentConn.send((id, data))
self._logger.debug('Sent request %s to process [ %s ]' %
(id, repr(data)))
else:
self._logger.debug(
'Process not listening. There was a problem while starting it.'
)
if callback:
callback({
'error':
'not_listening',
'details':
'The process is not currently listening to requests'
})
else:
self._logger.debug('Process not running. Trying to restart')
self.startProcess()
if self.processRunning:
self._sendReqToProcess(data, callback)
else:
self._logger.error('Unable to re-start pipeline process.')
if callback:
callback({
'error': 'no_process',
'details': 'Unable to re-start process'
})
def start_sampler(task_queue, response_queue):
p = Process(target=sampler.Sampler.sampler,
args=(task_queue, response_queue))
p.start()
assert (p.is_alive() == True)
return p
from multiprocessing import Process
def run_test(name1, name2, name3, **kwargs):
print('子进程:%s' % os.getpid())
print(locals())
print('子进程终止')
if __name__ == '__main__':
print('主进程:%s' % os.getpid())
# 元祖里面一个参数，带上“，”
p = Process(target=run_test,
args=(
1,
2,
'test',
),
kwargs={
'name4': 4,
'name5': 5
})
p.start()
print('子进程pid：%s，子进程状态：%s' % (p.pid, p.is_alive()))
# 主进程等待子进程,timeout超时时间
p.join(timeout=100)
print('主进程终止')
class HumanTrackedEventWatcher(ALModule):
""" A module to react to HumanTracked and PeopleLeft events """
def __init__(self):
ALModule.__init__(self, "humanEventWatcher")
global memory
memory = ALProxy("ALMemory", ip_robot, port_robot)
memory.subscribeToEvent("ALBasicAwareness/HumanTracked",
"humanEventWatcher",
"onHumanTracked")
memory.subscribeToEvent("ALBasicAwareness/PeopleLeft",
"humanEventWatcher",
"onPeopleLeft")
#memory.subscribeToEvent('WordRecognized', ip_robot, 'wordRecognized')
self.speech_reco = ALProxy("ALSpeechRecognition", ip_robot, port_robot)
self.text_to_speech=ALProxy("ALTextToSpeech", ip_robot, port_robot)
self.is_speech_reco_started = False
self.photo_apture=ALProxy("ALPhotoCapture", ip_robot, port_robot)
self.cameraMap = {
'Top': 0,
'Bottom': 1
}
self.camera_id=0
self.recordFolder = "/home/nao/recordings/cameras/"
self.p = Process(target=interat)
def onHumanTracked(self, key, value, msg):
""" callback for event HumanTracked """
print "got HumanTracked: detected person with ID:", str(value)
if value >= 0: # found a new person
self.start_speech_reco()
position_human = self.get_people_perception_data(value)
[x, y, z] = position_human
print "The tracked person with ID", value, "is at the position:", \
"x=", x, "/ y=", y, "/ z=", z
print "Aha, I saw a human!"
self.p.is_alive()
if (
self.p.is_alive()):
pass
else:
self.p = Process(target=interat)
self.p.start()
else:
pass
def onPeopleLeft(self, key, value, msg):
""" callback for event PeopleLeft """
print "got PeopleLeft: lost person", str(value)
self.p.terminate()
self.stop_speech_reco()
def start_speech_reco(self):
""" start asr when someone's detected in event handler class """
if not self.is_speech_reco_started:
try:
data = memory.getData("WordRecognized")
print(data)
except RuntimeError:
print "ASR already started"
self.speech_reco.setVisualExpression(True)
self.speech_reco.subscribe("BasicAwareness_Test")
self.is_speech_reco_started = True
print "start ASR"
def stop_speech_reco(self):
""" stop asr when someone's detected in event handler class """
if self.is_speech_reco_started:
self.speech_reco.unsubscribe("BasicAwareness_Test")
self.is_speech_reco_started = False
print "stop ASR"
def get_people_perception_data(self, id_person_tracked):
memory = ALProxy("ALMemory", ip_robot, port_robot)
memory_key = "PeoplePerception/Person/" + str(id_person_tracked) + \
"/PositionInWorldFrame"
return memory.getData(memory_key)
try:
queue__key_exchanges = Queue()
queue__key_exchanges_signaller = Queue()
queue__audio_receiver = Queue()
queue__audio_receiver_signaller = Queue()
process__key_exchange_facilitator = Process(
target=ke.handle_client_connections,
args=(queue__key_exchanges, queue__key_exchanges_signaller))
process__audio_receiver = Process(target=audio_server.handle_audio_receipt,
args=(queue__audio_receiver,
queue__audio_receiver_signaller))
process__key_exchange_facilitator.start()
process__audio_receiver.start()
logging.debug("Status of process__key_exchange_facilitator: " +
str(process__key_exchange_facilitator.is_alive()))
logging.debug("Status of process__audio_receiver: " +
str(process__audio_receiver.is_alive()))
countdown = 5
iterate_flag = True
while iterate_flag:
logging.debug("Waiting to receive data from <key_exchanges> queue")
queue_data = queue__key_exchanges.get(block=True)
assert queue_data == signals.SIG_CHECKPOINT
participant_count = queue__key_exchanges.get(block=True)
queue__audio_receiver.put(participant_count)
countdown -= 1
if countdown <= 0:
iterate_flag = False
class MidiOverNetSender(object):
def __init__(self):
self._midiInputList = None
self._midiOverNetProcess = None
self._midiOverNetQueue = Queue(32)
self._statusQueue = Queue(1024)
self._debugPrintQueue = Queue(1024)
def scanInputs(self):
pygame.midi.init()
self._midiInputList = []
midiDevices = pygame.midi.get_count()
for i in range(midiDevices):
midiSys, midiName, midiInput, midiOutput, midiOpen = pygame.midi.get_device_info(i) #@UnusedVariable
if(midiInput == 1):
if(midiOpen == 1):
midiState = "Already in use."
else:
midiState = "Available."
print("input id: %d \"%s\" %s" % (i, midiName, midiState))
self._midiInputList.append((midiName, midiOpen, i))
if(midiOutput == 1):
if(midiOpen == 1):
midiState = "Already in use."
else:
midiState = "Available."
print("output id: %d \"%s\" %s" % (i, midiName, midiState))
return self._midiInputList
# self._host = "10.242.10.145"
# self._port = 2020
def startMidiOverNetProcess(self, inputId, host, port, guiHost, guiPort, useBroadcast, filterClock):
if((inputId < len(self._midiInputList)) and (inputId >= 0)):
midiName, midiOpen, pygameMidiId = self._midiInputList[inputId] #@UnusedVariable
if(useBroadcast == True):
host = '<broadcast>'
clockInfo = "Sending MIDI clock."
if(filterClock == True):
clockInfo = "Filtering MIDI clock!"
print "Starting MidiOverNetPorcess. From MIDI input: \"%s\" to host: %s:%d %s" %(midiName, host, port, clockInfo)
self._midiOverNetProcess = Process(target=midiOverNetProcess, args=(host, port, guiHost, guiPort, useBroadcast, filterClock, pygameMidiId, self._midiOverNetQueue, self._statusQueue, self._debugPrintQueue))
self._midiOverNetProcess.name = "midiUdpSender"
self._midiOverNetProcess.start()
def getMidiName(self, inputId):
if((inputId < len(self._midiInputList)) and (inputId >= 0)):
midiName, midiOpen, pygameMidiId = self._midiInputList[inputId] #@UnusedVariable
return midiName
return ""
def stopMidiOverNetProcess(self):
if(self._midiOverNetProcess != None):
print "Stopping midiUdpSender"
self._midiOverNetQueue.put("QUIT")
self._midiOverNetProcess.join(20.0)
if(self._midiOverNetProcess.is_alive()):
print "midiUdpSender did not respond to quit command. Terminating."
self._midiOverNetProcess.terminate()
self._midiOverNetProcess = None
def getMidiStatus(self):
run = True
clocks = 0
midis = 0
daemon = 0
if(self._midiOverNetProcess != None):
while(run):
try:
status = self._statusQueue.get_nowait()
if(status == 1):
clocks += 1
elif(status == 2):
midis += 1
daemon += 1
except Empty:
run = False
try:
for i in range(512): #@UnusedVariable
message = self._debugPrintQueue.get_nowait()
print "midiOverNetProcess: " + message
except Empty:
pass
return (daemon, clocks, midis)
def test_distributed_with_partition_servers(self):
sync_path = TemporaryDirectory()
self.addCleanup(sync_path.cleanup)
entity_name = "e"
relation_config = RelationSchema(name="r",
lhs=entity_name,
rhs=entity_name)
base_config = ConfigSchema(
dimension=10,
relations=[relation_config],
entities={entity_name: EntitySchema(num_partitions=4)},
entity_path=None, # filled in later
edge_paths=[], # filled in later
checkpoint_path=self.checkpoint_path.name,
num_machines=2,
num_partition_servers=1,
distributed_init_method="file://%s" %
os.path.join(sync_path.name, "sync"),
)
dataset = generate_dataset(base_config,
num_entities=100,
fractions=[0.4])
self.addCleanup(dataset.cleanup)
train_config = attr.evolve(
base_config,
entity_path=dataset.entity_path.name,
edge_paths=[dataset.relation_paths[0].name],
)
# Just make sure no exceptions are raised and nothing crashes.
trainer0 = Process(name="trainer#0",
target=train,
args=(train_config, ),
kwargs={"rank": 0})
trainer1 = Process(name="trainer#1",
target=train,
args=(train_config, ),
kwargs={"rank": 1})
partition_server = Process(name="partition server#0",
target=run_partition_server,
args=(train_config, ),
kwargs={"rank": 0})
# FIXME In Python 3.7 use kill here.
self.addCleanup(trainer0.terminate)
self.addCleanup(trainer1.terminate)
self.addCleanup(partition_server.terminate)
trainer0.start()
trainer1.start()
partition_server.start()
done = [False, False]
while not all(done):
time.sleep(1)
if not trainer0.is_alive() and not done[0]:
self.assertEqual(trainer0.exitcode, 0)
done[0] = True
if not trainer1.is_alive() and not done[1]:
self.assertEqual(trainer1.exitcode, 0)
done[1] = True
if not partition_server.is_alive():
self.fail("Partition server died with exit code %d" %
partition_server.exitcode)
partition_server.terminate() # Cannot be shut down gracefully.
partition_server.join()
logging.info("Partition server died with exit code %d",
partition_server.exitcode)
self.assertCheckpointWritten(train_config, version=1)
#父进程创建消息队列，并传给各个子进程：
q = Queue(3)
pw = Process(target=write, args=(
q,
3,
))
pr = Process(target=read, args=(q, ))
#启动子进程pw,写入：
pw.start()
#启动子进程pr，读取：
pr.start()
is_running('pw')
is_running('pr')
#等待pw结束：
pw.join()
print('Is writing process alive?', pw.is_alive())
is_running('pw')
is_running('pr')
time.sleep(5) #等待pr读完最后一个数据
#pr进程是死循环，若无消息读取，会一直阻塞自己，只能强制终止：
print('Is reading process alive?', pr.is_alive())
is_running('pw')
is_running('pr')
pr.terminate()
print('Main Process kills the reading process')
print('Is reading process alive?', pr.is_alive())
is_running('pw')
is_running('pr')
print('Is reading process alive?', pr.is_alive())
print('All data have been written and read.')
print('Is reading process alive?', pr.is_alive())
frame = pygame.surfarray.make_surface(back)
#frame = pygame.transform.scale(frame, tuple(sc_shape))
rects.append(screen.blit(frame, (0, 0)))
pgClock.tick(pgFps)
pygame.display.update(rects)
if __name__ == "__main__":
cam_q = Queue() # the Queue for camera img
is_Running = Value(c_bool, True)
saving = Value(c_bool, False)
man = Manager()
form = man.list([None, None])
camera = Process(args=(cam_q, is_Running, form, ), target=grabCam,
kwargs={'mode': "camera", "c_num": 0, "secs": 100, "saving": saving})
windows = Process(target=showImg, args=(cam_q, is_Running, form,),
kwargs={'mode': "traj", 'calibrate': False, "full": False, "saving": saving})
windows.start()
camera.start()
while True:
time.sleep(1)
if not (camera.is_alive() and windows.is_alive()):
windows.terminate()
camera.terminate()
camera.join()
windows.join()
break
class BaseIndex(object):
"""Search Index Interface
"""
config = {
'async_reindex': 0,
'ignore_errors': 0,
'reindex_check': '1,1',
'number_of_shards': 4,
'index_parallel': 1,
'index_speed': 100,
'error_wait': 10,
}
index_settings_keys = ['number_of_shards']
allow_async_reindex = False
force_next_reindex = False
magic_exit_code = 84
check_all_field = True
skip_check_count = False
reindex_process = None
next_index_name = None
last_current_index = None
def __init__(self, engine, source, config={}):
assert(self.__index_name__)
if config:
self.config.update(config)
self.config['index_speed'] = float(self.config['index_speed'])
rename_key = 'rename_' + self.__index_name__
if rename_key in self.config:
self.__index_name__ = self.config[rename_key]
if self.allow_async_reindex:
self.allow_async_reindex = self.config['async_reindex']
self.set_reindex_options(self.config.get('reindex', ''),
self.config.get('reindex_check', ''))
self.source = source
self.engine = engine
self.engine.add_index(self)
self.after_init()
def __del__(self):
self.stop_childs()
def __str__(self):
return self.__index_name__
def __repr__(self):
return self.__index_name__
@classmethod
def name(klass):
return klass.__index_name__
@staticmethod
def index_created_time(name):
prefix, suffix = name.rsplit('_', 1)
try:
s_time = time.strptime(suffix, SUFFIX_FORMAT)
suffix = time.mktime(s_time)
except:
suffix = 0
return suffix
@property
def current_index(self):
key = self.__index_name__
return self.engine.get_index(key)
def index_age(self, name=None):
if not name:
name = self.current_index
if not name:
return time.time()
suffix = BaseIndex.index_created_time(name)
return int(time.time() - int(suffix))
def set_reindex_options(self, reindex_period, reindex_check):
# reindex_period - two digits, first is age in days, seconds is weekday
if reindex_period:
self.max_age, self.reindex_day = map(int, reindex_period.split(','))
self.max_age *= 86400
# reindex_check - two digits, first is min docs count, second is max age of last doc
if reindex_check:
self.rc_mindocs, self.rc_max_age = map(int, reindex_check.split(','))
self.rc_max_age *= 86400
def after_init(self):
pass
def need_reindex(self):
return not self.current_index
def create_index(self, name):
return
def new_index(self, is_async=False):
index_key = self.__index_name__
index_key_next = "{}.next".format(index_key)
# try restore last index (in case of crash)
name = self.engine.get_index(index_key_next)
current_index = self.current_index
if current_index and name <= current_index:
name = None
if name and not name.startswith(index_key):
name = None
if name and not self.engine.index_exists(name):
name = None
if name and self.index_age(name) > 5 * 24 * 3600:
name = None
if name:
logger.info("Use already created index %s", name)
else:
suffix = time.strftime(SUFFIX_FORMAT)
name = "{}_{}".format(index_key, suffix)
self.create_index(name)
self.engine.set_index(index_key_next, name)
# check current not same to new
assert name != current_index, "same index name"
return name
def delete_index(self, name):
index_key = self.__index_name__
index_key_prev = "{}.prev".format(index_key)
self.engine.set_index(index_key_prev, name)
def set_current(self, name):
if self.engine.should_exit:
return
index_key = self.__index_name__
old_index = self.current_index
if name != old_index:
logger.info("Change current %s index %s -> %s",
index_key, old_index, name)
if self.check_index(name):
self.engine.set_index(index_key, name)
self.last_current_index = name
# assert(self.current_index == name)
if old_index:
self.delete_index(old_index)
# remove index.next key
index_key_next = "{}.next".format(index_key)
if self.engine.get_index(index_key_next) == name:
self.engine.set_index(index_key_next, '')
return name
def test_exists(self, index_name, info):
return self.engine.test_exists(index_name, info)
def test_noindex(self, item):
return False
def before_index_item(self, item):
return True
def handle_error(self, error, exc_info):
if self.config['ignore_errors']:
logger.error("%s %s (ignored)", type(error).__name__, str(error))
else:
raise exc_info[0], exc_info[1], exc_info[2]
def indexing_stat(self, index_name, fetched, indexed, iter_count, last_date):
if not last_date and fetched < 10 and indexed < 1:
return
logger.info("[%s] Fetched %d indexed %d last %s",
index_name, fetched, indexed, last_date or '-')
pause = float(iter_count) / float(self.config['index_speed'] or 1)
if pause > 1:
logger.info("Wait %1.1f sec", pause)
if pause > 0.01:
self.engine.sleep(pause)
def index_item(self, index_name, item):
if not item.get('meta') or not item.get('data'):
logger.error("[%s] No data %s", index_name, str(item))
return None
if item['meta']['dateModified'] != item['data']['dateModified']:
logger.error("[%s] dateModified mismatch %s", index_name, str(item))
return None
if self.test_noindex(item):
if self.engine.debug:
logger.debug("[%s] Noindex %s %s", index_name,
item['data'].get('id', ''),
item['data'].get('tenderID', ''))
return None
self.before_index_item(item)
return self.engine.index_item(index_name, item)
def index_source(self, index_name=None, reset=False, reindex=False):
if self.engine.slave_mode:
if not self.engine.heartbeat(self.source):
self.engine.sleep(1)
return
if not index_name:
# also check maybe current index was changed
if self.last_current_index != self.current_index:
self.last_current_index = self.current_index
reset = True
index_name = self.current_index
if not index_name:
if not self.reindex_process:
logger.warning("No current index for %s", repr(self))
return
if reset or self.source.need_reset():
self.source.reset()
index_count = 0
total_count = 0
# heartbeat always True in master mode
# heartbeat return True in slave mode only if master fail
while self.engine.heartbeat(self.source):
info = {}
iter_count = 0
items_list = self.source.items()
if not items_list:
break
for info in items_list:
if self.engine.should_exit:
return
if not self.test_exists(index_name, info):
try:
item = self.source.get(info)
if self.index_item(index_name, item):
index_count += 1
except Exception as e:
self.handle_error(e, sys.exc_info())
# update statistics
iter_count += 1
total_count += 1
# update heartbeat for long indexing
if iter_count >= 500:
self.indexing_stat(
index_name, total_count, index_count,
iter_count, info.get('dateModified', '-'))
if not self.engine.heartbeat(self.source):
break
iter_count = 0
if self.engine.should_exit:
return
# break if nothing iterated
if iter_count > 0:
self.indexing_stat(index_name, total_count, index_count,
iter_count, info.get('dateModified', '-'))
elif getattr(self.source, 'last_skipped', None):
last_skipped = self.source.last_skipped or ""
logger.info("[%s] Fetched %d, last_skipped %s",
index_name, total_count, last_skipped or '-')
elif not info:
break
# break on each iteration if not in full reindex mode
if not reindex and self.config['index_parallel'] and index_count < total_count:
logger.debug("[%s] Swith loop", index_name)
return
return index_count
def stop_childs(self):
if self.source:
self.source.should_exit = True
if not self.reindex_process or not self.reindex_process.pid:
return
if self.reindex_process.pid == os.getpid():
return
logger.info("Terminate subprocess %s pid %s",
self.reindex_process.name, str(self.reindex_process.pid))
try:
self.reindex_process.terminate()
except (AttributeError, OSError):
pass
def check_subprocess(self):
if self.reindex_process:
self.reindex_process.join(1)
if not self.reindex_process or self.reindex_process.is_alive():
return
if self.reindex_process.exitcode == self.magic_exit_code:
logger.info("Reindex-%s subprocess success, reset source",
self.__index_name__)
if self.next_index_name:
self.set_current(self.next_index_name)
self.next_index_name = None
self.source.reset()
else:
logger.error("Reindex-%s subprocess fail, exitcode = %d",
self.__index_name__, self.reindex_process.exitcode)
# close process
self.reindex_process = None
def check_index(self, index_name):
if not index_name or self.engine.should_exit:
return False
# check index mappings by check _all field
if self.check_all_field:
try:
info = self.engine.index_info(index_name)
stat = self.engine.index_stats(index_name)
except Exception as e:
logger.error("[%s] Check index failed: %s", index_name, str(e))
self.force_next_reindex = True
return False
doc_type = self.source.__doc_type__
if '_all' not in info['mappings'][doc_type]:
logger.error("[%s] Check index failed: _all field not found, please reindex!",
index_name)
self.force_next_reindex = True
return False
if self.skip_check_count:
if self.check_all_field:
logger.info("[%s] Total docs %d, last indexed not tested",
index_name, stat['docs']['count'])
return True
# check index docs count afetr reindex
body = {
"query": {"match_all": {}},
"sort": {"dateModified": {"order": "desc"}}
}
try:
res = self.engine.search(body, start=0, limit=1, index=index_name)
except:
res = None
if not res or not res.get('items'):
logger.error("[%s] Check failed: empty or corrupted index", index_name)
return False
logger.info("[%s] Total docs %d, last indexed %s",
index_name, res['total'], res['items'][0]['dateModified'])
if self.rc_mindocs and res['total'] < self.rc_mindocs:
logger.error("[%s] Check index failed: not enought docs %d, required %d",
index_name, res['total'], self.rc_mindocs)
return False
if self.rc_max_age:
min_date = datetime.now() - timedelta(seconds=self.rc_max_age)
iso_min_date = min_date.isoformat()
last_indexed = res['items'][0]['dateModified']
if last_indexed < iso_min_date:
logger.error("[%s] Check index failed: last indexed is too old %s, "+
"required %s", index_name, last_indexed, iso_min_date)
return False
return True
def check_on_start(self):
if not self.current_index:
return True
if self.check_index(self.current_index):
return True
if not self.engine.index_exists(self.current_index):
self.set_current('')
def async_reindex(self):
logger.info("*** Start Reindex-%s in subprocess",
self.__index_name__)
# reconnect elatic and prevent future stop_childs
self.engine.start_in_subprocess()
self.index_source(self.next_index_name, reset=True, reindex=True)
if self.check_index(self.next_index_name):
exit_code = self.magic_exit_code
else:
exit_code = 1
# exit with specific code to signal master process reset source
logger.info("*** Exit subprocess")
sys.exit(exit_code)
def reindex(self):
# check reindex process is alive
if self.reindex_process and self.reindex_process.is_alive():
return
# clear reindex flag
if self.force_next_reindex:
self.force_next_reindex = False
logger.info("Need reindex %s", self.__index_name__)
# create new index and save name
self.next_index_name = self.new_index()
# reindex in old-way sync mode
if not self.allow_async_reindex:
self.index_source(self.next_index_name, reset=True, reindex=True)
if self.check_index(self.next_index_name):
self.set_current(self.next_index_name)
return True
return False
# reindex in async mode, start new reindex process
proc_name = "Reindex-%s" % self.__index_name__
self.reindex_process = Process(
target=self.async_reindex,
name=proc_name)
self.reindex_process.daemon = True
self.reindex_process.start()
# wait for child
retry_count = 0
while not self.reindex_process.is_alive() and retry_count < 30:
self.engine.sleep(1)
retry_count += 1
# check child is alive
if self.reindex_process.is_alive():
logger.info("Subprocess started %s pid %d",
self.reindex_process.name, self.reindex_process.pid)
else:
logger.error("Can't start subprocess")
def process(self, allow_reindex=True):
if self.engine.should_exit:
return
if self.reindex_process:
self.check_subprocess()
if self.need_reindex() and allow_reindex:
self.reindex()
return self.index_source()
t = threading.Thread(target=io, args=(run, q))
t.start()
# Start display process
p = Process(target=display, args=('bob', q))
p.start()
#custom code here ..
for i in range(0, 10):
#time.sleep(1)
print("wait")
print("Close Graph to stop and save data")
while p.is_alive():
time.sleep(0.33)
pmtoff()
laseroff()
run.clear()
print("Waiting for scheduler thread to join...")
t.join()
print("Waiting for graph window process to join...")
err(mydll.rcDisconnect())
print("Disconnecting laser!")
p.join()
print("Finished!")
class RTLSDRTCP(Device):
BYTES_PER_SAMPLE = 2 # RTLSDR device produces 8 bit unsigned IQ data
MAXDATASIZE = 65536
ENDIAN = "big"
RTL_TCP_CONSTS = [
"NULL", "centerFreq", "sampleRate", "tunerGainMode", "tunerGain",
"freqCorrection", "tunerIFGain", "testMode", "agcMode",
"directSampling", "offsetTuning", "rtlXtalFreq", "tunerXtalFreq",
"gainByIndex", "bandwidth", "biasTee"
]
def __init__(self, freq, gain, srate, device_number, is_ringbuffer=False):
super().__init__(0, freq, gain, srate, is_ringbuffer)
self.open() #open("127.0.0.1", 1234)
self.success = 0
self.is_receiving_p = Value('i', 0)
"""
Shared Value to communicate with the receiving process.
"""
#self.bandwidth_is_adjustable = hasattr(rtlsdr, "set_tuner_bandwidth") # e.g. not in Manjaro Linux / Ubuntu 14.04
self._max_frequency = 6e9
self._max_sample_rate = 3200000
self._max_frequency = 6e9
self._max_bandwidth = 3200000
self._max_gain = 500 # Todo: Consider get_tuner_gains for allowed gains here
self.device_number = device_number
def open(self, hostname="127.0.0.1", port=1234):
try:
# Create socket and connect
self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM,
socket.IPPROTO_TCP)
self.sock.settimeout(1.0) # Timeout 1s
self.sock.connect((hostname, port))
# Receive rtl_tcp initial data
init_data = self.sock.recv(self.MAXDATASIZE)
if len(init_data) != 12:
return False
if init_data[0:4] != b'RTL0':
return False
# Extract tuner name
tuner_number = int.from_bytes(init_data[4:8], self.ENDIAN)
if tuner_number == 1:
self.tuner = "E4000"
elif tuner_number == 2:
self.tuner = "FC0012"
elif tuner_number == 3:
self.tuner = "FC0013"
elif tuner_number == 4:
self.tuner = "FC2580"
elif tuner_number == 5:
self.tuner = "R820T"
elif tuner_number == 6:
self.tuner = "R828D"
else:
self.tuner = "Unknown"
# Extract IF and RF gain
self.if_gain = int.from_bytes(init_data[8:10], self.ENDIAN)
self.rf_gain = int.from_bytes(init_data[10:12], self.ENDIAN)
except OSError as e:
logger.info("Could not connect to rtl_tcp", hostname, port, "(",
str(e), ")")
def close(self):
self.sock.close()
def set_parameter(self, param, value): # returns error (True/False)
msg = self.RTL_TCP_CONSTS.index(param).to_bytes(
1, self.ENDIAN) # Set param at bits 0-7
msg += value.to_bytes(4, self.ENDIAN) # Set value at bits 8-39
try:
self.sock.sendall(msg) # Send data to rtl_tcp
except OSError as e:
logger.info("Could not set parameter", param, value, msg, "(",
str(e), ")")
return True
return False
def start_rx_mode(self):
self.init_recv_buffer()
self.is_open = True
self.is_receiving = True
self.receive_process = Process(
target=receive_sync,
args=(self.child_conn, self.device_number, self.frequency,
self.sample_rate, self.gain))
self.receive_process.daemon = True
self._start_read_rcv_buffer_thread()
self.receive_process.start()
def stop_rx_mode(self, msg):
self.is_receiving = False
self.parent_conn.send("stop")
logger.info("RTLSDRTCP: Stopping RX Mode: " + msg)
if hasattr(self, "receive_process"):
self.receive_process.join(0.3)
if self.receive_process.is_alive():
logger.warning(
"RTLSDRTCP: Receive process is still alive, terminating it"
)
self.receive_process.terminate()
self.receive_process.join()
self.parent_conn, self.child_conn = Pipe()
if hasattr(self, "read_queue_thread"
) and self.read_recv_buffer_thread.is_alive():
try:
self.read_recv_buffer_thread.join(0.001)
logger.info("RTLSDRTCP: Joined read_queue_thread")
except RuntimeError:
logger.error("RTLSDRTCP: Could not join read_queue_thread")
def read_sync(self):
return self.sock.recv(self.MAXDATASIZE)
def set_device_frequency(self, frequency):
error = self.set_parameter("centerFreq", int(frequency))
self.log_retcode(error, "Set center frequency")
return error
def set_device_sample_rate(self, sample_rate):
error = self.set_parameter("sampleRate", int(sample_rate))
self.log_retcode(error, "Set sample rate")
return error
def set_freq_correction(self, ppm):
error = self.set_parameter("freqCorrection", int(ppm))
self.log_retcode(error, "Set frequency correction")
return error
def set_offset_tuning(self, on: bool):
error = self.set_parameter("offsetTuning", on)
self.log_retcode(error, "Set offset tuning")
return error
def set_gain_mode(self, manual: bool):
error = self.set_parameter("tunerGainMode", manual)
self.log_retcode(error, "Set gain mode manual")
return error
def set_if_gain(self, gain):
error = self.set_parameter("tunerIFGain", int(gain))
self.log_retcode(error, "Set IF gain")
return error
def set_gain(self, gain):
error = self.set_parameter("tunerGain", int(gain))
self.log_retcode(error, "Set tuner gain")
return error
def set_bandwidth(self, bandwidth):
error = self.set_parameter("bandwidth", int(bandwidth))
self.log_retcode(error, "Set tuner bandwidth")
return error
@staticmethod
def unpack_complex(buffer, nvalues: int):
"""
The raw, captured IQ data is 8 bit unsigned data.
:return:
"""
result = np.empty(nvalues, dtype=np.complex64)
unpacked = np.frombuffer(buffer,
dtype=[('r', np.uint8), ('i', np.uint8)])
result.real = (unpacked['r'] / 127.5) - 1.0
result.imag = (unpacked['i'] / 127.5) - 1.0
return result
@staticmethod
def pack_complex(complex_samples: np.ndarray):
return (127.5 * (complex_samples.view(np.float32) + 1.0)).astype(
np.uint8).tostring()
class AppTask(WatchTask):
template_files = '.html', '.jinja', '.jinja2'
def __init__(self, config: Config, loop: asyncio.AbstractEventLoop):
self._config = config
self._reloads = 0
self._session = ClientSession(loop=loop)
self._runner = None
super().__init__(self._config.code_directory, loop)
async def _run(self):
try:
self._start_dev_server()
running = True
while running:
try:
service_next = self._awatch.__anext__()
common_next = self._commonql_awatch.__anext__()
done, pending = await asyncio.wait((service_next, common_next), return_when=asyncio.FIRST_COMPLETED)
for pending_task in pending:
pending_task.cancel()
changes = done.pop().result()
except StopAsyncIteration:
running = False
else:
self._reloads += 1
if any(f.endswith('.py') for _, f in changes):
logger.debug('%d changes, restarting server', len(changes))
self._stop_dev_server()
self._start_dev_server()
await self._src_reload_when_live()
elif len(changes) > 1 or any(f.endswith(self.template_files) for _, f in changes):
# reload all pages
await src_reload(self._app)
else:
# a single (non template) file has changed, reload a single file.
await src_reload(self._app, changes.pop()[1])
except Exception as exc:
logger.exception(exc)
await self._session.close()
raise AiohttpDevException('error running dev server')
async def _src_reload_when_live(self, checks=20):
if self._app[WS]:
url = 'http://localhost:{.main_port}/?_checking_alive=1'.format(self._config)
logger.debug('checking app at "%s" is running before prompting reload...', url)
for i in range(checks):
await asyncio.sleep(0.1, loop=self._app.loop)
try:
async with self._session.get(url):
pass
except OSError as e:
logger.debug('try %d | OSError %d app not running', i, e.errno)
else:
logger.debug('try %d | app running, reloading...', i)
await src_reload(self._app)
return
def _start_dev_server(self):
act = 'Start' if self._reloads == 0 else 'Restart'
logger.info('%sing dev server at http://%s:%s ●', act, self._config.host, self._config.main_port)
try:
tty_path = os.ttyname(sys.stdin.fileno())
except OSError: # pragma: no branch
# fileno() always fails with pytest
tty_path = '/dev/tty'
except AttributeError:
# on windows, without a windows machine I've no idea what else to do here
tty_path = None
self._process = Process(target=serve_main_app, args=(self._config, tty_path))
self._process.start()
def _stop_dev_server(self):
if self._process.is_alive():
logger.debug('stopping server process...')
os.kill(self._process.pid, signal.SIGINT)
self._process.join(5)
if self._process.exitcode is None:
logger.warning('process has not terminated, sending SIGKILL')
os.kill(self._process.pid, signal.SIGKILL)
self._process.join(1)
else:
logger.debug('process stopped')
else:
logger.warning('server process already dead, exit code: %s', self._process.exitcode)
async def close(self, *args):
self.stopper.set()
self._stop_dev_server()
await asyncio.gather(super().close(), self._session.close())
def run_task(task, internal_storage):
"""
Runs a single job within a separate process
"""
start_tstamp = time.time()
setup_lithops_logger(task.log_level)
backend = os.environ.get('__LITHOPS_BACKEND', '')
logger.info("Lithops v{} - Starting {} execution".format(__version__, backend))
logger.info("Execution ID: {}/{}".format(task.job_key, task.id))
if task.runtime_memory:
logger.debug('Runtime: {} - Memory: {}MB - Timeout: {} seconds'
.format(task.runtime_name, task.runtime_memory, task.execution_timeout))
else:
logger.debug('Runtime: {} - Timeout: {} seconds'.format(task.runtime_name, task.execution_timeout))
env = task.extra_env
env['LITHOPS_WORKER'] = 'True'
env['PYTHONUNBUFFERED'] = 'True'
env['LITHOPS_CONFIG'] = json.dumps(task.config)
env['__LITHOPS_SESSION_ID'] = '-'.join([task.job_key, task.id])
os.environ.update(env)
call_status = CallStatus(task.config, internal_storage)
call_status.response['worker_start_tstamp'] = start_tstamp
call_status.response['host_submit_tstamp'] = task.host_submit_tstamp
call_status.response['call_id'] = task.id
call_status.response['job_id'] = task.job_id
call_status.response['executor_id'] = task.executor_id
show_memory_peak = strtobool(os.environ.get('SHOW_MEMORY_PEAK', 'False'))
try:
# send init status event
call_status.send('__init__')
if show_memory_peak:
mm_handler_conn, mm_conn = Pipe()
memory_monitor = Thread(target=memory_monitor_worker, args=(mm_conn, ))
memory_monitor.start()
task.stats_file = os.path.join(task.task_dir, 'task_stats.txt')
handler_conn, jobrunner_conn = Pipe()
taskrunner = TaskRunner(task, jobrunner_conn, internal_storage)
logger.debug('Starting TaskRunner process')
jrp = Process(target=taskrunner.run) if is_unix_system() else Thread(target=taskrunner.run)
jrp.start()
jrp.join(task.execution_timeout)
logger.debug('TaskRunner process finished')
if jrp.is_alive():
# If process is still alive after jr.join(job_max_runtime), kill it
try:
jrp.terminate()
except Exception:
# thread does not have terminate method
pass
msg = ('Function exceeded maximum time of {} seconds and was '
'killed'.format(task.execution_timeout))
raise TimeoutError('HANDLER', msg)
if show_memory_peak:
mm_handler_conn.send('STOP')
memory_monitor.join()
peak_memory_usage = int(mm_handler_conn.recv())
logger.info("Peak memory usage: {}".format(sizeof_fmt(peak_memory_usage)))
call_status.response['peak_memory_usage'] = peak_memory_usage
if not handler_conn.poll():
logger.error('No completion message received from JobRunner process')
logger.debug('Assuming memory overflow...')
# Only 1 message is returned by jobrunner when it finishes.
# If no message, this means that the jobrunner process was killed.
# 99% of times the jobrunner is killed due an OOM, so we assume here an OOM.
msg = 'Function exceeded maximum memory and was killed'
raise MemoryError('HANDLER', msg)
if os.path.exists(task.stats_file):
with open(task.stats_file, 'r') as fid:
for l in fid.readlines():
key, value = l.strip().split(" ", 1)
try:
call_status.response[key] = float(value)
except Exception:
call_status.response[key] = value
if key in ['exception', 'exc_pickle_fail', 'result', 'new_futures']:
call_status.response[key] = eval(value)
except Exception:
# internal runtime exceptions
print('----------------------- EXCEPTION !-----------------------')
traceback.print_exc(file=sys.stdout)
print('----------------------------------------------------------')
call_status.response['exception'] = True
pickled_exc = pickle.dumps(sys.exc_info())
pickle.loads(pickled_exc) # this is just to make sure they can be unpickled
call_status.response['exc_info'] = str(pickled_exc)
finally:
call_status.response['worker_end_tstamp'] = time.time()
# Flush log stream and save it to the call status
task.log_stream.flush()
with open(task.log_file, 'rb') as lf:
log_str = base64.b64encode(zlib.compress(lf.read())).decode()
call_status.response['logs'] = log_str
call_status.send('__end__')
# Unset specific env vars
for key in task.extra_env:
os.environ.pop(key, None)
os.environ.pop('__LITHOPS_TOTAL_EXECUTORS', None)
logger.info("Finished")
def crawl(self, cp_engines=None):
def _browser_launch(queue, cp_engines, opts, urls, users, browser_id,
logfile, workdir, pt_suite):
cpbr = CPBrowserRunner(cp_engines, opts, urls, users, browser_id,
logfile, workdir, pt_suite)
if browser_id:
try:
cpbr.run()
except RuntimeError:
logging.error(traceback.format_exc())
else:
cpbr.run()
if queue:
queue.put(cpbr)
return cpbr
if not cp_engines:
cp_engines = [CPEngineBase]
if self.opts is None:
raise CPCrawlerException(
"the init_opts() method must be called before run()")
users = self._gen_users(self.opts.user)
if users and len(users) > 1 and len(users) > self.opts.real_browsers:
raise CPCrawlerException(
"ERROR: number of users (%d) can't be higher than number of browsers (%d)"
% (len(users), self.opts.real_browsers))
if self.opts.real_browsers > 0:
real_browsers = []
cpbr_objs = {}
for i in range(1, self.opts.real_browsers + 1):
cpbr_objs[i] = Queue()
b = Process(target=_browser_launch,
args=(cpbr_objs[i], cp_engines, self.opts,
self.urls, users, i, self.logfile,
self.workdir, self.pt_suite))
b.start()
real_browsers.append(b)
time.sleep(self.opts.instances_delay)
delay = 3
telemetry_fname = os.path.join(self.workdir, "telemetry.log")
report_fname = os.path.join(self.workdir, "report.html")
print("")
print("Notes:")
print(" * Tests executed by %d %s browsers%s%s" %
(self.opts.real_browsers, self.opts.browser,
" and %d python browsers" %
(self.opts.real_browsers * self.opts.python_browsers)
if self.opts.python_browsers else "",
" on %d users:\n %s" %
(len(users), "\n ".join(users)) if users else ""))
print(" * All data is stored in %s" % self.workdir)
print(" * The %s file is updated every %d sec" %
(report_fname, delay))
print("")
while True:
time.sleep(delay)
all_dead = True
for b in real_browsers:
if not b.is_alive():
continue
all_dead = False
if all_dead:
break
page_stats_summary = PageStatsSummary()
for i in range(1, self.opts.real_browsers + 1):
cpbr = cpbr_objs[i].get()
if os.path.exists(cpbr.stderr_fname):
f = open(cpbr.stderr_fname, 'r')
data = f.read()
f.close()
if data:
print("browser.%d stderr:\n%s" % (i, data))
for ps in cpbr.page_stats_summary.page_stats:
page_stats_summary.add_page_stats(ps)
report_fname = os.path.join(self.workdir, "report.html")
else:
cpbr = _browser_launch(None, cp_engines, self.opts, self.urls,
users, 0, self.logfile, self.workdir,
self.pt_suite)
page_stats_summary = cpbr.page_stats_summary
return page_stats_summary.page_stats
class Transformer:
'''The transformer'''
TICK = 0.2
WORLD_TICK = 0.1
def __init__(self,
initial_state: (float, float),
goal_state: (float, float),
agent_ctor,
action_velocities=((-0.5, 0.5), (-0.5, 0.5)),
action_time_max=0.5):
self.transformers = {}
self.state = TransformerIState.WaitAction
self.reverse_actions = []
self.queue = Queue()
self.thread = None
self.agent = agent_ctor(self, initial_state, goal_state, action_velocities, action_time_max)
self.other_starts_goals = []
self.log = logging.getLogger(f'tf-{self.agent.id}')
self.need_acknowledgement = set()
self.conflicts_with = set()
self.current_action = None
self.is_leader = self.get_id() == 0
self.tick_count = 0
self.reverse_schedule = None
self.awaited_reversions = None
self.pose = self.agent.pose
def compute_reverse_schedule(self, action_log):
'''Figure out how far back to go and what actions can run in parallel'''
conflicted_agents = self.conflicts_with.copy()
back_idx = -1
while conflicted_agents:
layer = action_log[back_idx]
return []
def update_ticks(self, _, count):
'''Update the internal tick counter'''
self.tick_count = max(count, self.tick_count)
def register_transformer(self, transformer):
'''Register a new transformer'''
self.transformers[transformer.get_id()] = TransformerState(transformer.get_start(), [],
None, False, None,
transformer.get_queue(), False)
self.other_starts_goals.append(transformer.get_start())
self.other_starts_goals.append(transformer.get_goal())
def get_start(self):
'''Return the agent's start'''
return self.agent.pose
def get_goal(self):
'''Return the agent's goal'''
return self.agent.goal
def get_id(self) -> int:
'''Getter for this transformer's ID (same as its agent's ID)'''
return self.agent.id
def get_queue(self):
'''Getter for this transformer's queue'''
return self.queue
def get_agent(self):
'''Getter for this transformer's agent'''
return self.agent
def start(self):
'''Start the process'''
if not self.thread:
other_starts_goals = np.vstack(self.other_starts_goals)
self.thread = Process(target=self.run, args=(self.queue, other_starts_goals))
self.thread.start()
return self.thread
def stop(self):
'''Stop the process'''
if self.thread and self.thread.is_alive():
self.state = TransformerIState.AtGoal
self.thread.join()
def agent_request(self, other_starts_goals, _, action_req):
'''Handle an action request from an agent'''
if self.state is not TransformerIState.WaitAction:
self.log.warning('Ignoring action request; mode is %s', self.state)
return
self.log.info('Received agent request')
self.conflicts_with.clear()
action, start_pose = action_req
self.current_action = action_req
for step in np.arange(Transformer.WORLD_TICK, action.time, Transformer.WORLD_TICK):
pose = start_pose + step * action.vel_vec
pose_array = np.full_like(other_starts_goals, pose)
# Check that the action doesn't conflict with starts/goals
if np.linalg.norm(pose_array - other_starts_goals,
-np.inf) <= Agent.GOAL_THRESHOLD + Agent.RADIUS:
# We come too close to some start or goal; reject
self.log.debug('Agent request conflicts with start or goal; rejecting')
self.agent.queue.put(
Msg(SenderTypes.Transformer, MessageTypes.ActionReject, self.get_id(), action_req))
return
# Check that the action doesn't conflict with known current/potential actions
for (tf_id, transformer) in self.transformers.items():
if test_conflict((action, pose), (ZERO_ACTION, transformer.pose)):
# We'd run into where an agent currently is
self.agent.queue.put(
Msg(SenderTypes.Transformer, MessageTypes.ActionReject, self.get_id(), action_req))
return
if transformer.curr_action:
tf_action, tf_pose = transformer.curr_action
# if transformer.running:
# ticks_elapsed = self.tick_count - transformer.start_tick
# tf_pose += tf_action.vel_vec * ticks_elapsed * Transformer.WORLD_TICK
if test_conflict((action, pose), (tf_action, tf_pose)):
# The action conflicts with a currently executing or approved action
self.conflicts_with.add(tf_id)
self.log.debug('Agent request conflicts with approved action; rejecting')
self.agent.queue.put(
Msg(SenderTypes.Transformer, MessageTypes.ActionReject, self.get_id(), action_req))
return
# We can skip the reverse schedulability check because this is the special case where actions
# are exactly reversible
# Check with the other transformers
self.need_acknowledgement = set(
tf_id for tf_id in self.transformers if not self.transformers[tf_id].at_goal)
if self.need_acknowledgement:
self.log.debug('Sending out for approval')
self.send_to_all(
Msg(SenderTypes.Transformer, MessageTypes.ActionRequest, self.get_id(), action_req))
self.log.debug('Need approval from %s', ic.format(self.need_acknowledgement))
self.state = TransformerIState.WaitAcknowledge
else:
self.start_action()
def send_to_all(self, msg):
'''Send the same message to all transformers'''
for transformer in self.transformers.values():
if not transformer.at_goal:
transformer.queue.put(msg)
def agent_blocked(self, _id, _data):
'''Handle the agent signalling they have no more actions to suggest'''
self.state = TransformerIState.Conflicted
self.send_to_all(Msg(SenderTypes.Transformer, MessageTypes.Conflicted, self.get_id()))
def agent_action_done(self, _id, data):
'''Handle the agent completing an action'''
if self.state not in (TransformerIState.RevertingFollow, TransformerIState.RevertingLeader):
self.state = TransformerIState.WaitAction
self.current_action = None
self.pose = data
self.send_to_all(
Msg(SenderTypes.Transformer, MessageTypes.ActionDone, self.get_id(), self.pose))
def agent_at_goal(self, _id, pose):
'''Handle the agent reaching its goal'''
self.log.success('Reached goal!')
self.state = TransformerIState.AtGoal
self.pose = pose
self.send_to_all(Msg(SenderTypes.Transformer, MessageTypes.AtGoal, self.get_id(), pose))
def tf_request(self, tf_id, req):
'''Handle a transformer requesting an action'''
if self.state in (TransformerIState.RevertingFollow, TransformerIState.RevertingLeader):
self.log.warning('Received action request from %d while reverting. Ignoring it...', tf_id)
return
self.log.debug('Received request from %d', tf_id)
transformer = self.transformers[tf_id]
# Check for conflict
if self.current_action and test_conflict(self.current_action, req):
# The actions are incompatible!
self.log.notice('Action from %d conflicts with my action', tf_id)
if self.get_id() > tf_id:
self.log.notice('Approving action from %d and entering Conflicted', tf_id)
transformer.queue.put(
Msg(SenderTypes.Transformer, MessageTypes.ActionAccept, self.get_id()))
self.transformers[tf_id].curr_action = req
self.transformers[tf_id].running = False
self.conflicts_with.add(tf_id)
self.state = TransformerIState.Conflicted
self.send_to_all(Msg(SenderTypes.Transformer, MessageTypes.Conflicted, self.get_id()))
else:
self.log.debug('Rejecting action from %d', tf_id)
transformer.queue.put(
Msg(SenderTypes.Transformer, MessageTypes.ActionReject, self.get_id()))
return
if test_conflict((ZERO_ACTION, self.pose), req):
self.log.debug('Rejecting action from %d', tf_id)
transformer.queue.put(Msg(SenderTypes.Transformer, MessageTypes.ActionReject, self.get_id()))
return
transformer.queue.put(Msg(SenderTypes.Transformer, MessageTypes.ActionAccept, self.get_id()))
self.transformers[tf_id].curr_action = req
self.transformers[tf_id].running = False
def tf_reject(self, tf_id, _data):
'''Handle an action being rejected'''
self.log.debug('Action rejected by %d', tf_id)
self.agent.queue.put(
Msg(SenderTypes.Transformer, MessageTypes.ActionReject, self.get_id(), self.current_action))
self.current_action = None
self.state = TransformerIState.WaitAction
def start_action(self):
'''Utility method to trigger an approved action'''
self.log.debug('Starting action')
self.agent.queue.put(
Msg(SenderTypes.Transformer, MessageTypes.ActionAccept, self.get_id(),
self.current_action[0]))
self.send_to_all(Msg(SenderTypes.Transformer, MessageTypes.ActionStart, self.get_id()))
self.state = TransformerIState.WaitAction
def tf_accept(self, tf_id, _):
'''Handle an action being accepted'''
self.log.debug('Action accepted by %d', tf_id)
self.need_acknowledgement.discard(tf_id)
if not self.need_acknowledgement and self.state is TransformerIState.WaitAcknowledge:
self.reverse_actions.append((self.current_action, self.tick_count))
self.start_action()
def tf_cancel(self, tf_id, _):
'''Handle another transformer cancelling an action request'''
self.transformers[tf_id].curr_action = None
self.transformers[tf_id].running = False
if self.state is TransformerIState.Conflicted and tf_id in self.conflicts_with:
self.conflicts_with.remove(tf_id)
if not self.conflicts_with:
self.state = TransformerIState.WaitAcknowledge
self.start_action()
def tf_at_goal(self, tf_id, pose):
'''Handle another transformer reaching its goal'''
self.is_leader = tf_id == self.get_id() - 1
if tf_id in self.conflicts_with:
self.conflicts_with.remove(tf_id)
if not self.conflicts_with:
self.start_action()
self.transformers[tf_id].at_goal = True
self.transformers[tf_id].pose = pose
# A transformer reaching its goal may cause an action with pending accepts to be ready
self.tf_accept(tf_id, None)
def tf_action_start(self, tf_id, _):
'''Handle another transformer starting an action'''
self.transformers[tf_id].running = True
self.transformers[tf_id].start_tick = self.tick_count
self.transformers[tf_id].rev_path.append(
(self.transformers[tf_id].curr_action, self.tick_count))
def tf_action_done(self, tf_id, pose):
'''Handle another transformer finishing an action'''
self.transformers[tf_id].running = False
self.transformers[tf_id].start_tick = None
self.transformers[tf_id].curr_action = None
self.transformers[tf_id].pose = pose
if self.awaited_reversions:
self.awaited_reversions.discard(tf_id)
def tf_conflicted(self, tf_id, _):
'''Handle another transformer announcing it is conflicted'''
self.transformers[tf_id].conflicted = True
# Note: This is only to make the formatting happier
self_conflicted = self.state is TransformerIState.Conflicted
if self_conflicted and all(transformer.conflicted for transformer in self.transformers):
if self.is_leader:
self.send_to_all(Msg(SenderTypes.Transformer, MessageTypes.StartReversion, self.get_id()))
self.state = TransformerIState.RevertingLeader
self.reverse_schedule = self.compute_reverse_schedule(
[self.reverse_actions] +
[self.transformers[tf_id].rev_path for tf_id in self.transformers])
def tf_reversion_start(self, _id, _data):
'''Handle the leader signalling the start of reversion'''
self.state = TransformerIState.RevertingFollow
def tf_reversion_end(self, _id, _data):
'''Handle the leader signalling the end of reversion'''
self.state = TransformerIState.WaitAction
def tf_reversion_step(self, _id, reverse_action):
'''Handle the leader requiring a reverse action step'''
self.agent.queue.put(
Msg(SenderTypes.Transformer, MessageTypes.StepReversion, self.get_id(), reverse_action))
def run(self, queue, other_starts_goals):
'''The main loop'''
handlers = {
(SenderTypes.Simulator, MessageTypes.TickUpdate):
self.update_ticks,
(SenderTypes.Agent, MessageTypes.ActionRequest):
partial(self.agent_request, other_starts_goals),
(SenderTypes.Agent, MessageTypes.NoValidActions):
self.agent_blocked,
(SenderTypes.Agent, MessageTypes.ActionDone):
self.agent_action_done,
(SenderTypes.Agent, MessageTypes.AtGoal):
self.agent_at_goal,
(SenderTypes.Transformer, MessageTypes.ActionRequest):
self.tf_request,
(SenderTypes.Transformer, MessageTypes.ActionReject):
self.tf_reject,
(SenderTypes.Transformer, MessageTypes.ActionAccept):
self.tf_accept,
(SenderTypes.Transformer, MessageTypes.ActionCancelled):
self.tf_cancel,
(SenderTypes.Transformer, MessageTypes.AtGoal):
self.tf_at_goal,
(SenderTypes.Transformer, MessageTypes.ActionStart):
self.tf_action_start,
(SenderTypes.Transformer, MessageTypes.ActionDone):
self.tf_action_done,
(SenderTypes.Transformer, MessageTypes.Conflicted):
self.tf_conflicted,
(SenderTypes.Transformer, MessageTypes.StartReversion):
self.tf_reversion_start,
(SenderTypes.Transformer, MessageTypes.EndReversion):
self.tf_reversion_end,
(SenderTypes.Transformer, MessageTypes.StepReversion):
self.tf_reversion_step
}
while self.state is not TransformerIState.AtGoal:
while not queue.empty():
msg = queue.get()
msg.dispatch(handlers)
if self.state is TransformerIState.RevertingLeader:
if not self.reverse_schedule:
# TODO: Need to handle the case where the leader had to revert N steps
self.start_action()
self.send_to_all(Msg(SenderTypes.Transformer, MessageTypes.EndReversion, self.get_id()))
elif not self.awaited_reversions:
reversion_layer = self.reverse_schedule.pop()
self.awaited_reversions = {action.agent_id for action in reversion_layer}
for action in reversion_layer:
self.transformers[action.agent_id].queue.put(
Msg(SenderTypes.Transformer, MessageTypes.StepReversion, self.get_id(),
Action(action.agent_id, action.reverse, [], action.time)))
sleep(Transformer.TICK)
class Cluster(object):
def __init__(self, broker=None):
self.broker = broker or get_broker()
self.sentinel = None
self.stop_event = None
self.start_event = None
self.pid = current_process().pid
self.host = socket.gethostname()
self.timeout = Conf.TIMEOUT
signal.signal(signal.SIGTERM, self.sig_handler)
signal.signal(signal.SIGINT, self.sig_handler)
def start(self):
# Start Sentinel
self.stop_event = Event()
self.start_event = Event()
self.sentinel = Process(target=Sentinel,
args=(self.stop_event, self.start_event,
self.broker, self.timeout))
self.sentinel.start()
logger.info(_('Q Cluster-{} starting.').format(self.pid))
while not self.start_event.is_set():
sleep(0.1)
return self.pid
def stop(self):
if not self.sentinel.is_alive():
return False
logger.info(_('Q Cluster-{} stopping.').format(self.pid))
self.stop_event.set()
self.sentinel.join()
logger.info(_('Q Cluster-{} has stopped.').format(self.pid))
self.start_event = None
self.stop_event = None
return True
def sig_handler(self, signum, frame):
logger.debug(
_('{} got signal {}').format(
current_process().name,
Conf.SIGNAL_NAMES.get(signum, 'UNKNOWN')))
self.stop()
@property
def stat(self):
if self.sentinel:
return Stat.get(self.pid)
return Status(self.pid)
@property
def is_starting(self):
return self.stop_event and self.start_event and not self.start_event.is_set(
)
@property
def is_running(self):
return self.stop_event and self.start_event and self.start_event.is_set(
)
@property
def is_stopping(self):
return self.stop_event and self.start_event and self.start_event.is_set(
) and self.stop_event.is_set()
@property
def has_stopped(self):
return self.start_event is None and self.stop_event is None and self.sentinel
class EzhilFileExecuter(EzhilRedirectOutput):
""" run on construction - build a Ezhil lexer/parser/runtime and execute the file pointed to by @files;
When constructed with a @TIMEOUT value, the process may terminate without and output, otherwise it dumps the output
to a file named,
"""
def get_output(self):
return [self.tmpf_name, self.fProcName, self.data]
def __delete__(self):
if self.tmpf and hasattr(self.tmpf, 'name'):
os.unlink(self.tmpf.name)
self.tmpf = None
if self.fProcName:
os.unlink(self.fProcName)
self.fProcName = None
if hasattr(self.p, 'terminate'):
self.p.terminate()
pass
def __init__(self,
file_input,
debug=False,
redirectop=False,
TIMEOUT=None):
EzhilRedirectOutput.__init__(self, redirectop, debug)
self.dbg_msg(u"ezil file executer\n")
self.fProcName = ""
self.data = ""
self.tmpf_name = ""
self.p = None
self.TIMEOUT = TIMEOUT
if (not redirectop): #run serially and exit.
try:
ezhil_file_parse_eval(file_input, self.redirectop, self.debug)
self.exitcode = 0
except Exception as e:
self.exitcode = -1
traceback.print_tb(sys.exc_info()[2])
raise e
else:
self.dbg_msg("EzhilFileExecuter - entering the redirect mode\n")
self.p = Process(target=ezhil_file_parse_eval,
kwargs={
'file_input': file_input,
'redirectop': redirectop,
'debug': debug
})
def run(self):
if self.p:
try:
self.dbg_msg("begin redirect mode\n")
self.p.start()
if (self.TIMEOUT is not None):
start = time()
self.dbg_msg("timeout non-zero\n")
raise_timeout = False
while self.p.is_alive():
self.dbg_msg("in busy loop : %d , %d \n" %
(time() - start, self.TIMEOUT))
self.dbg_msg("SLEEP\n")
sleep(5) #poll every 5 seconds
if ((time() - start) > self.TIMEOUT):
self.dbg_msg("Reached timeout = %d\n" %
self.TIMEOUT)
raise_timeout = True
break
# now you try and read all the data from file, , and unlink it all up.
self.fProcName = EzhilRedirectOutput.pidFileName(
self.p.pid)
self.tmpf_name = self.tmpf.name
# dump stuff from fProcName into the stdout
fp = open(self.fProcName, 'r')
print(
u"######### ------- dump output ------- ##############"
)
self.data = fp.read()
print(self.data)
fp.close()
if raise_timeout:
raise TimeoutException(self.TIMEOUT)
#os.unlink( fProcName)
except Exception as e:
print("exception ", unicode(e))
traceback.print_tb(sys.exc_info()[2])
raise e
finally:
# reset the buffers
if (self.redirectop):
#self.tmpf.close()
sys.stdout = self.old_stdout
sys.stderr = self.old_stderr
sys.stdout.flush()
sys.stderr.flush()
# cleanup the cruft files
#if self.tmpf and hasattr(self.tmpf,'name'):
# os.unlink( self.tmpf.name )
#self.tmpf = None
#if self.fProcName:
# os.unlink( self.fProcName )
#self.fProcName = None
# nuke the process
if hasattr(self.p, 'terminate'):
self.p.terminate()
self.exitcode = self.p.exitcode
else:
pass #nothing to run
class MultiProcessScheduler:
def __init__(self):
self.cond = Condition() # default to RLock
# If duplex is False then the pipe is unidirectional
# conn1 for receiving messages and conn2 for sending messages.
conn1, conn2 = Pipe(duplex=False)
self.connREAD = conn1
self.connWRITE = conn2
# a holder to the closest task to execute
# it is not safe to access this variable directly as
# there might be data on the pipe, use self.__getClosestTask()
self._closestTask = None
# multiprocessing.Queue is used here to exchange task between the add
# call and the service running __run() method
self.queue = SimpleQueue()
# dummy Process, the correct one will be created when the first
# task is added
self.service = Process()
# TODO create destructor to avoid leaving with items on queue
def __getClosestTask(self):
'''
return the closest task to execute (i.e., top on pq)
'''
if self.connREAD.poll():
ret = None
while self.connREAD.poll():
ret = self.connREAD.recv()
self._closestTask = ret
print("[conn] closestTaskUpdate: ", self._closestTask)
return self._closestTask
def add(self, task):
if type(task) is not Task:
raise TypeError
self.queue.put(task)
if not self.service.is_alive():
# it seams that Process.run() is a blocking call
# so the only way to re-run the process is to create another one
self.service = Process(target=MultiProcessScheduler.__run,
args=(self.cond, self.queue,
self.connWRITE),
daemon=False)
self.service.start()
else:
# notify the condition variable if the new task has the
# closest execution time
closestTask = self.__getClosestTask()
if closestTask and task.time < closestTask.time:
self.cond.acquire()
self.cond.notify()
self.cond.release()
@staticmethod
def __run(cond, queue, conn):
tasksQueue = []
print("[run] starting", queue.empty())
while True:
# remove tasks from queue and add to
# internal priorityQueue (tasksQueue)
while not queue.empty():
task = queue.get()
heappush(tasksQueue, task)
print("[run] adding task to pq: ", task)
# if there are task on the priority queue,
# check when the closest one should be runned
if tasksQueue:
etime, _, _ = task = tasksQueue[0]
now = time()
if etime < now:
# only pop before running
# if a task is not being running in a given time,
# the next this loop runs that task might not be the
# closest one
_, fn, args = heappop(tasksQueue)
print("[run] running:", task)
p = Process(target=fn, args=args, daemon=False)
p.start()
else:
delay = etime - now
print("[run] sleeping for ", delay, task)
# send the closest task to the pipe
conn.send(task)
cond.acquire()
cond.wait(timeout=delay)
if not tasksQueue and queue.empty():
# only stop the service if there are no task anwhere
break
print("[run] done")
