class CaptureProcess(Process):
"""A process that fills a queue with images as captured from
a camera feed"""
def __init__(self, capture, imageQueue):
Process.__init__(self, name="Capture")
self.imageQueue = imageQueue
self.capture = capture
self.keepGoing = Event()
self.keepGoing.set()
self.daemon = True
def run(self):
print "CaptureProcess pid: %s" % (self.pid,)
while self.keepGoing.is_set():
image = captureImage(self.capture)
# sys.stdout.write(".")
try:
self.imageQueue.put(serializeImage(image), block=True, timeout=0.25)
except FullException:
try:
_ = self.imageQueue.get_nowait()
except:
pass # Try to clear the queue, but don't worry if someone snatches it first
def stop(self):
self.keepGoing.clear()
class BroadcastClient(Process):
def __init__(self, port, datagram_size, name="BroadcastClient"):
Process.__init__(self, name=name)
self.logger = multiprocessing.get_logger()
self.event = Event()
self.port = port
self.datagram_size = datagram_size
self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, socket.IPPROTO_UDP)
self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self.sock.settimeout(1)
self.sock.bind(("", self.port))
def run(self):
self.event.set()
self.logger.debug("PID: %d" % multiprocessing.current_process().pid)
while self.event.is_set():
try:
message, (ip, port) = self.sock.recvfrom(self.datagram_size)
teacher_discovered.send(sender=self)
self.logger.debug("Received: %s from: %s" % (message, ip))
except socket.timeout:
self.logger.debug("%s timeout" % multiprocessing.current_process().name)
time.sleep(1)
def stop(self):
self.logger.debug("Client will halt.")
self.event.clear()
self.sock.close()
self.terminate()
def wait_for(self, key, value):
d = Manager().dict()
d[key] = None
v = Manager().Value('s', ' ')
e = Event()
p_state = Process(target=self.state_refresher, args=(self, d, e))
p_input = Process(target=self.input_waiter, args=(self, v, e))
p_state.start()
p_input.start()
while v.value != 'exit' and dict(d.items())[key] != value:
e.wait()
e.clear()
self.state = d['state']
self.current = d['current']
self.enemy = d['enemy']
self.battlefield = d['battlefield']
p_state.terminate()
p_input.terminate()
curses.endwin()
p_state.join()
p_input.join()
return True if dict(d.items())[key] == value else False
class SharedFile(object):
def __init__(self, filename):
self.filename = filename
self.fevent = Event()
# self.state = Value('i', 0)
self.fevent.set()
def write(self, mode, data):
# print("Write {}".format(inspect.stack()[1][3]))
self.wait_freedom_and_lock()
f = open(self.filename, mode)
f.write(data)
f.close
self.unlock()
def read(self):
# print("Read {}".format(inspect.stack()[1][3]))
self.wait_freedom_and_lock()
f = open(self.filename, 'r')
data = f.read()
f.close
self.unlock()
return data
def wait_freedom_and_lock(self):
self.fevent.wait()
self.fevent.clear()
# return
def unlock(self):
self.fevent.set()
class ClassifierWorkerPool(object):
def __init__(self):
self.queue = Queue(100)
self.workers = []
self.stop = Event()
self.stop.clear()
self.queue_feeder = QueueFeeder(self.queue, self.stop)
row = TrainedClassifiers.objects(name=config.classifier).first()
if not row:
raise Exception("Classifier %s does not exists" % config.classifier)
self.trained_classifier = row.get_classifier()
def start(self):
self.queue_feeder.start()
for i in range(0, config.classifier_pool_size):
worker = ClassifierWorker(self.trained_classifier, self.queue, self.stop)
worker.start()
self.workers.append(worker)
def terminate(self):
self.stop.set()
self.queue_feeder.join()
for w in self.workers:
w.join()
class TestProxyData(TestData):
def setup(self):
create_link('dummyX', 'dummy')
t_url = 'unix://\0%s' % (uuid.uuid4())
p_url = 'unix://\0%s' % (uuid.uuid4())
self.connect = Event()
self.release = Event()
target = Process(target=_run_remote_uplink,
args=(t_url, self.connect, self.release))
target.daemon = True
target.start()
self.connect.wait()
self.connect.clear()
proxy = Process(target=_run_remote_uplink,
args=(p_url, self.connect, self.release))
proxy.daemon = True
proxy.start()
self.connect.wait()
self.connect.clear()
self.ip = IPRoute(do_connect=False)
link, proxy = self.ip.connect(p_url)
self.ip.register('bala', proxy)
link, host = self.ip.connect(t_url, addr=proxy)
service = self.ip.discover(self.ip.default_target, addr=host)
self.ip.default_peer = host
self.ip.default_dport = service
self.dev = self.ip.link_lookup(ifname='dummyX')
class Logger(object):
def __init__(self, filename):
self.qtag = Queue()
self.done = Event()
self.tag = None
self.filename = filename
self.file = None
def start(self):
self.file = open(self.filename, 'w')
print 'Opened',self.filename,'for writing.'
def set_tag(self, tag):
self.qtag.put(tag)
def set_done(self):
self.done.set()
def log(self, nodeid, msgid, data):
if not self.qtag.empty():
self.tag = self.qtag.get()
if self.done.is_set():
self.done.clear()
return True
L = ['%f'%time.time(), '%d'%nodeid, '%d'%msgid] + map(str,data)
if self.tag:
L.append(self.tag)
print >>self.file, ','.join(L)
self.file.flush()
def close(self):
if self.file:
self.file.close()
print 'File closed.'
class BroadcastServer(Process):
def __init__(self, ip, port, message, name="BroadcastServer"):
Process.__init__(self, name=name)
self.logger = multiprocessing.get_logger()
self.event = Event()
self.message = message
self.ip = ip
self.port = port
self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self.sock.bind(("", 0))
def run(self):
self.event.set()
self.logger.debug("PID: %d" % multiprocessing.current_process().pid)
while self.event.is_set():
self.logger.debug("Sending: %s" % self.message)
self.sock.sendto(self.message, (self.ip, self.port))
time.sleep(1)
def stop(self):
self.logger.debug("Server will halt.")
self.event.clear()
self.terminate()
def SetupBricks():
allsuccess = False
allprocs = []
startvols = False
events = {}
ev = None
for brick in BRICKS_IPADDRS:
ev = Event ()
ev.clear()
events[brick] = ev
for brick in BRICKS_IPADDRS:
mychan,brickchan = Pipe()
startvols = False
if brick == NFSSERVER_ADDR:
startvols = True
proc = Process (target=SetupBrick, args=(brick, brickchan, startvols, events,))
allprocs.append ((proc, mychan, brickchan))
if FUSE and NFSCLIENT_ADDR not in BRICKS_IPADDRS:
ev = Event ()
ev.clear()
events[NFSCLIENT_ADDR] = ev
mychan,brickchan = Pipe()
startvols = False
proc = Process (target=SetupBrick, args=(NFSCLIENT_ADDR, brickchan, startvols, events,))
allprocs.append ((proc, mychan, brickchan))
allsuccess = RunProcMonitorLoop(allprocs)
return allsuccess
class MistProcess(Process):
def __init__(self, gpio, sleep=1, name='MistProcess'):
Process.__init__(self, name=name)
self.logger = multiprocessing.get_logger()
self.event = Event()
self.name = name
self.gpio = gpio
self.sleep = sleep
self.mist = mraa.Gpio(self.gpio)
self.mist.dir(mraa.DIR_OUT)
def _mist_on(self):
self.logger.debug('Mist on')
self.mist.write(1)
def _mist_off(self):
self.logger.debug('Mist off')
if self.mist:
self.mist.write(0)
def run(self):
self.event.set()
self.logger.debug('PID: %d' % multiprocessing.current_process().pid)
while self.event.is_set():
self._mist_on()
time.sleep(self.sleep)
def stop(self):
self.logger.debug('Process {} will halt.'.format(self.name))
self.event.clear()
self._mist_off()
class StoppableProcess(Process):
""" Base class for Processes which require the ability
to be stopped by a process-safe method call
"""
def __init__(self):
self._should_stop = Event()
self._should_stop.clear()
super(StoppableProcess, self).__init__()
def join(self, timeout=0):
""" Joins the current process and forces it to stop after
the timeout if necessary
:param timeout: Timeout duration in seconds
"""
self._should_stop.wait(timeout)
if not self.should_stop():
self.stop()
super(StoppableProcess, self).join(0)
def stop(self):
self._should_stop.set()
def should_stop(self):
return self._should_stop.is_set()
def __repr__(self):
return "<%s(should_stop=%s)>" % (
self.__class__.__name__, self.should_stop())
class fmanager:
def __init__(self,data,fn):
self.sf = Event()
self.sf.clear()
self.nproc=cpu_count()
self.pipes = [Pipe() for i in xrange(self.nproc)]
self.e = [evaluator(self.pipes[i][1],self.sf,data,fn) for i in xrange(self.nproc)]
null = [i.start() for i in self.e]
return
def __del__(self):
self.sf.set()
null = [i.join() for i in self.e]
null = [i.terminate() for i in self.e]
return
def eval(self,x):
nd = len(x)
for i in xrange(nd):
self.pipes[i % self.nproc][0].send([i, x[i]])
solns = []
while len(solns) < nd:
for i in xrange(self.nproc):
if self.pipes[i][0].poll(0.005):
solns.append(self.pipes[i][0].recv())
solns.sort(key=lambda i: i[0])
return [i[1] for i in solns]
class ChildChecker(threading.Thread):
def __init__(self, killEvent):
super(ChildChecker, self).__init__()
self.killEvent = killEvent
self.event = Event()
self.process = Process(target=childsPlay, args=(self.event,))
def run(self):
self.process.start()
while not self.killEvent.is_set():
self.event.wait()
print "Child checked, and is done playing"
if raw_input("Do again? y/n:") == "y":
self.event.clear()
self.process = Process(target=endlessChildsPlay, args=(self.event,))
self.process.start()
else:
self.cleanChild()
self.killEvent.set()
def join(self):
print "Joining child process"
# Timeout on 5 seconds
self.process.join(5)
if self.process.is_alive():
print "Child did not join! Killing.."
self.process.terminate()
print "Joining ChildChecker thread"
super(ChildChecker, self).join()
def cleanChild(self):
print "Cleaning up the child..."
class QueueTask:
def __init__(self):
self.queue = JoinableQueue()
self.event = Event()
atexit.register( self.queue.join )
process = Process(target=self.work)
process.daemon = True
process.start()
def work(self):
while True:
func, args, wait_for = self.queue.get()
for evt in wait_for:
evt.wait()
func(*args)
self.event.set()
self.queue.task_done()
def enqueue(self, func, args=[], wait_for=[]):
self.event.clear()
self.queue.put( (func, args, wait_for) )
return self.event
def face_proc(self, child_face_recog: Pipe, e_new_person: Event):
"""
Parallel process of saving people for face recognition
Arguments:
child_face_recog {Pipe} -- pipe for communication with parent process,
recieve ROI ndarray type of recognized object
"""
if not os.path.exists('humans'):
print('created', os.getcwd())
os.mkdir('humans')
else:
print('exist')
os.chdir(os.path.join(os.getcwd(), 'humans'))
is_first = True
counter = 0
while True:
if e_new_person.is_set():
counter = 0
if not is_first:
os.chdir('..')
new_dir = datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S")
os.mkdir(new_dir)
print('Created', os.getcwd() + new_dir)
os.chdir(os.path.join(os.getcwd(), new_dir))
e_new_person.clear()
is_first = False
image = child_face_recog.recv()
cv.imwrite(filename=str(counter) + '.jpg', img=image)
print('image saved:', os.getcwd() + str(counter) + '.jpg')
counter += 1
class KafkaQueue(object):
def __init__(self, client, topic, partitions, producer_config={}, consumer_config={}):
"""
KafkaQueue a Queue-like object backed by a Kafka producer and some number of consumers
Params
======
client: KafkaClient object
topic: str, the topic name
partitions: list of ints, the partions to consume from
producer_config: dict, see below
consumer_config: dict, see below
Consumer Config
===============
consumer_sleep: int, time in milliseconds a consumer should sleep when it reaches
the end of a partition. Default is 200
Producer Config
===============
producer_timeout: int, time in milliseconds a producer should wait for messages to
enqueue for producing. Default is 100
producer_flush_timeout: int, time in milliseconds a producer should allow messages
to accumulate before sending to Kafka. Default is 2000
producer_flush_buffer: int, number of messages a producer should allow to accumulate.
Default is 500
"""
self.in_queue = Queue()
self.out_queue = Queue()
self.consumers = []
self.barrier = Event()
# Initialize and start consumer threads
for partition in partitions:
consumer = KafkaConsumerProcess(client, topic, partition, self.in_queue, self.barrier, **consumer_config)
consumer.start()
self.consumers.append(consumer)
# Initialize and start producer thread
self.producer = KafkaProducerProcess(client, topic, self.out_queue, self.barrier, **producer_config)
self.producer.start()
# Trigger everything to start
self.barrier.set()
def get(self, block=True, timeout=None):
return self.in_queue.get(block, timeout).payload
def put(self, msg, block=True, timeout=None):
return self.out_queue.put(msg, block, timeout)
def close(self):
self.in_queue.close()
self.out_queue.close()
self.barrier.clear()
self.producer.join()
for consumer in self.consumers:
consumer.join()
def add_poster(self, num=1):
if num <= 0:
return
else:
evt_death = Event()
evt_death.clear()
self._posters.append( Poster(self.name + "_Poster_" + str(num), self.out_dir, self.in_dir, self.q_gpu2s3, evt_death, self.sqs_name,self.in_sqs_q, self.s3bucket_name))
self._reaper.append(evt_death)
self._posters[-1].daemon = True
self._posters[-1].start()
self.add_poster(num - 1)
def add_retriever(self, num=1):
if num <= 0:
return
else:
evt_death = Event()
evt_death.clear()
self._retrievers.append( Retriever(self.name + "_Retriever_" + str(num), self.in_dir, self.q_ret2gpu, evt_death, self.sqs_name, self.s3bucket_name, max_q_size=10*(num+1) ) )
self._reaper.append(evt_death)
self._retrievers[-1].daemon = True
self._retrievers[-1].start()
self.add_retriever(num - 1)
def parent_run():
# Make sure we can connect
conn = psycopg2.connect(conn_string)
curs = conn.cursor()
curs.execute(dummy_statement)
curs.fetchall()
curs.close()
conn.close()
# Spawn workers
children = []
result_queue = Queue()
start_lock = Event()
for i in xrange(0, n_workers):
done_event = Event()
child = Process(target=child_run, args=(i,result_queue,start_lock,done_event))
children.append(child)
child.done_event = done_event
for child in children:
child.start()
# and start them working
start_lock.set()
# wait a while for children to thrash the system
# then ask them to finish
time.sleep(10)
start_lock.clear()
# Wait until all children have reported in
sys.stderr.write("Waiting for children to complete...")
sys.stderr.flush()
for child in children:
child.done_event.wait()
sys.stderr.write(" done\n")
# Read the results
results = []
for i in range(0, len(children)):
(worker_number, num_tx) = result_queue.get(False)
results.append(num_tx)
total_tx = sum(results)
mean_tx = float(total_tx) / float(len(results))
# and wait for children to exit
for child in children:
child.join()
sys.stderr.write("Done, executed %i statements with %i conns and %i workers\n" % (total_tx, n_conns, n_workers))
sys.stderr.write("Mean tx per worker was %.2f\n" % mean_tx)
class EngineManager(ObjectsManager, Process):
"""Uses header values to handle directed requests and results."""
def __init__(self, *args, **kwargs):
super(EngineManager, self).__init__(*args, **kwargs)
self.manager = Manager()
self.results = self.manager.dict()
self.block = Event()
def retrieve(self, jid=None, inputs=False, timeout=None, block=True):
"""Retrieve a job given a jid"""
if jid is None:
jid = self.cjid
if block:
self.block.wait()
hdr, func, args, kwargs, rvalue = self.results[jid]
self.block.clear()
if inputs:
return func, args, kwargs, rvalue
else:
return rvalue
def ready(self, jid):
"""Check if a job id is ready."""
return jid in self.results
def __getattr__(self, attr):
"""Call a method on the underlying threaded object"""
def method(*args, **kwargs):
"""A threaded method"""
jid = id((attr, args, kwargs))
self.cjid = jid
self.hdr["jid"] = jid
self.input.put((self.hdr, attr, args, kwargs))
return jid
return method
def start(self):
"""Start"""
super(EngineManager, self).start()
def run(self):
"""Run the subthread to move things off the output."""
running = True
while running:
hdr, func, args, kwargs, rvalue = self.output.get()
if "stop" in hdr:
running = False
else:
self.results[hdr["jid"]] = (hdr, func, args, kwargs, rvalue)
self.block.set()
def main():
# Setup process pooling
manager = Manager()
# Make queue for processes to grab a thread number from
# also used so that the last worker knows it's last.
thread_number = manager.Queue()
[thread_number.put(t) for t in range(getthreads())]
# Set event for handling worker ready condition
workers_ready = Event()
workers_idle = Event()
workers_idle.set()
# Set event for handling daemon quit condition for stuff that doesnt exit clean
daemon_exit = Event()
# Set event for handling new file condition
new_file = Event()
# Setup workqueue
queue = manager.Queue()
print("Starting daemons")
watcherd = Process(name="watcherd", target=watcher, daemon=True, args=(queue, new_file,))
watcherd.start()
print("Daemons started.")
print("Initializing workers...")
workers = Pool(processes=thread_number.qsize(),
initializer=initworker,
initargs=(thread_number, workers_ready,))
workers_ready.wait()
print("All workers ready")
# Set signal handler on main process
def exit_handler(signum, frame):
cleanup(workers, daemon_exit, workers_idle)
signal(SIGINT, exit_handler)
signal(SIGTERM, exit_handler)
# Main loop
while not daemon_exit.is_set():
# wait until new file is on queue, forced check every 5 minutes.
status = new_file.wait(timeout=300.0)
if status: new_file.clear()
workers_idle.clear()
# Send to workers
workers.map(compressors.dyncompress,[queue.get() for q in range(queue.qsize())])
workers_idle.set()
class DeviceWatcher(object):
def __init__(self, port=None, watcher='arduino'):
''' Initialize a new instance of the class
'''
self.ipc_queue = Queue()
self.halt_event = Event()
arguments = (self.halt_event, self.ipc_queue, port, watcher)
self.process = Process(target=self.__watcher, args=arguments)
self.process.daemon = True
def start(self):
''' Start the processor running
:returns: The intermessage queue to read from
'''
_logger.info("starting the device monitor")
self.halt_event.clear()
self.process.start()
return self.ipc_queue
def stop(self):
''' Stop the processor from running
'''
_logger.info("stopping the device monitor")
self.halt_event.set()
self.process.join(5)
if self.process.is_alive():
_logger.info("force stopping the device monitor")
self.process.terminate()
@classmethod
def __watcher(cls, halt_event, ipc_queue, port, watcher):
''' The main runner for pulling messages off the serial
bus and throwing them into the database
'''
if watcher == 'arduino':
client = ArduinoWatcher(port=port, baudrate=9600, timeout=5)
else: client = TinyosWatcher(port)
_logger.info("device monitor initialized")
try:
for message in client:
if halt_event.is_set():
_logger.info("device monitor was told to stop")
break # someone told us to stop
if message: ipc_queue.put(message)
except Exception:
_logger.exception("something bad happend in the device monitor")
ipc_queue.put(None) # we are broke, force an exit
client.close()
_logger.info("device monitor exiting")
class conductor(Process):
def __init__(self, bus_data_queue, cld_e, park_e, open_once_e):
Process.__init__(self)
self.park_e = park_e
self.closedoor_e = cld_e
self.open_once_e = open_once_e
self.bus_data_queue = bus_data_queue
def run(self):
once = 0
while True:
self.park_e.wait()
self.open_once_e.wait()
self.open_once_e.clear()
self.opendoor()
self.closedoor_e.set()
self.closedoor()
self.selltickets()
def opendoor(self):
msg = 'open the door'
self.bus_data_queue.put(msg)
timeit = random.random()
time.sleep(timeit)
def closedoor(self):
msg = 'close the door'
self.bus_data_queue.put(msg)
timeit = random.random()
time.sleep(timeit)
def selltickets(self):
msg = 'selling tickets...'
self.bus_data_queue.put(msg)
timeit = random.random()
time.sleep(timeit)
self.closedoor_e = Event()
self.park_e = Event()
self.open_once_e = Event()
self.closedoor_e.set()
self.park_e.clear()
self.open_once_e.set()
self.driver1 = driver(self.bus_data_queue, self.closedoor_e, self.park_e, self.open_once_e)
self.driver1.start()
self.conductor1 = conductor(self.bus_data_queue, self.closedoor_e, self.park_e, self.open_once_e)
self.conductor1.start()
def main():
event = Event()
queue = Queue()
proc_pool = [MyProcess(queue, event, top_func) for _ in range(4)]
event.set()
for i in range(1000):
queue.put(i)
for proc in proc_pool:
proc.start()
while not queue.empty():
time.sleep(1)
event.clear()
for proc in proc_pool:
proc.join()
class IndexerFather(object):
def __init__(self, count):
Process.__init__(self)
self.is_alive = False
self.count = count # Indexers count
self.alive_event = Event() # Event to alive Indexers
self.work_event = Event() # Event to signal Indexers work
self.queue = Queue() # Next text to process
self.index_queue = [] # List for saving tasks
self.indexer_pool = []
def start(self):
if not self.alive_event.is_set():
for task in self.index_queue:
self.queue.put(task)
for i in range(self.count):
self.indexer_pool.append(Indexer(self.queue,
self.alive_event,
self.work_event))
self.alive_event.set()
self.work_event.set()
self.is_alive = True
for indexer in self.indexer_pool:
indexer.start()
def stop(self):
self.work_event.clear()
is_stop_working = False
while not is_stop_working:
for indexer in self.indexer_pool:
is_stop_working = not (indexer.is_working and is_stop_working)
while not self.queue.empty():
self.index_queue.append(self.queue.get())
self.alive_event.clear()
for indexer in self.indexer_pool:
indexer.join()
self.indexer_pool.clear() # After every join processes must stoped
logging.debug("All processes is stoped")
return self.index_queue
def add_text(self, url, text):
self.queue.put((url, text))
def test_worker(self):
queue = Queue(100)
stop = Event()
stop.clear()
w = ClassifierWorker(self.cls, queue, stop)
w.start()
for d in self.data:
queue.put(d)
while not queue.empty():
time.sleep(0.1)
time.sleep(0.5)
stop.set()
w.join()
self.assertEqual(ClassifiedStream.objects.count(), 5)
class Processer(object):
"""docstring for Worker"""
def __init__(self, arg ):
super(Processer, self).__init__()
self.arg = arg
self.be_exit = Event()
self.be_exit.clear()
def stop_main_loop(self):
self.be_exit.set()
def main_loop(self):
while not self.be_exit.is_set():
try:
GetTaskFromTBus( self.arg )
time.sleep(1)
except Exception, e:
logger.exception( e )
class WeatherProcess(Process):
def __init__(self, city, name='WeatherProcess'):
Process.__init__(self, name=name)
self.logger = multiprocessing.get_logger()
self.event = Event()
self.name = name
self.city = city
def run(self):
self.event.set()
self.logger.debug('PID: %d' % multiprocessing.current_process().pid)
while self.event.is_set():
time.sleep(1)
def stop(self):
self.logger.debug('Process {} will halt.'.format(self.name))
self.event.clear()
def make_server_manager(self, port, ip, authkey):
""" Create a manager for the server, listening on the given port.
Return a manager object with get_job_q and get_result_q methods.
"""
try: # Make_server_manager definition try block
job_q = Queue(maxsize=100)
result_q = Queue()
shutdown = Event()
shutdown.clear()
mode_bit_1 = Event()
mode_bit_2 = Event()
try: # JobQueueManager/Lambda functions Try Block
self.JobQueueManager.register('get_job_q', callable=lambda: job_q)
self.JobQueueManager.register('get_result_q', callable=lambda: result_q)
self.JobQueueManager.register('get_shutdown', callable=lambda: shutdown)
self.JobQueueManager.register('get_mode_bit_1', callable=lambda: mode_bit_1)
self.JobQueueManager.register('get_mode_bit_2', callable=lambda: mode_bit_2)
except Exception as inst:
print "============================================================================================"
print "ERROR: An exception was thrown in Make_server_Manager: " \
+ "JobQueueManager/Lambda functions Try Block"
#the exception instance
print type(inst)
#arguments stored in .args
print inst.args
#_str_ allows args tto be printed directly
print inst
print "============================================================================================"
manager = self.JobQueueManager(address=(ip, port), authkey=authkey)
manager.start()
print 'Server started at port %s' % port
return manager
except Exception as inst:
print "============================================================================================="
print "ERROR: An exception was thrown in Make_server_manager definition Try block"
#the exception instance
print type(inst)
#arguments stored in .args
print inst.args
#_str_ allows args tto be printed directly
print inst
print "============================================================================================="
class ConsumerProxy(Process):
""" Consumer Proxy
Consumer Proxy act as a broker for consumer in Producer-Consumer Pattern.
It deals with the process model for the real consumer.
"""
def __init__(self, queue, consumer):
Process.__init__(self)
self._queue = queue
self._consumer = consumer
self._stop = Event()
self._stop.clear()
def consume(self, task):
self._consumer.consume(task)
def close(self):
self._consumer.close()
def stop(self):
self._stop.set()
def run(self):
signal.signal(signal.SIGINT, signal.SIG_IGN)
while True:
if self._stop.is_set():
self.close()
break
try:
task = self._queue.get(timeout=BLOCK_TIMEOUT)
except Queue.Empty:
continue
try:
self.consume(task)
except Exception:
traceback.print_exc()
finally:
self._queue.task_done()
class RegistryServer(Process):
__KEY = b'NETAPPapiS3CR3Tk8'
__PORT = int(os.environ.get('NETAPP_API_CR_PORT', '12342'))
__HOST = 'localhost'
def __init__(self):
super(RegistryServer, self).__init__()
self.notify = Event()
def wait_complete(self):
while not self.notify.is_set():
self.notify.wait(1)
self.notify.clear()
@staticmethod
def GET_CLIENT_REGISTRY():
RegistryManager.register('get_datadict')
RegistryManager.register('get_datalock')
RegistryManager.register('get_sd_event')
manager = RegistryManager(address=(RegistryServer.__HOST,
RegistryServer.__PORT),
authkey=RegistryServer.__KEY)
manager.connect()
return ClientRegistry(manager.get_datadict(), manager.get_datalock(),
manager.get_sd_event())
def run(self):
LOGGER.debug('starting up registry server')
data_dict = {
'apistatus': 'TBD', # the status of the NetApp api
'grpc': {
'accessmsg': 'TBD' # the string to return for connection
},
'clientsessionstatus': {
# for each client --> ID : status .. [RUNNING, SHUTDOWN]
}
}
data_lock = Lock()
sd_event = Event()
RegistryManager.register('get_datadict',
callable=lambda: data_dict,
proxytype=mp_Managers.DictProxy)
RegistryManager.register('get_datalock',
callable=lambda: data_lock,
proxytype=mp_Managers.AcquirerProxy)
RegistryManager.register('get_sd_event',
callable=lambda: sd_event,
proxytype=mp_Managers.EventProxy)
manager = RegistryManager(address=(RegistryServer.__HOST,
RegistryServer.__PORT),
authkey=RegistryServer.__KEY)
LOGGER.debug('registry server created')
manager.start()
LOGGER.debug('registry server started @ %s:%s', RegistryServer.__HOST,
RegistryServer.__PORT)
# indicate to outside world that start is complete
self.notify.set()
# sleep until shutdown is issued
while not sd_event.is_set():
sd_event.wait(1)
LOGGER.debug('registry server shutting down')
# shutdown the manager
manager.shutdown()
manager.join()
LOGGER.debug('registry server exiting')
# indicate to outside world that stop is complete
self.notify.set()
class Publisher:
""" Publisher class in which the queue for publishing messages is defined and also a separated process is started.
It is important to have a new process, since the serialization/deserialization of messages from the QUEUE may be
a bottleneck for performance.
"""
def __init__(self, port=5555):
self.logger = get_logger(name=__name__)
self._port = port
self._queue = Queue(
) # The publisher will grab and broadcast the messages from this queue
self._event = Event() # This event is used to stop the process
self._process = Process(target=publisher,
args=[self._queue, self._event, self._port])
self.logger.info('Initialized published on port {}'.format(port))
def start(self):
""" Start a new process that will be responsible for broadcasting the messages.
"""
self._event.clear()
self._process.start()
sleep(1) # This forces the start to block until the publisher is ready
def stop(self):
self._event.set()
self.empty_queue()
def empty_queue(self):
""" If the publisher stops before broadcasting all the messages, the Queue may still be using some memory. This
method is simply getting all the elements in order to free memory. Can be useful for garbage collection or
better control of the downstream program.
"""
self.logger.info('Emptying the queue of the publisher')
self.logger.debug('Queue length: {}'.format(self._queue.qsize()))
self._queue.close()
def publish(self, topic, data):
""" Adapts the data to make it faster to broadcast
:param str topic: Topic in which to publish the data
:param data: Data to be published
:return: None
"""
self.logger.debug('Adding data of type {} to topic {}'.format(
type(data), topic))
try:
self._queue.put({'topic': topic, 'data': data})
except AssertionError:
# This means the queue has been closed already
pass
@property
def port(self):
return self._port
@port.setter
def port(self, new_port):
if new_port != self._port:
self._port = new_port
self.logger.warning(
'Changing the port requires restarting the publisher process')
self.logger.debug(
'Setting the new publisher port to {}'.format(new_port))
self.stop()
self._process.join()
self._process = Process(
target=publisher, args=[self._queue, self._event, self._port])
self.start()
else:
self.logger.warning(
'New port {} is the same as the old port'.format(new_port))
def join(self, timeout=0):
if self._process.is_alive():
self.logger.debug('Waiting for Publisher process to finish')
self._process.join(timeout)
def main(q):
v = 0
t = time()
while v != (size * 3):
try:
if time() - t > 0.4:
print 'Queue.qsize:', q.qsize()
t = time()
q.put('test' + str(v))
v += 1
except Full:
pass
s = time()
p1 = Process(target = worker, args=(q, run))
p1.start()
p2 = Process(target = worker, args=(q, run))
p2.start()
main(q)
qs = q.qsize()
while qs > 0:
print 'Queue.qsize:', qs
sleep(0.4)
qs = q.qsize()
run.clear()
p1.join()
p2.join()
print 'nox.shm.Queue put/get: %s' %(time() - s)
class KafkaQueue(object):
def __init__(self,
client,
topic,
partitions,
producer_config=None,
consumer_config=None):
"""
KafkaQueue a Queue-like object backed by a Kafka producer and some
number of consumers
Messages are eagerly loaded by the consumer in batches of size
consumer_fetch_size.
Messages are buffered in the producer thread until
producer_flush_timeout or producer_flush_buffer is reached.
Params
======
client: KafkaClient object
topic: str, the topic name
partitions: list of ints, the partions to consume from
producer_config: dict, see below
consumer_config: dict, see below
Consumer Config
===============
consumer_fetch_size: int, number of bytes to fetch in one call
to Kafka. Default is 1024
consumer_sleep: int, time in milliseconds a consumer should sleep
when it reaches the end of a partition. Default is 200
Producer Config
===============
producer_timeout: int, time in milliseconds a producer should
wait for messages to enqueue for producing.
Default is 100
producer_flush_timeout: int, time in milliseconds a producer
should allow messages to accumulate before
sending to Kafka. Default is 2000
producer_flush_buffer: int, number of messages a producer should
allow to accumulate. Default is 500
"""
producer_config = {} if producer_config is None else producer_config
consumer_config = {} if consumer_config is None else consumer_config
self.in_queue = Queue()
self.out_queue = Queue()
self.consumers = []
self.barrier = Event()
# Initialize and start consumer threads
for partition in partitions:
consumer = KafkaConsumerProcess(client, topic, partition,
self.in_queue, self.barrier,
**consumer_config)
consumer.start()
self.consumers.append(consumer)
# Initialize and start producer thread
self.producer = KafkaProducerProcess(client, topic, self.out_queue,
self.barrier, **producer_config)
self.producer.start()
# Trigger everything to start
self.barrier.set()
def get(self, block=True, timeout=None):
"""
Consume a message from Kafka
Params
======
block: boolean, default True
timeout: int, number of seconds to wait when blocking, default None
Returns
=======
msg: str, the payload from Kafka
"""
return self.in_queue.get(block, timeout).payload
def put(self, msg, block=True, timeout=None):
"""
Send a message to Kafka
Params
======
msg: std, the message to send
block: boolean, default True
timeout: int, number of seconds to wait when blocking, default None
"""
self.out_queue.put(msg, block, timeout)
def close(self):
"""
Close the internal queues and Kafka consumers/producer
"""
self.in_queue.close()
self.out_queue.close()
self.barrier.clear()
self.producer.join()
for consumer in self.consumers:
consumer.join()
log("{} timed out.".format(name))
to_remove.append(i)
avail_jobs.append(job_index)
elif not proc.is_alive():
to_remove.append(i)
proc.join()
avail_jobs.append(job_index)
# Reverse index order so as to pop elements correctly
for i in reversed(to_remove):
workers.pop(i)
time.sleep(0.01)
# Conclude and write results to file.
run_signal.clear()
flush_proc.join(1)
# it's important to count the total after proc.join
total = len(res_list)
log(f"\nFinished {total} contracts...\n")
vulnerability_counts = defaultdict(int)
analytics_sums = defaultdict(int)
meta_counts = defaultdict(int)
all_files = set()
for contract, files, meta, analytics in res_list:
for f in files:
all_files.add(f)
for m in meta:
meta_counts[m] += 1
for k, a in analytics.items():
class ClockBaseInterruptBehavior(ClockBaseBehavior):
'''A kivy clock which can be interrupted during a frame to execute events.
'''
interupt_next_only = False
_event = None
_get_min_timeout_func = None
def __init__(self, interupt_next_only=False, **kwargs):
super(ClockBaseInterruptBehavior, self).__init__(**kwargs)
self._event = MultiprocessingEvent() if PY2 else ThreadingEvent()
self.interupt_next_only = interupt_next_only
self._get_min_timeout_func = self.get_min_timeout
def usleep(self, microseconds):
self._event.clear()
self._event.wait(microseconds / 1000000.)
def on_schedule(self, event):
fps = self._max_fps
if not fps:
return
if not event.timeout or (
not self.interupt_next_only
and event.timeout <= 1 / fps - # remaining time
(self.time() - self._last_tick) + # elapsed time
4 / 5. * self.get_resolution()): # resolution fudge factor
self._event.set()
def idle(self):
fps = self._max_fps
event = self._event
resolution = self.get_resolution()
if fps > 0:
done, sleeptime = self._check_ready(fps, resolution,
4 / 5. * resolution)
if not done:
event.wait(sleeptime)
current = self.time()
self._dt = current - self._last_tick
self._last_tick = current
event.clear()
# anything scheduled from now on, if scheduled for the upcoming frame
# will cause a timeout of the event on the next idle due to on_schedule
# `self._last_tick = current` must happen before clear, otherwise the
# on_schedule computation is wrong when exec between the clear and
# the `self._last_tick = current` bytecode.
return current
def _check_ready(self, fps, min_sleep, undershoot):
if self._event.is_set():
return True, 0
t = self._get_min_timeout_func()
if not t:
return True, 0
if not self.interupt_next_only:
curr_t = self.time()
sleeptime = min(1 / fps - (curr_t - self._last_tick), t - curr_t)
else:
sleeptime = 1 / fps - (self.time() - self._last_tick)
return sleeptime - undershoot <= min_sleep, sleeptime - undershoot
class GenericCam(Process):
def __init__(self, outQ=None, recorderpar=None, refreshperiod=1 / 20.):
super(GenericCam, self).__init__()
self.name = ''
self.cam_id = None
self.h = None
self.w = None
self.nchan = 1
self.close_event = Event()
self.start_trigger = Event()
self.stop_trigger = Event()
self.saving = Event()
self.nframes = Value('i', 0)
self.queue = outQ
self.camera_ready = Event()
self.eventsQ = Queue()
self._init_controls()
self._init_ctrevents()
self.cam_is_running = False
self.was_saving = False
self.recorderpar = recorderpar
self.recorder = None
self.refresh_period = refreshperiod
self._tupdate = time.time()
self.daemon = True
#self.memmanager = Manager()
#self.memlist = self.memmanager.list()
#self.memlist.append(None)
self.lasttime = 0
def get_img(self):
return self.img #self.memlist[0]
def stop_saving(self):
# This will send a stop to stop saving and close the writer.
#if self.saving.is_set():
self.saving.clear()
def _init_controls(self):
return
def _init_ctrevents(self):
if hasattr(self, 'ctrevents'):
for c in self.ctrevents.keys():
self.ctrevents[c][
'call'] = 'self.' + self.ctrevents[c]['function']
def _init_variables(self, dtype=np.uint8):
if dtype == np.uint8:
cdtype = ctypes.c_ubyte
else:
cdtype = ctypes.c_ushort
self.frame = Array(
cdtype,
np.zeros([self.h, self.w, self.nchan], dtype=dtype).flatten())
self.img = np.frombuffer(self.frame.get_obj(), dtype=cdtype).reshape(
[self.h, self.w, self.nchan])
def _start_recorder(self):
if not self.recorderpar is None:
extrapar = {}
if 'binary' in self.recorderpar['recorder'].lower():
from .io import BinaryCamWriter as rec
elif 'tiff' in self.recorderpar['recorder'].lower():
from .io import TiffCamWriter as rec
elif 'ffmpeg' in self.recorderpar['recorder'].lower():
from .io import FFMPEGCamWriter as rec
if 'hwaccel' in self.recorderpar:
if 'hwaccel' in self.recorderpar.keys():
extrapar['hwaccel'] = self.recorderpar['hwaccel']
if 'compression' in self.recorderpar.keys():
extrapar['compression'] = self.recorderpar[
'compression']
else:
display('Recorder {0} not implemented'.format(
self.recorderpar['recorder']))
if 'rec' in dir():
self.recorder = rec(
self,
inQ=self.queue,
filename=self.recorderpar['filename'],
pathformat=self.recorderpar['pathformat'],
dataname=self.recorderpar['dataname'],
datafolder=self.recorderpar['datafolder'],
framesperfile=self.recorderpar['framesperfile'],
incrementruns=True,
**extrapar)
def run(self):
self._init_ctrevents()
self.img = np.frombuffer(self.frame.get_obj(),
dtype=self.dtype).reshape(
[self.h, self.w, self.nchan])
self.close_event.clear()
self._start_recorder()
while not self.close_event.is_set():
self._cam_init()
if self.stop_trigger.is_set():
break
self._cam_waitsoftwaretrigger()
if not self.stop_trigger.is_set():
self._cam_startacquisition()
self.cam_is_running = True
while not self.stop_trigger.is_set():
frame, metadata = self._cam_loop()
self._handle_frame(frame, metadata)
self._parse_command_queue()
# to be able to pause acquisition on software trigger
if not self.start_trigger.is_set(
) and not self.stop_trigger.is_set():
self._cam_stopacquisition()
self._cam_waitsoftwaretrigger()
if not self.stop_trigger.is_set():
self._cam_startacquisition()
self.cam_is_running = True
display('[Camera] Stop trigger set.')
self.start_trigger.clear()
self._cam_close()
self.cam_is_running = False
if self.was_saving:
self.was_saving = False
if self.recorder is None:
display('[Camera] Sending stop signal to the recorder.')
self.queue.put(['STOP'])
else:
self.recorder.close_run()
self.stop_trigger.clear()
if self.close_event.is_set():
break
def _handle_frame(self, frame, metadata):
#display('loop rate : {0}'.format(1./(timestamp - self.lasttime)))
if self.saving.is_set():
self.was_saving = True
if not frame is None:
if not metadata[0] == self.lastframeid:
if not self.recorder is None:
self.recorder.save(frame, metadata)
else:
self.queue.put((frame, metadata))
elif self.was_saving:
if self.recorder is None:
self.was_saving = False
display('[Camera] Sending stop signal to the recorder.')
self.queue.put(['STOP'])
else:
self.was_saving = False
self.recorder.close_run()
if not frame is None:
frameID, timestamp = metadata[:2]
if not frameID == self.lastframeid:
t = time.time()
if (t - self._tupdate) > self.refresh_period:
self._update_buffer(frame, frameID)
self._tupdate = t
#self.nframes.value += 1
self.lastframeid = frameID
self.lasttime = timestamp
def _update_buffer(self, frame, frameID):
self.img[:] = np.reshape(frame, self.img.shape)[:]
#self.memlist[0] = np.reshape(frame,[self.h,self.w,self.nchan])
def _parse_command_queue(self):
if not self.eventsQ.empty():
cmd = self.eventsQ.get()
if '=' in cmd:
cmd = cmd.split('=')
if hasattr(self, 'ctrevents'):
self._call_event(cmd[0], cmd[1])
if cmd[0] == 'filename':
if not self.recorder is None:
if hasattr(self, 'recorder'):
self.recorder.set_filename(cmd[1])
self.recorderpar['filename'] = cmd[1]
elif cmd[0] == 'log':
msg = '# {0},{1} - {2}'.format(self.lastframeid,
self.lasttime, cmd[1])
if self.recorder is None:
self.queue.put([msg])
else:
self.recorder.logfile.write(msg)
def _call_event(self, eventname, eventvalue):
if eventname in self.ctrevents.keys():
val = eval(self.ctrevents[eventname]['type'] + '(' +
str(eventvalue) + ')')
eval(self.ctrevents[eventname]['call'] + '(val)')
#print(self.ctrevents[eventname])
# else:
# display('No event found {0} {1}'.format(eventname,eventvalue))
def _cam_init(self):
'''initialize the camera'''
pass
def _cam_startacquisition(self):
'''start camera acq'''
pass
def _cam_stopacquisition(self):
'''stop camera acq'''
pass
def _cam_close(self):
'''close cam - release driver'''
pass
def _cam_loop(self):
'''get a frame and move on, returns frame,(frameID,timestamp)'''
pass
def _cam_waitsoftwaretrigger(self):
'''wait for software trigger'''
display('[{0} {1}] waiting for software trigger.'.format(
self.drivername, self.cam_id))
while not self.start_trigger.is_set() or self.stop_trigger.is_set():
# limits resolution to 1 ms
time.sleep(0.001)
if self.close_event.is_set() or self.stop_trigger.is_set():
break
self._handle_frame(
None, None) # to stop saving while waiting for triggers
if self.close_event.is_set() or self.stop_trigger.is_set():
return
self.camera_ready.clear()
def stop_acquisition(self):
self.stop_trigger.set()
def close(self):
self.close_event.set()
self.stop_acquisition()
raise AvExpt(
0,
'movie_info',
)
if (avnum_queue.empty() is True) and (proc_event.is_set()):
break
if __name__ == '__main__':
parse_genre_proc = []
log_expt_proc = []
genre_proc_queue = Queue()
avnum_proc_queue = Queue()
expt_proc_queue = Queue()
parse_genre_task_done = Event()
parse_genre_task_done.clear()
genres = database.get_genre_id()
for genre_item in genres:
genre_proc_queue.put(genre_item[0])
freeze_support()
for i in range(GENRE_PROC_NUM):
proc_name = 'Genre process ' + str(i + 1)
parse_genre_proc.append(
Process(name=proc_name,
target=parse_genre,
args=(genre_proc_queue, avnum_proc_queue, expt_proc_queue,
i, parse_genre_task_done)))
log_movie_proc = Process(name='log_movie',
target=log_movie_info,
args=(avnum_proc_queue, expt_proc_queue,
parse_genre_task_done, 'log_movie'))
if 'WifiOnOff' in actions.keys():
total_actions += 1
if priority is None:
wifi_bt_sync_event.set()
else:
priority = 'toggle'
p1 = Process(name="WifiOnOff", target=toggle_wifi,
args=(serial, wifi_bt_sync_event),
kwargs={'iteration': actions["WifiOnOff"], 'q': result_q})
# p1.start()
process.append(p1)
if "A2dpBtConnect" in actions.keys():
total_actions += 1
priority = "a2dp"
wifi_bt_sync_event.clear()
p1 = Process(name="A2dpConnect", target=pair_a2dp_device,
args=(serial, bt_a2dp_name, wifi_bt_sync_event),
kwargs={'q': result_q})
process.append(p1)
# if 'ConnectToAp' in actions.keys():
# total_actions += 1
# p1 = Process(name="ConnectAp", target=connectToAp, args=(serial, ),
# kwargs={'iteration': actions["ConnectToAp"], 'q': result_q})
# #p1.start()
# process.append(p1)
if multiple_iteration_count > 1:
log.error("Invalid iteration count as only one operation can iterate "
"multiple times")
class Artifacts(object):
max_preview_size_bytes = 65536
_flush_frequency_sec = 300.
# notice these two should match
_save_format = '.csv.gz'
_compression = 'gzip'
# hashing constants
_hash_block_size = 65536
_pd_artifact_type = 'data-audit-table'
class _ProxyDictWrite(dict):
""" Dictionary wrapper that updates an arguments instance on any item set in the dictionary """
def __init__(self, artifacts_manager, *args, **kwargs):
super(Artifacts._ProxyDictWrite, self).__init__(*args, **kwargs)
self._artifacts_manager = artifacts_manager
# list of artifacts we should not upload (by name & weak-reference)
self.artifact_metadata = {}
# list of hash columns to calculate uniqueness for the artifacts
self.artifact_hash_columns = {}
def __setitem__(self, key, value):
# check that value is of type pandas
if pd and isinstance(value, pd.DataFrame):
super(Artifacts._ProxyDictWrite, self).__setitem__(key, value)
if self._artifacts_manager:
self._artifacts_manager.flush()
else:
raise ValueError(
'Artifacts currently support pandas.DataFrame objects only'
)
def unregister_artifact(self, name):
self.artifact_metadata.pop(name, None)
self.pop(name, None)
def add_metadata(self, name, metadata):
self.artifact_metadata[name] = deepcopy(metadata)
def get_metadata(self, name):
return self.artifact_metadata.get(name)
def add_hash_columns(self, artifact_name, hash_columns):
self.artifact_hash_columns[artifact_name] = hash_columns
def get_hash_columns(self, artifact_name):
return self.artifact_hash_columns.get(artifact_name)
@property
def registered_artifacts(self):
# type: () -> Dict[str, Artifact]
return self._artifacts_container
@property
def summary(self):
# type: () -> str
return self._summary
def __init__(self, task):
self._task = task
# notice the double link, this is important since the Artifact
# dictionary needs to signal the Artifacts base on changes
self._artifacts_container = self._ProxyDictWrite(self)
self._last_artifacts_upload = {}
self._unregister_request = set()
self._thread = None
self._flush_event = Event()
self._exit_flag = False
self._summary = ''
self._temp_folder = []
self._task_artifact_list = []
self._task_edit_lock = RLock()
self._storage_prefix = None
def register_artifact(self,
name,
artifact,
metadata=None,
uniqueness_columns=True):
# type: (str, DataFrame, Optional[dict], Union[bool, Sequence[str]]) -> ()
"""
:param str name: name of the artifacts. Notice! it will override previous artifacts if name already exists.
:param pandas.DataFrame artifact: artifact object, supported artifacts object types: pandas.DataFrame
:param dict metadata: dictionary of key value to store with the artifact (visible in the UI)
:param list uniqueness_columns: list of columns for artifact uniqueness comparison criteria. The default value
is True, which equals to all the columns (same as artifact.columns).
"""
# currently we support pandas.DataFrame (which we will upload as csv.gz)
if name in self._artifacts_container:
LoggerRoot.get_base_logger().info(
'Register artifact, overwriting existing artifact \"{}\"'.
format(name))
self._artifacts_container.add_hash_columns(
name,
list(artifact.columns
if uniqueness_columns is True else uniqueness_columns))
self._artifacts_container[name] = artifact
if metadata:
self._artifacts_container.add_metadata(name, metadata)
def unregister_artifact(self, name):
# type: (str) -> ()
# Remove artifact from the watch list
self._unregister_request.add(name)
self.flush()
def upload_artifact(self,
name,
artifact_object=None,
metadata=None,
preview=None,
delete_after_upload=False,
auto_pickle=True,
wait_on_upload=False):
# type: (str, Optional[object], Optional[dict], Optional[str], bool, bool, bool) -> bool
if not Session.check_min_api_version('2.3'):
LoggerRoot.get_base_logger().warning(
'Artifacts not supported by your ClearML-server version, '
'please upgrade to the latest server version')
return False
if name in self._artifacts_container:
raise ValueError(
"Artifact by the name of {} is already registered, use register_artifact"
.format(name))
# cast preview to string
if preview is not None and not (isinstance(preview, bool)
and preview is False):
preview = str(preview)
# evaluate lazy proxy object
if isinstance(artifact_object, LazyEvalWrapper):
# noinspection PyProtectedMember
artifact_object = LazyEvalWrapper._load_object(artifact_object)
pathlib_types = (
Path,
pathlib_Path,
) if pathlib_Path is not None else (Path, )
local_filename = None
# try to convert string Path object (it might reference a file/folder)
# dont not try to serialize long texts.
if isinstance(artifact_object, six.string_types
) and artifact_object and len(artifact_object) < 2048:
# noinspection PyBroadException
try:
artifact_path = Path(artifact_object)
if artifact_path.exists():
artifact_object = artifact_path
elif '*' in artifact_object or '?' in artifact_object:
# hackish, detect wildcard in tr files
folder = Path('').joinpath(*artifact_path.parts[:-1])
if folder.is_dir() and folder.parts:
wildcard = artifact_path.parts[-1]
if list(Path(folder).rglob(wildcard)):
artifact_object = artifact_path
except Exception:
pass
store_as_pickle = False
artifact_type_data = tasks.ArtifactTypeData()
artifact_type_data.preview = ''
override_filename_in_uri = None
override_filename_ext_in_uri = None
uri = None
if np and isinstance(artifact_object, np.ndarray):
artifact_type = 'numpy'
artifact_type_data.content_type = 'application/numpy'
artifact_type_data.preview = preview or str(
artifact_object.__repr__())
override_filename_ext_in_uri = '.npz'
override_filename_in_uri = name + override_filename_ext_in_uri
fd, local_filename = mkstemp(prefix=quote(name, safe="") + '.',
suffix=override_filename_ext_in_uri)
os.close(fd)
np.savez_compressed(local_filename, **{name: artifact_object})
delete_after_upload = True
elif pd and isinstance(artifact_object, pd.DataFrame):
artifact_type = 'pandas'
artifact_type_data.content_type = 'text/csv'
artifact_type_data.preview = preview or str(
artifact_object.__repr__())
override_filename_ext_in_uri = self._save_format
override_filename_in_uri = name
fd, local_filename = mkstemp(prefix=quote(name, safe="") + '.',
suffix=override_filename_ext_in_uri)
os.close(fd)
artifact_object.to_csv(local_filename,
compression=self._compression)
delete_after_upload = True
elif isinstance(artifact_object, Image.Image):
artifact_type = 'image'
artifact_type_data.content_type = 'image/png'
desc = str(artifact_object.__repr__())
artifact_type_data.preview = preview or desc[1:desc.find(' at ')]
override_filename_ext_in_uri = '.png'
override_filename_in_uri = name + override_filename_ext_in_uri
fd, local_filename = mkstemp(prefix=quote(name, safe="") + '.',
suffix=override_filename_ext_in_uri)
os.close(fd)
artifact_object.save(local_filename)
delete_after_upload = True
elif isinstance(artifact_object, dict):
artifact_type = 'JSON'
artifact_type_data.content_type = 'application/json'
# noinspection PyBroadException
try:
json_text = json.dumps(artifact_object,
sort_keys=True,
indent=4)
except Exception:
if not auto_pickle:
raise
LoggerRoot.get_base_logger().warning(
"JSON serialization of artifact \'{}\' failed, reverting to pickle"
.format(name))
store_as_pickle = True
json_text = None
if json_text is not None:
override_filename_ext_in_uri = '.json'
override_filename_in_uri = name + override_filename_ext_in_uri
fd, local_filename = mkstemp(
prefix=quote(name, safe="") + '.',
suffix=override_filename_ext_in_uri)
os.write(fd, bytes(json_text.encode()))
os.close(fd)
preview = preview or json_text
if len(preview) < self.max_preview_size_bytes:
artifact_type_data.preview = preview
else:
artifact_type_data.preview = '# full json too large to store, storing first {}kb\n{}'.format(
self.max_preview_size_bytes // 1024,
preview[:self.max_preview_size_bytes])
delete_after_upload = True
elif isinstance(artifact_object, pathlib_types):
# check if single file
artifact_object = Path(artifact_object)
artifact_object.expanduser().absolute()
# noinspection PyBroadException
try:
create_zip_file = not artifact_object.is_file()
except Exception: # Hack for windows pathlib2 bug, is_file isn't valid.
create_zip_file = True
else: # We assume that this is not Windows os
if artifact_object.is_dir():
# change to wildcard
artifact_object /= '*'
if create_zip_file:
folder = Path('').joinpath(*artifact_object.parts[:-1])
if not folder.is_dir() or not folder.parts:
raise ValueError(
"Artifact file/folder '{}' could not be found".format(
artifact_object.as_posix()))
wildcard = artifact_object.parts[-1]
files = list(Path(folder).rglob(wildcard))
override_filename_ext_in_uri = '.zip'
override_filename_in_uri = folder.parts[
-1] + override_filename_ext_in_uri
fd, zip_file = mkstemp(
prefix=quote(folder.parts[-1], safe="") + '.',
suffix=override_filename_ext_in_uri)
try:
artifact_type_data.content_type = 'application/zip'
archive_preview = 'Archive content {}:\n'.format(
artifact_object.as_posix())
with ZipFile(zip_file,
'w',
allowZip64=True,
compression=ZIP_DEFLATED) as zf:
for filename in sorted(files):
if filename.is_file():
relative_file_name = filename.relative_to(
folder).as_posix()
archive_preview += '{} - {}\n'.format(
relative_file_name,
format_size(filename.stat().st_size))
zf.write(filename.as_posix(),
arcname=relative_file_name)
except Exception as e:
# failed uploading folder:
LoggerRoot.get_base_logger().warning(
'Exception {}\nFailed zipping artifact folder {}'.
format(folder, e))
return False
finally:
os.close(fd)
artifact_type_data.preview = preview or archive_preview
artifact_object = zip_file
artifact_type = 'archive'
artifact_type_data.content_type = mimetypes.guess_type(
artifact_object)[0]
local_filename = artifact_object
delete_after_upload = True
else:
if not artifact_object.is_file():
raise ValueError(
"Artifact file '{}' could not be found".format(
artifact_object.as_posix()))
override_filename_in_uri = artifact_object.parts[-1]
artifact_type_data.preview = preview or '{} - {}\n'.format(
artifact_object, format_size(
artifact_object.stat().st_size))
artifact_object = artifact_object.as_posix()
artifact_type = 'custom'
artifact_type_data.content_type = mimetypes.guess_type(
artifact_object)[0]
local_filename = artifact_object
elif isinstance(artifact_object, (list, tuple)) and all(
isinstance(p, pathlib_types) for p in artifact_object):
# find common path if exists
list_files = [Path(p) for p in artifact_object]
override_filename_ext_in_uri = '.zip'
override_filename_in_uri = quote(
name, safe="") + override_filename_ext_in_uri
common_path = get_common_path(list_files)
fd, zip_file = mkstemp(prefix='artifact_folder.',
suffix=override_filename_ext_in_uri)
try:
artifact_type_data.content_type = 'application/zip'
archive_preview = 'Archive content:\n'
with ZipFile(zip_file,
'w',
allowZip64=True,
compression=ZIP_DEFLATED) as zf:
for filename in sorted(list_files):
if filename.is_file():
relative_file_name = filename.relative_to(Path(common_path)).as_posix() \
if common_path else filename.as_posix()
archive_preview += '{} - {}\n'.format(
relative_file_name,
format_size(filename.stat().st_size))
zf.write(filename.as_posix(),
arcname=relative_file_name)
else:
LoggerRoot.get_base_logger().warning(
"Failed zipping artifact file '{}', file not found!"
.format(filename.as_posix()))
except Exception as e:
# failed uploading folder:
LoggerRoot.get_base_logger().warning(
'Exception {}\nFailed zipping artifact files {}'.format(
artifact_object, e))
return False
finally:
os.close(fd)
artifact_type_data.preview = preview or archive_preview
artifact_object = zip_file
artifact_type = 'archive'
artifact_type_data.content_type = mimetypes.guess_type(
artifact_object)[0]
local_filename = artifact_object
delete_after_upload = True
elif (isinstance(artifact_object, six.string_types)
and len(artifact_object) < 4096
and urlparse(artifact_object).scheme in remote_driver_schemes):
# we should not upload this, just register
local_filename = None
uri = artifact_object
artifact_type = 'custom'
artifact_type_data.content_type = mimetypes.guess_type(
artifact_object)[0]
if preview:
artifact_type_data.preview = preview
elif isinstance(artifact_object, six.string_types):
# if we got here, we should store it as text file.
artifact_type = 'string'
artifact_type_data.content_type = 'text/plain'
if preview:
artifact_type_data.preview = preview
elif len(artifact_object) < self.max_preview_size_bytes:
artifact_type_data.preview = artifact_object
else:
artifact_type_data.preview = '# full text too large to store, storing first {}kb\n{}'.format(
self.max_preview_size_bytes // 1024,
artifact_object[:self.max_preview_size_bytes])
if artifact_object:
delete_after_upload = True
override_filename_ext_in_uri = '.txt'
override_filename_in_uri = name + override_filename_ext_in_uri
fd, local_filename = mkstemp(
prefix=quote(name, safe="") + '.',
suffix=override_filename_ext_in_uri)
os.close(fd)
# noinspection PyBroadException
try:
with open(local_filename, 'wt') as f:
f.write(artifact_object)
except Exception:
# cleanup and raise exception
os.unlink(local_filename)
raise
elif artifact_object is None or (isinstance(artifact_object, str)
and artifact_object == ""):
artifact_type = ''
store_as_pickle = False
elif auto_pickle:
# revert to pickling the object
store_as_pickle = True
else:
raise ValueError("Artifact type {} not supported".format(
type(artifact_object)))
# revert to serializing the object with pickle
if store_as_pickle:
# if we are here it means we do not know what to do with the object, so we serialize it with pickle.
artifact_type = 'pickle'
artifact_type_data.content_type = 'application/pickle'
# noinspection PyBroadException
try:
artifact_type_data.preview = preview or str(
artifact_object.__repr__())[:self.max_preview_size_bytes]
except Exception:
artifact_type_data.preview = preview or ''
delete_after_upload = True
override_filename_ext_in_uri = '.pkl'
override_filename_in_uri = name + override_filename_ext_in_uri
fd, local_filename = mkstemp(prefix=quote(name, safe="") + '.',
suffix=override_filename_ext_in_uri)
os.close(fd)
# noinspection PyBroadException
try:
with open(local_filename, 'wb') as f:
pickle.dump(artifact_object, f)
except Exception:
# cleanup and raise exception
os.unlink(local_filename)
raise
# verify preview not out of scope:
if artifact_type_data.preview and len(artifact_type_data.preview) > (
self.max_preview_size_bytes + 1024):
artifact_type_data.preview = '# full preview too large to store, storing first {}kb\n{}'.format(
self.max_preview_size_bytes // 1024,
artifact_type_data.preview[:self.max_preview_size_bytes])
# remove from existing list, if exists
for artifact in self._task_artifact_list:
if artifact.key == name:
if artifact.type == self._pd_artifact_type:
raise ValueError(
"Artifact of name {} already registered, "
"use register_artifact instead".format(name))
self._task_artifact_list.remove(artifact)
break
if not local_filename:
file_size = None
file_hash = None
else:
# check that the file to upload exists
local_filename = Path(local_filename).absolute()
if not local_filename.exists() or not local_filename.is_file():
LoggerRoot.get_base_logger().warning(
'Artifact upload failed, cannot find file {}'.format(
local_filename.as_posix()))
return False
file_hash, _ = sha256sum(local_filename.as_posix(),
block_size=Artifacts._hash_block_size)
file_size = local_filename.stat().st_size
uri = self._upload_local_file(
local_filename,
name,
delete_after_upload=delete_after_upload,
override_filename=override_filename_in_uri,
override_filename_ext=override_filename_ext_in_uri,
wait_on_upload=wait_on_upload)
timestamp = int(time())
artifact = tasks.Artifact(
key=name,
type=artifact_type,
uri=uri,
content_size=file_size,
hash=file_hash,
timestamp=timestamp,
type_data=artifact_type_data,
display_data=[(str(k), str(v))
for k, v in metadata.items()] if metadata else None)
# update task artifacts
self._add_artifact(artifact)
return True
def flush(self):
# type: () -> ()
# start the thread if it hasn't already:
self._start()
# flush the current state of all artifacts
self._flush_event.set()
def stop(self, wait=True):
# type: (bool) -> ()
# stop the daemon thread and quit
# wait until thread exists
self._exit_flag = True
self._flush_event.set()
if wait:
if self._thread:
self._thread.join()
# remove all temp folders
for f in self._temp_folder:
# noinspection PyBroadException
try:
Path(f).rmdir()
except Exception:
pass
def _start(self):
# type: () -> ()
""" Start daemon thread if any artifacts are registered and thread is not up yet """
if not self._thread and self._artifacts_container:
# start the daemon thread
self._flush_event.clear()
self._thread = Thread(target=self._daemon)
self._thread.daemon = True
self._thread.start()
def _daemon(self):
# type: () -> ()
while not self._exit_flag:
self._flush_event.wait(self._flush_frequency_sec)
self._flush_event.clear()
artifact_keys = list(self._artifacts_container.keys())
for name in artifact_keys:
try:
self._upload_data_audit_artifacts(name)
except Exception as e:
LoggerRoot.get_base_logger().warning(str(e))
# create summary
self._summary = self._get_statistics()
def _add_artifact(self, artifact):
if not self._task:
raise ValueError("Task object not set")
with self._task_edit_lock:
if artifact not in self._task_artifact_list:
self._task_artifact_list.append(artifact)
# noinspection PyProtectedMember
self._task._add_artifacts(self._task_artifact_list)
def _upload_data_audit_artifacts(self, name):
# type: (str) -> ()
logger = self._task.get_logger()
pd_artifact = self._artifacts_container.get(name)
pd_metadata = self._artifacts_container.get_metadata(name)
# remove from artifacts watch list
if name in self._unregister_request:
try:
self._unregister_request.remove(name)
except KeyError:
pass
self._artifacts_container.unregister_artifact(name)
if pd_artifact is None:
return
override_filename_ext_in_uri = self._save_format
override_filename_in_uri = name
fd, local_csv = mkstemp(prefix=quote(name, safe="") + '.',
suffix=override_filename_ext_in_uri)
os.close(fd)
local_csv = Path(local_csv)
pd_artifact.to_csv(local_csv.as_posix(),
index=False,
compression=self._compression)
current_sha2, file_sha2 = sha256sum(
local_csv.as_posix(),
skip_header=32,
block_size=Artifacts._hash_block_size)
if name in self._last_artifacts_upload:
previous_sha2 = self._last_artifacts_upload[name]
if previous_sha2 == current_sha2:
# nothing to do, we can skip the upload
# noinspection PyBroadException
try:
local_csv.unlink()
except Exception:
pass
return
self._last_artifacts_upload[name] = current_sha2
# If old clearml-server, upload as debug image
if not Session.check_min_api_version('2.3'):
logger.report_image(title='artifacts',
series=name,
local_path=local_csv.as_posix(),
delete_after_upload=True,
iteration=self._task.get_last_iteration(),
max_image_history=2)
return
# Find our artifact
artifact = None
for an_artifact in self._task_artifact_list:
if an_artifact.key == name:
artifact = an_artifact
break
file_size = local_csv.stat().st_size
# upload file
uri = self._upload_local_file(
local_csv,
name,
delete_after_upload=True,
override_filename=override_filename_in_uri,
override_filename_ext=override_filename_ext_in_uri)
# update task artifacts
with self._task_edit_lock:
if not artifact:
artifact = tasks.Artifact(key=name,
type=self._pd_artifact_type)
artifact_type_data = tasks.ArtifactTypeData()
artifact_type_data.data_hash = current_sha2
artifact_type_data.content_type = "text/csv"
artifact_type_data.preview = str(
pd_artifact.__repr__()) + '\n\n' + self._get_statistics(
{name: pd_artifact})
artifact.type_data = artifact_type_data
artifact.uri = uri
artifact.content_size = file_size
artifact.hash = file_sha2
artifact.timestamp = int(time())
artifact.display_data = [
(str(k), str(v)) for k, v in pd_metadata.items()
] if pd_metadata else None
self._add_artifact(artifact)
def _upload_local_file(self,
local_file,
name,
delete_after_upload=False,
override_filename=None,
override_filename_ext=None,
wait_on_upload=False):
# type: (str, str, bool, Optional[str], Optional[str], bool) -> str
"""
Upload local file and return uri of the uploaded file (uploading in the background)
"""
from clearml.storage import StorageManager
upload_uri = self._task.output_uri or self._task.get_logger(
).get_default_upload_destination()
if not isinstance(local_file, Path):
local_file = Path(local_file)
ev = UploadEvent(
metric='artifacts',
variant=name,
image_data=None,
upload_uri=upload_uri,
local_image_path=local_file.as_posix(),
delete_after_upload=delete_after_upload,
override_filename=override_filename,
override_filename_ext=override_filename_ext,
override_storage_key_prefix=self._get_storage_uri_prefix())
_, uri = ev.get_target_full_upload_uri(upload_uri, quote_uri=False)
# send for upload
# noinspection PyProtectedMember
if wait_on_upload:
StorageManager.upload_file(local_file.as_posix(),
uri,
wait_for_upload=True,
retries=ev.retries)
if delete_after_upload:
try:
os.unlink(local_file.as_posix())
except OSError:
LoggerRoot.get_base_logger().warning(
'Failed removing temporary {}'.format(local_file))
else:
self._task._reporter._report(ev)
_, quoted_uri = ev.get_target_full_upload_uri(upload_uri)
return quoted_uri
def _get_statistics(self, artifacts_dict=None):
# type: (Optional[Dict[str, Artifact]]) -> str
summary = ''
artifacts_dict = artifacts_dict or self._artifacts_container
thread_pool = ThreadPool()
try:
# build hash row sets
artifacts_summary = []
for a_name, a_df in artifacts_dict.items():
hash_cols = self._artifacts_container.get_hash_columns(a_name)
if not pd or not isinstance(a_df, pd.DataFrame):
continue
if hash_cols is True:
hash_col_drop = []
else:
hash_cols = set(hash_cols)
missing_cols = hash_cols.difference(a_df.columns)
if missing_cols == hash_cols:
LoggerRoot.get_base_logger().warning(
'Uniqueness columns {} not found in artifact {}. '
'Skipping uniqueness check for artifact.'.format(
list(missing_cols), a_name))
continue
elif missing_cols:
# missing_cols must be a subset of hash_cols
hash_cols.difference_update(missing_cols)
LoggerRoot.get_base_logger().warning(
'Uniqueness columns {} not found in artifact {}. Using {}.'
.format(list(missing_cols), a_name,
list(hash_cols)))
hash_col_drop = [
col for col in a_df.columns if col not in hash_cols
]
a_unique_hash = set()
def hash_row(r):
a_unique_hash.add(hash(bytes(r)))
a_shape = a_df.shape
# parallelize
a_hash_cols = a_df.drop(columns=hash_col_drop)
thread_pool.map(hash_row, a_hash_cols.values)
# add result
artifacts_summary.append((
a_name,
a_shape,
a_unique_hash,
))
# build intersection summary
for i, (name, shape, unique_hash) in enumerate(artifacts_summary):
summary += '[{name}]: shape={shape}, {unique} unique rows, {percentage:.1f}% uniqueness\n'.format(
name=name,
shape=shape,
unique=len(unique_hash),
percentage=100 * len(unique_hash) / float(shape[0]))
for name2, shape2, unique_hash2 in artifacts_summary[i + 1:]:
intersection = len(unique_hash & unique_hash2)
summary += '\tIntersection with [{name2}] {intersection} rows: {percentage:.1f}%\n'.format(
name2=name2,
intersection=intersection,
percentage=100 * intersection /
float(len(unique_hash2)))
except Exception as e:
LoggerRoot.get_base_logger().warning(str(e))
finally:
thread_pool.close()
thread_pool.terminate()
return summary
def _get_temp_folder(self, force_new=False):
# type: (bool) -> str
if force_new or not self._temp_folder:
new_temp = mkdtemp(prefix='artifacts_')
self._temp_folder.append(new_temp)
return new_temp
return self._temp_folder[0]
def _get_storage_uri_prefix(self):
# type: () -> str
if not self._storage_prefix:
# noinspection PyProtectedMember
self._storage_prefix = self._task._get_output_destination_suffix()
return self._storage_prefix
class Coordinator(object):
"""A coordinator for processes.
This class implements a simple mechanism to coordinate the termination of a
set of processes.
```python
with coord.stop_on_exception():
while not coord.should_stop():
...do some work...
```
"""
def __init__(self):
"""Create a new Coordinator.
"""
# Protects all attributes.
self._lock = Lock()
# Event set when processes must stop.
self._stop_event = Event()
# Python exc_info to report.
# If not None, it should hold the returned value of sys.exc_info(), which is
# a tuple containing exception (type, value, traceback).
self._exc_info_to_raise = None
# True if we have called join() already.
self._joined = Value(ctypes.c_int, 0, lock=False)
# Set of processes registered for joining when join() is called. These
# will be joined in addition to the processes passed to the join()
# call. It's ok if processes are both registered and passed to the join()
# call.
self._registered_processes = set()
def request_stop(self, ex=None):
"""Request that the processes stop.
After this is called, calls to `should_stop()` will return `True`.
Note: If an exception is being passed in, in must be in the context of
handling the exception (i.e. `try: ... except Exception as ex: ...`) and not
a newly created one.
Args:
ex (Exception or exc_info tuple): Optional `Exception`, or
Python `exc_info` tuple as returned by `sys.exc_info()`.
If this is the first call to `request_stop()` the
corresponding exception is recorded and re-raised from `join()`.
"""
with self._lock:
# If we have already joined the coordinator the exception will not have a
# chance to be reported, so just raise it normally. This can happen if
# you continue to use a session have having stopped and joined the
# coordinator process.
if self.joined:
if isinstance(ex, tuple):
six.reraise(*ex)
elif ex is not None:
# NOTE(touts): This is bogus if request_stop() is not called
# from the exception handler that raised ex.
six.reraise(*sys.exc_info())
if not self._stop_event.is_set():
if ex and self._exc_info_to_raise is None:
if isinstance(ex, tuple):
logging.info("Error reported to Coordinator: %s",
str(ex[1]),
exc_info=ex)
self._exc_info_to_raise = ex
else:
logging.info("Error reported to Coordinator: %s, %s",
type(ex), str(ex))
self._exc_info_to_raise = sys.exc_info()
# self._exc_info_to_raise should contain a tuple containing exception
# (type, value, traceback)
if (len(self._exc_info_to_raise) != 3
or not self._exc_info_to_raise[0]
or not self._exc_info_to_raise[1]):
# Raise, catch and record the exception here so that error happens
# where expected.
try:
raise ValueError(
"ex must be a tuple or sys.exc_info must "
"return the current exception: %s" %
self._exc_info_to_raise)
except ValueError:
# Record this error so it kills the coordinator properly.
# NOTE(touts): As above, this is bogus if request_stop() is not
# called from the exception handler that raised ex.
self._exc_info_to_raise = sys.exc_info()
self._stop_event.set()
def clear_stop(self):
"""Clears the stop flag.
After this is called, calls to `should_stop()` will return `False`.
"""
with self._lock:
self._joined.value = 0
self._exc_info_to_raise = None
if self._stop_event.is_set():
self._stop_event.clear()
def should_stop(self):
"""Check if stop was requested.
Returns:
True if a stop was requested.
"""
return self._stop_event.is_set()
@contextlib.contextmanager
def stop_on_exception(self):
"""Context manager to request stop when an Exception is raised.
Code that uses a coordinator must catch exceptions and pass
them to the `request_stop()` method to stop the other processes
managed by the coordinator.
This context handler simplifies the exception handling.
Use it as follows:
```python
with coord.stop_on_exception():
# Any exception raised in the body of the with
# clause is reported to the coordinator before terminating
# the execution of the body.
...body...
```
This is completely equivalent to the slightly longer code:
```python
try:
...body...
except:
coord.request_stop(sys.exc_info())
```
Yields:
nothing.
"""
try:
yield
except: # pylint: disable=bare-except
self.request_stop(ex=sys.exc_info())
def wait_for_stop(self, timeout=None):
"""Wait till the Coordinator is told to stop.
Args:
timeout: Float. Sleep for up to that many seconds waiting for
should_stop() to become True.
Returns:
True if the Coordinator is told stop, False if the timeout expired.
"""
return self._stop_event.wait(timeout)
def register_process(self, process):
"""Register a process to join.
Args:
process: A python.multiprocessing.Process to join.
"""
with self._lock:
self._registered_processes.add(process)
def join(self,
processes=None,
stop_grace_period_secs=120,
ignore_live_processes=False):
"""Wait for processes to terminate.
This call blocks until a set of processes have terminated. The set of process
is the union of the processes passed in the `processes` argument and the list
of processes that registered with the coordinator by calling
`Coordinator.register_process()`.
After the processes stop, if an `exc_info` was passed to `request_stop`, that
exception is re-raised.
Grace period handling: When `request_stop()` is called, processes are given
'stop_grace_period_secs' seconds to terminate. If any of them is still
alive after that period expires, a `RuntimeError` is raised. Note that if
an `exc_info` was passed to `request_stop()` then it is raised instead of
that `RuntimeError`.
Args:
processes (list of `Processes`): The started processes to join in
addition to the registered processes.
stop_grace_period_secs: Number of seconds given to processes to stop after
`request_stop()` has been called.
ignore_live_processes: If `False`, raises an error if any of the processes are
still alive after `stop_grace_period_secs`.
Raises:
RuntimeError: If any process is still alive after `request_stop()`
is called and the grace period expires.
"""
# processes registered after this call will not be joined.
with self._lock:
if processes is None:
processes = self._registered_processes
else:
processes = self._registered_processes.union(set(processes))
# Copy the set into a list to avoid race conditions where a new process
# is added while we are waiting.
processes = list(processes)
# Wait for all processes to stop or for request_stop() to be called.
while any(t.is_alive()
for t in processes) and not self.wait_for_stop(1.0):
pass
# If any process is still alive, wait for the grace period to expire.
# By the time this check is executed, processes may still be shutting down,
# so we add a sleep of increasing duration to give them a chance to shut
# down without losing too many cycles.
# The sleep duration is limited to the remaining grace duration.
stop_wait_secs = 0.001
while any(t.is_alive()
for t in processes) and stop_grace_period_secs >= 0.0:
time.sleep(stop_wait_secs)
stop_grace_period_secs -= stop_wait_secs
stop_wait_secs = 2 * stop_wait_secs
# Keep the waiting period within sane bounds.
# The minimum value is to avoid decreasing stop_wait_secs to a value
# that could cause stop_grace_period_secs to remain unchanged.
stop_wait_secs = max(min(stop_wait_secs, stop_grace_period_secs),
0.001)
# List the processes still alive after the grace period.
stragglers = [t.name for t in processes if t.is_alive()]
# Terminate with an exception if appropriate.
with self._lock:
self._joined.value = 1
self._registered_processes = set()
if self._exc_info_to_raise:
six.reraise(*self._exc_info_to_raise)
elif stragglers:
if ignore_live_processes:
logging.info(
"Coordinator stopped with processes still running: %s",
" ".join(stragglers))
else:
raise RuntimeError(
"Coordinator stopped with processes still running: %s"
% " ".join(stragglers))
@property
def joined(self):
return self._joined.value
def raise_requested_exception(self):
"""If an exception has been passed to `request_stop`, this raises it."""
with self._lock:
if self._exc_info_to_raise:
six.reraise(*self._exc_info_to_raise)
class Plant(object):
def __init__(self,
params=None,
x0=None,
S0=None,
dt=0.01,
noise=None,
name='Plant',
angle_dims=[]):
self.name = name
self.params = params
self.x0 = x0
self.S0 = S0
self.x = np.array(x0, dtype=np.float64).flatten()
self.u = None
self.t = 0
self.dt = dt
self.noise = noise
self.running = Event()
self.done = False
self.plant_thread = None
self.angle_dims = angle_dims
def apply_control(self, u):
self.u = np.array(u, dtype=np.float64)
if len(self.u.shape) < 2:
self.u = self.u[:, None]
def get_plant_state(self):
if self.angle_dims is None:
return self.x.flatten(), self.t
else:
return gTrig_np(self.x, self.angle_dims).flatten(), self.t
def run(self):
start_time = time()
print('Starting plant loop', self.name)
while self.running.is_set():
exec_time = time()
self.step(self.dt)
exec_time = time() - exec_time
sleep(max(self.dt - exec_time, 0))
def start(self):
if self.plant_thread is not None and self.plant_thread.is_alive():
while self.plant_thread.is_alive():
sleep(1.0)
self.plant_thread = Thread(target=self.run)
self.plant_thread.daemon = True
self.running.set()
self.plant_thread.start()
def stop(self):
self.running.clear()
if self.plant_thread is not None and self.plant_thread.is_alive():
# wait until thread stops
self.plant_thread.join(10)
# create new thread object, since python threads can only be started once
self.plant_thread = Thread(target=self.run)
self.plant_thread.daemon = True
def step(self):
raise NotImplementedError(
"You need to implement the step method in your Plant subclass.")
def reset_state(self):
raise NotImplementedError(
"You need to implement the reset_state method in your Plant subclass."
)
class PlantDraw(object):
def __init__(self, plant, refresh_period=(1.0 / 24), name='PlantDraw'):
super(PlantDraw, self).__init__()
self.name = name
self.plant = plant
self.drawing_thread = None
self.polling_thread = None
self.dt = refresh_period
self.scale = 150 # pixels per meter
self.center_x = 0
self.center_y = 0
self.running = Event()
self.polling_pipe, self.drawing_pipe = Pipe()
def init_ui(self):
self.fig = plt.figure(self.name) #,figsize=(16,10))
plt.xlim([-1.5, 1.5])
plt.ylim([-1.5, 1.5])
self.ax = plt.gca()
self.ax.set_aspect('equal', 'datalim')
self.ax.grid(True)
self.bg = self.fig.canvas.copy_from_bbox(self.ax.bbox)
self.init_artists()
self.fig.canvas.draw()
self.cursor = Cursor(self.ax, useblit=True, color='red', linewidth=2)
plt.ion()
plt.show()
def drawing_loop(self, drawing_pipe):
# start the matplotlib plotting
self.init_ui()
while self.running.is_set():
exec_time = time()
# get any data from the polling loop
updts = None
while drawing_pipe.poll():
data_from_plant = drawing_pipe.recv()
if data_from_plant is None:
self.running.clear()
break
# get the visuzlization updates from the latest state
state, t = data_from_plant
updts = self.update(state, t)
if updts is not None:
# update the drawing from the plant state
self.fig.canvas.restore_region(self.bg)
for artist in updts:
self.ax.draw_artist(artist)
self.fig.canvas.blit(self.ax.bbox)
# sleep to guarantee the desired frame rate
exec_time = time() - exec_time
sleep(max(self.dt - exec_time, 0))
# close the matplotlib windows, clean up
plt.ioff()
plt.close(self.fig)
def polling_loop(self, polling_pipe):
current_t = -1
while self.running.is_set():
exec_time = time()
state, t = self.plant.get_plant_state()
if t != current_t:
polling_pipe.send((state, t))
# sleep to guarantee the desired frame rate
exec_time = time() - exec_time
sleep(max(self.dt - exec_time, 0))
def start(self):
self.drawing_thread = Process(target=self.drawing_loop,
args=(self.drawing_pipe, ))
self.drawing_thread.daemon = True
self.polling_thread = Thread(target=self.polling_loop,
args=(self.polling_pipe, ))
self.polling_thread.daemon = True
#self.drawing_thread = Process(target=self.run)
self.running.set()
self.polling_thread.start()
self.drawing_thread.start()
def stop(self):
self.running.clear()
if self.drawing_thread is not None and self.drawing_thread.is_alive():
# wait until thread stops
self.drawing_thread.join(10)
if self.polling_thread is not None and self.polling_thread.is_alive():
# wait until thread stops
self.polling_thread.join(10)
def update(self):
raise NotImplementedError(
"You need to implement the self.update() method in your PlantDraw class."
)
def init_artists(self):
raise NotImplementedError(
"You need to implement the self.init_artists() method in your PlantDraw class."
)
class ValkkaProcess(Process):
"""
Semantics:
Frontend: the part of the forked process that keeps running in the current, user virtual memory space
Backend : the part of the forked process that runs in its own virtual memory space (e.g. "in the background")
This class has both backend and frontend methods:
- Backend methods should only be called from backend. They are designated with "_".
- Frontend methods should only be called from frontend
To avoid confusion, backend methods are designated with "_", except for the "run()" method, that's always in the backend
Frontend methods use a pipe to send a signal to backend that then handles the signal with a backend method having the same name (but with "_" in the end)
Backend methods can, in a similar fashion, send signals to the frontend using a pipe. In frontend, a listening thread is needed. That thread can then call
the handleSignal method that chooses the correct frontend method to call
TODO: add the possibility to bind the process to a certain processor
"""
# incoming signals : from frontend to backend
incoming_signal_defs = { # each key corresponds to a front- and backend methods
"test_": {"test_int": int, "test_str": str},
"stop_": []
}
# outgoing signals : from back to frontend. Don't use same names as for
# incoming signals ..
outgoing_signal_defs = {
"test_o": {"test_int": int, "test_str": str},
}
def __init__(self, name, affinity=-1, **kwargs):
super().__init__()
self.pre = self.__class__.__name__ + " : " + name + \
" : " # auxiliary string for debugging output
self.name = name
self.affinity = affinity
self.signal_in = Event()
self.signal_out = Event()
# communications pipe. Frontend uses self.pipe, backend self.childpipe
self.pipe, self.childpipe = Pipe()
self.signal_in.clear()
self.signal_out.clear()
# print(self.pre, "init")
def getPipe(self):
"""Returns communication pipe for front-end
"""
return self.pipe
def preRun_(self):
"""After the fork, but before starting the process loop
"""
if (self.affinity > -1):
os.system("taskset -p -c %d %d" % (self.affinity, os.getpid()))
def postRun_(self):
"""Just before process exit
"""
print(self.pre, "post: bye!")
def cycle_(self):
# Do whatever your process should be doing, remember timeout every now
# and then
time.sleep(5)
print(self.pre, "hello!")
def startAsThread(self):
from threading import Thread
t = Thread(target=self.run)
t.start()
def run(self): # No "_" in the name, but nevertheless, running in the backed
"""After the fork. Now the process starts running
"""
# print(self.pre," ==> run")
self.preRun_()
self.running = True
while(self.running):
self.cycle_()
self.handleSignal_()
self.postRun_()
def handleSignal_(self):
"""Signals handling in the backend
"""
if (self.signal_in.is_set()):
signal_dic = self.childpipe.recv()
method_name = signal_dic.pop("name")
method = getattr(self, method_name)
method(**signal_dic)
self.signal_in.clear()
self.signal_out.set()
def sendSignal(self, **kwargs): # sendSignal(name="test",test_int=1,test_str="kokkelis")
"""Incoming signals: this is used by frontend methods to send signals to the backend
"""
try:
name = kwargs.pop("name")
except KeyError:
raise(AttributeError("Signal name missing"))
# a dictionary: {"parameter_name" : parameter_type}
model = self.incoming_signal_defs[name]
for key in kwargs:
# raises error if user is using undefined signal
model_type = model[key]
parameter_type = kwargs[key].__class__
if (model_type == parameter_type):
pass
else:
raise(AttributeError("Wrong type for parameter " + str(key)))
kwargs["name"] = name
self.pipe.send(kwargs)
self.signal_out.clear()
self.signal_in. set() # indicate that there is a signal
self.signal_out.wait() # wait for the backend to clear the signal
def handleSignal(self, signal_dic):
"""Signal handling in the frontend
"""
method_name = signal_dic.pop("name")
method = getattr(self, method_name)
method(**signal_dic)
def sendSignal_(self, **kwargs): # sendSignal_(name="test_out",..)
"""Outgoing signals: signals from backend to frontend
"""
try:
name = kwargs.pop("name")
except KeyError:
raise(AttributeError("Signal name missing"))
# a dictionary: {"parameter_name" : parameter_type}
model = self.outgoing_signal_defs[name]
for key in kwargs:
# raises error if user is using undefined signal
try:
model_type = model[key]
except KeyError:
print("your outgoing_signal_defs for",name,"is:", model)
print("you requested key:", key)
raise
parameter_type = kwargs[key].__class__
if (model_type == parameter_type):
pass
else:
raise(AttributeError("Wrong type for parameter " + str(key)))
kwargs["name"] = name
self.childpipe.send(kwargs)
# *** backend methods corresponding to each incoming signals ***
def stop_(self):
self.running = False
def test_(self, test_int=0, test_str="nada"):
print(self.pre, "test_ signal received with", test_int, test_str)
# ** frontend methods corresponding to each incoming signal: these communicate with the backend via pipes **
def stop(self):
self.sendSignal(name="stop_")
def test(self, **kwargs):
dictionaryCheck(self.incoming_signal_defs["test_"], kwargs)
kwargs["name"] = "test_"
self.sendSignal(**kwargs)
# ** frontend methods corresponding to each outgoing signal **
# typically, there is a QThread in the frontend-side reading the process pipe
# the QThread reads kwargs dictionary from the pipe, say
# {"name":"test_o", "test_str":"eka", "test_int":1}
# And calls handleSignal(kwargs)
def test_o(self, **kwargs):
pass
class MultiProcessConsumer(Consumer):
"""
A consumer implementation that consumes partitions for a topic in
parallel using multiple processes
client: a connected KafkaClient
group: a name for this consumer, used for offset storage and must be unique
topic: the topic to consume
auto_commit: default True. Whether or not to auto commit the offsets
auto_commit_every_n: default 100. How many messages to consume
before a commit
auto_commit_every_t: default 5000. How much time (in milliseconds) to
wait before commit
num_procs: Number of processes to start for consuming messages.
The available partitions will be divided among these processes
partitions_per_proc: Number of partitions to be allocated per process
(overrides num_procs)
Auto commit details:
If both auto_commit_every_n and auto_commit_every_t are set, they will
reset one another when one is triggered. These triggers simply call the
commit method on this class. A manual call to commit will also reset
these triggers
"""
def __init__(self, client, group, topic, auto_commit=True,
auto_commit_every_n=AUTO_COMMIT_MSG_COUNT,
auto_commit_every_t=AUTO_COMMIT_INTERVAL,
num_procs=1, partitions_per_proc=0):
# Initiate the base consumer class
super(MultiProcessConsumer, self).__init__(
client, group, topic,
partitions=None,
auto_commit=auto_commit,
auto_commit_every_n=auto_commit_every_n,
auto_commit_every_t=auto_commit_every_t)
# Variables for managing and controlling the data flow from
# consumer child process to master
self.queue = MPQueue(1024) # Child consumers dump messages into this
self.start = Event() # Indicates the consumers to start fetch
self.exit = Event() # Requests the consumers to shutdown
self.pause = Event() # Requests the consumers to pause fetch
self.size = Value('i', 0) # Indicator of number of messages to fetch
partitions = self.offsets.keys()
# If unspecified, start one consumer per partition
# The logic below ensures that
# * we do not cross the num_procs limit
# * we have an even distribution of partitions among processes
if not partitions_per_proc:
partitions_per_proc = round(len(partitions) * 1.0 / num_procs)
if partitions_per_proc < num_procs * 0.5:
partitions_per_proc += 1
# The final set of chunks
chunker = lambda *x: [] + list(x)
chunks = map(chunker, *[iter(partitions)] * int(partitions_per_proc))
self.procs = []
for chunk in chunks:
chunk = filter(lambda x: x is not None, chunk)
args = (client.copy(),
group, topic, chunk,
self.queue, self.start, self.exit,
self.pause, self.size)
proc = Process(target=_mp_consume, args=args)
proc.daemon = True
proc.start()
self.procs.append(proc)
def __repr__(self):
return '<MultiProcessConsumer group=%s, topic=%s, consumers=%d>' % \
(self.group, self.topic, len(self.procs))
def stop(self):
# Set exit and start off all waiting consumers
self.exit.set()
self.pause.set()
self.start.set()
for proc in self.procs:
proc.join()
proc.terminate()
super(MultiProcessConsumer, self).stop()
def __iter__(self):
"""
Iterator to consume the messages available on this consumer
"""
# Trigger the consumer procs to start off.
# We will iterate till there are no more messages available
self.size.value = 0
self.pause.set()
while True:
self.start.set()
try:
# We will block for a small while so that the consumers get
# a chance to run and put some messages in the queue
# TODO: This is a hack and will make the consumer block for
# at least one second. Need to find a better way of doing this
partition, message = self.queue.get(block=True, timeout=1)
except Empty:
break
# Count, check and commit messages if necessary
self.offsets[partition] = message.offset + 1
self.start.clear()
self.count_since_commit += 1
self._auto_commit()
yield message
self.start.clear()
def get_messages(self, count=1, block=True, timeout=10):
"""
Fetch the specified number of messages
count: Indicates the maximum number of messages to be fetched
block: If True, the API will block till some messages are fetched.
timeout: If block is True, the function will block for the specified
time (in seconds) until count messages is fetched. If None,
it will block forever.
"""
messages = []
# Give a size hint to the consumers. Each consumer process will fetch
# a maximum of "count" messages. This will fetch more messages than
# necessary, but these will not be committed to kafka. Also, the extra
# messages can be provided in subsequent runs
self.size.value = count
self.pause.clear()
if timeout is not None:
max_time = time.time() + timeout
new_offsets = {}
while count > 0 and (timeout is None or timeout > 0):
# Trigger consumption only if the queue is empty
# By doing this, we will ensure that consumers do not
# go into overdrive and keep consuming thousands of
# messages when the user might need only a few
if self.queue.empty():
self.start.set()
try:
partition, message = self.queue.get(block, timeout)
except Empty:
break
messages.append(message)
new_offsets[partition] = message.offset + 1
count -= 1
if timeout is not None:
timeout = max_time - time.time()
self.size.value = 0
self.start.clear()
self.pause.set()
# Update and commit offsets if necessary
self.offsets.update(new_offsets)
self.count_since_commit += len(messages)
self._auto_commit()
return messages
from multiprocessing import Event
#生成事件对象
e = Event()
#检测事件对象，如果被设置则返回True否则返回false
print(e.is_set())
#设置事件对象
e.set()
#提供事件的阻塞
e.wait()
print("wait.....")
#清除对事件的设置
e.clear()
e.wait()
print("wait...wait.....")
class Que(Thread):
def __init__(self, name, frequency, que):
super(Que, self).__init__()
self.__commands = dict()
self.running = False
self.__frequency = frequency
self.__que = que
self.__ready = False
self.__paused = True
self.pauseEvent = Event()
self.resumeEvent = Event()
self.__readyEvent = Event()
self.daemon = True
self.name = name
self.deleteAfterPoll = False
logger.debug('Que {} declared'.format(self.name))
def run(self):
self.running = True
logger.info('Que {} starting'.format(self.name))
while self.running:
try:
if not self.__ready:
logger.debug('Not Ready on thread {}'.format(self.name))
self.__readyEvent.wait()
self.__readyEvent.clear()
logger.debug('Ready on thread {}'.format(self.name))
logger.debug('Que {} paused state = {}'.format(
self.name, self.__paused))
if not self.__paused:
for s in self.__commands:
lastPolled = time()
logger.debug(
'Que {} adding command {} to output queue'.format(
self.name, s))
self.__que.put(s)
if self.deleteAfterPoll:
self.removeCommand(s)
break
time_elapsed = time() - lastPolled
sleeptime = (1.0 / self.__frequency /
len(self.__commands)) - time_elapsed
sleep(sleeptime)
else:
logger.debug('Pausing thread {}'.format(self.name))
self.pauseEvent.set()
self.resumeEvent.wait()
self.resumeEvent.clear()
logger.debug('Resuming thread {}'.format(self.name))
except (KeyboardInterrupt, SystemExit):
self.running = False
continue
except (RuntimeError):
continue
except:
logger.critical('Exception caught in Queue Thread {}:'.format(
self.name),
exc_info=True,
stack_info=True)
continue
logger.info('Que {} stopped'.format(self.name))
def setFrequency(self, frequency):
self.__frequency = frequency
def addCommand(self, command, override=False):
logger.info('Appending command {} to que {}'.format(
command, self.name))
pausedState = self.__paused
if not self.__paused:
self.__paused = True # Pause the que when making changes
if self.__ready:
logger.debug('Waiting for main loop to stop on que {}'.format(
self.name))
self.pauseEvent.wait(
) # Wait for pauseWait event to ensure dict will not be accessed
self.pauseEvent.clear()
logger.debug('Main loop stopped on que {}'.format(self.name))
self.__commands[command] = override
if not self.__ready:
self.__readyEvent.set()
self.__ready = True
if not pausedState:
self.__paused = False # Resume que after update
self.resumeEvent.set()
logger.debug('Resuming main loop on que {}'.format(self.name))
def getCommands(self):
l = []
for c in self.__commands:
l.append(c)
return l
def removeCommand(self, command):
logger.debug('Removing sensor {} from que {}'.format(
command, self.name))
pausedState = self.__paused
if not self.__paused:
self.__paused = True
if self.__ready:
logger.debug('Waiting for main loop to stop on que {}'.format(
self.name))
self.pauseEvent.wait()
self.pauseEvent.clear()
logger.debug('Main loop stopped on que {}'.format(self.name))
if command in self.__commands:
del self.__commands[command]
if len(self.__commands) == 0:
self.__ready = False
logger.info('Que {} not ready due to zero length'.format(
self.name))
if not pausedState:
self.__paused = False
self.resumeEvent.set()
logger.debug('Resuming main loop on que {}'.format(self.name))
def status(self):
#returns a dict of que status
d = dict()
d['Name'] = self.name
d['Frequency'] = self.__frequency
d['Running'] = self.running
d['Ready'] = self.__ready
d['Paused'] = self.__paused
d['Length'] = len(self.__commands)
return d
def pause(self):
self.__paused = True
def resume(self):
self.__paused = False
self.resumeEvent.set()
class UDPConnectionProcess(Process):
"""UDPConnectionProcess polls and writes to a data queue.
Example::
# Server that prints each input and echos it to the client
# that is currently connected
from udp_connection import UDPConnectionProcess, Queue
receive_queue = Queue()
udp_p = UDPConnectionProcess(receive_queue=receive_queue)
udp_p.start()
udp_p.event_polling.set() # start polling
while True:
data = receive_queue.get()
print(data)
if data is not None:
udp_p.send_queue.put(data.string)
Example::
# connecting to a server
""" # DOC
def __init__(self, event_trigger=(), event_ignore_tag=None):
"""Initialize UDPConnectionProcess
Parameters
----------
receive_queue: multiprocessing.Queue
the queue to which the received data should be put
peer_ip : string
the IP of the peer to which the connection should be established
sync_clock : Clock
the internal clock for timestamps will synchronized with this clock
event_trigger: multiprocessing.Event() (or list of..)
event trigger(s) to be set. If Udp event is received and it is not a
command to set this event (typical of sensor recording processes).
event_ignore_tag:
udp data that start with this tag will be ignored for event triggering
""" # DOC
super(UDPConnectionProcess, self).__init__()
self.receive_queue = Queue()
self.send_queue = Queue()
self.event_is_connected = Event()
self._event_quit_request = Event()
self._event_is_polling = Event()
self._event_ignore_tag = event_ignore_tag
if isinstance(event_trigger, type(Event)):
event_trigger = (event_trigger)
try:
self._event_trigger = tuple(event_trigger)
except:
self._event_trigger = ()
atexit.register(self.quit)
@property
def my_ip(self):
return UDPConnection.MY_IP
def quit(self):
self._event_quit_request.set()
if self.is_alive():
self.join()
def pause(self):
self._event_is_polling.clear()
def start_polling(self):
self._event_is_polling.set()
def run(self):
udp_connection = UDPConnection(udp_port=5005)
self.start_polling()
ptp = PollingTimeProfile()
prev_event_polling = None
while not self._event_quit_request.is_set():
if prev_event_polling != self._event_is_polling.is_set():
# event pooling changed
prev_event_polling = self._event_is_polling.is_set()
if prev_event_polling:
logging.warning("UDP start, pid {}, priority {}".format(
self.pid, get_priority(self.pid)))
else:
logging.warning("UDP stop")
ptp.stop()
if not self._event_is_polling.is_set():
self._event_is_polling.wait(timeout=0.1)
else:
data = udp_connection.poll()
t = udp_connection.timer.time
ptp.update(t)
if data is not None:
d = UDPData(string=data, time=t)
self.receive_queue.put(d)
if self._event_ignore_tag is not None and \
not d.startswith(self._event_ignore_tag):
for ev in self._event_trigger:
# set all connected trigger
ev.set()
try:
udp_connection.send(self.send_queue.get_nowait())
except:
pass
# has connection changed?
if self.event_is_connected.is_set(
) != udp_connection.is_connected:
if udp_connection.is_connected:
self.event_is_connected.set()
else:
self.event_is_connected.clear()
if not udp_connection.is_connected:
udp_connection.timer.wait(200)
udp_connection.unconnect_peer()
logging.warning("UDP quit, {}".format(ptp.get_profile_str()))
class JobExecutor2:
"""
Execute jobs on multiple processes.
At a high level, we have worker executing on multiple processes. Each worker will be fed
by an input queue and results of the processing pushed to an output queue.
Pushing data on a queue is very fast BUT retrieving it from a different process takes time.
Even if PyTorch claims to have memory shared arrays, retrieving a large array from a queue
has a linear runtime complexity. To limit this copy penalty, we can use threads that copy
from the worker process to the main process (`pinning` threads. Here, sharing data between
threads is almost free).
Notes:
This class was designed for maximum speed and not reproducibility in mind.
The processed of jobs will not keep their ordering.
"""
def __init__(self,
nb_workers: int,
function_to_run: Callable[[Batch], Batch],
max_queue_size_per_worker: int = 2,
max_queue_size_pin_thread_per_worker: int = 3,
nb_pin_threads: Optional[int] = None,
wait_time: float = 0.02,
wait_until_processes_start: bool = True):
"""
Args:
nb_workers: the number of workers (processes) to execute the jobs
function_to_run: the job to be executed
max_queue_size_per_worker: define the maximum number of job results that can be stored
before a process is blocked (i.e., larger queue size will improve performance but will
require more memory to store the results). the pin_thread need to process the result before
the blocked process can continue its processing.
max_queue_size_pin_thread_per_worker: define the maximum number of results available on the main
process (i.e., larger queue size will improve performance but will require more memory
to store the results).
nb_pin_threads: the number of threads dedicated to collect the jobs processed by different processes.
Data copy from the worker process to main process takes time, in particular for large data. It
is advantageous to have multiple threads that copy these results to the main process. If `None`,
nb_workers // 2 threads will be used
wait_time: the default wait time for a process or thread to sleep if no job is available
wait_until_processes_start: if True, the main process will wait until the worker processes and
pin threads are fully running
"""
print(f'JobExecutor2 started on process={os.getpid()}')
self.wait_until_processes_start = wait_until_processes_start
self.wait_time = wait_time
self.max_queue_size_pin_thread_per_worker = max_queue_size_pin_thread_per_worker
self.max_queue_size_per_worker = max_queue_size_per_worker
self.function_to_run = function_to_run
self.nb_workers = nb_workers
self.abort_event = Event()
self.main_process = os.getpid()
if nb_pin_threads is None:
self.nb_pin_threads = max(1, nb_workers // 2)
else:
self.nb_pin_threads = nb_pin_threads
assert self.nb_pin_threads >= 1, 'must have at least one thread to collect the processed jobs!'
self.worker_control = 0
self.worker_input_queues = []
self.worker_output_queues = []
self.processes = []
self.jobs_processed = Value('i', 0)
self.jobs_queued = 0
self.pin_memory_threads = []
self.pin_memory_queue = None
# we can't cancel jobs, so instead record a session ID. If session of
# the worker and current session ID do not match
# it means the results of these tasks should be discarded
self.job_session_id = Value('i', 0)
self.start()
def start(self, timeout: float = 10.0) -> None:
"""
Start the processes and queues.
Args:
timeout:
Returns:
"""
if self.pin_memory_queue is None:
self.pin_memory_queue = ThreadQueue(
self.max_queue_size_pin_thread_per_worker * self.nb_workers)
if self.nb_workers == 0:
# nothing to do, this will be executed synchronously on
# the main process
return
if len(self.processes) != self.nb_workers:
logging.debug(
f'Starting jobExecutor={self}, on process={os.getpid()} nb_workers={self.nb_workers}'
)
self.close()
self.abort_event.clear()
with threadpool_limits(limits=1, user_api='blas'):
for i in range(self.nb_workers): #maxsize = 0
self.worker_input_queues.append(
Queue(maxsize=self.max_queue_size_per_worker))
self.worker_output_queues.append(
Queue(self.max_queue_size_per_worker))
p = Process(target=worker,
name=f'JobExecutorWorker-{i}',
args=(self.worker_input_queues[i],
self.worker_output_queues[i],
self.function_to_run, self.abort_event,
self.wait_time, i))
p.daemon = True
p.start()
self.processes.append(p)
logging.debug(
f'Child process={p.pid} for jobExecutor={self}')
self.pin_memory_threads = []
for i in range(self.nb_pin_threads):
pin_memory_thread = threading.Thread(
name=f'JobExecutorThreadResultCollector-{i}',
target=collect_results_to_main_process,
args=(self.job_session_id, self.jobs_processed,
self.worker_output_queues, self.pin_memory_queue,
self.abort_event, self.wait_time))
self.pin_memory_threads.append(pin_memory_thread)
pin_memory_thread.daemon = True
pin_memory_thread.start()
self.worker_control = 0
if self.wait_until_processes_start:
# wait until all the processes and threads are alive
waiting_started = perf_counter()
while True:
wait_more = False
for p in self.processes:
if not p.is_alive():
wait_more = True
continue
for t in self.pin_memory_threads:
if not t.is_alive():
wait_more = True
continue
if wait_more:
waiting_time = perf_counter() - waiting_started
if waiting_time < timeout:
sleep(self.wait_time)
else:
logging.error(
'the worker processes/pin threads were too slow to start!'
)
break
logging.debug(f'jobExecutor ready={self}')
def close(self, timeout: float = 10) -> None:
"""
Stops the processes and threads.
Args:
timeout: time allowed for the threads and processes to shutdown cleanly
before using `terminate()`
"""
# notify all the threads and processes to be shut down
print('Setting `abort_event` to interrupt Processes and threads!')
self.abort_event.set()
if os.getpid() != self.main_process:
logging.error(
f'attempting to close the executor from a '
f'process={os.getpid()} that did not create it! ({self.main_process})'
)
return
# give some time to the threads/processes to shutdown normally
shutdown_time_start = perf_counter()
while True:
wait_more = False
if len(self.processes) != 0:
for p in self.processes:
if p.is_alive():
wait_more = True
break
if len(self.pin_memory_threads):
for t in self.pin_memory_threads:
if t.is_alive():
wait_more = True
break
shutdown_time = perf_counter() - shutdown_time_start
if wait_more:
if shutdown_time < timeout:
sleep(0.1)
continue
else:
logging.error(
'a job did not respond to the shutdown request in the allotted time. '
'It could be that it needs a longer timeout or a deadlock. The processes'
'will now be forced to shutdown!')
# done normal shutdown or timeout
break
if len(self.processes) != 0:
#logging.debug(f'JobExecutor={self}: shutting down workers...')
[i.terminate() for i in self.processes]
for i, p in enumerate(self.processes):
self.worker_input_queues[i].close()
self.worker_input_queues[i].join_thread()
self.worker_output_queues[i].close()
self.worker_output_queues[i].join_thread()
self.worker_input_queues = []
self.worker_output_queues = []
self.processes = []
if len(self.pin_memory_threads) > 0:
for thread in self.pin_memory_threads:
thread.join()
del thread
self.pin_memory_threads = []
del self.pin_memory_queue
self.pin_memory_queue = None
def is_full(self) -> bool:
"""
Check if the worker input queues are full.
Returns:
True if full, False otherwise
"""
if self.nb_workers == 0:
return False
for i in range(self.nb_workers):
queue = self.worker_input_queues[self.worker_control]
if not queue.full():
return False
self.worker_control = (self.worker_control + 1) % self.nb_workers
return True
def put(self, data: Batch) -> bool:
"""
Queue a batch of data to be processed.
Warnings:
if the queues are full, the batch will NOT be appended
Args:
data: a batch of data to process
Returns:
True if the batch was successfully appended, False otherwise.
"""
if self.nb_workers == 0:
batch_in = copy.deepcopy(data)
batch_out = self.function_to_run(batch_in)
self.pin_memory_queue.put(batch_out)
self.jobs_queued += 1
return True
else:
for i in range(self.nb_workers):
queue = self.worker_input_queues[self.worker_control]
if not queue.full():
queue.put((self.job_session_id.value, data))
self.worker_control = (self.worker_control +
1) % self.nb_workers
self.jobs_queued += 1
return True
# all queues are full, we have to wait
return False
def is_idle(self) -> bool:
"""
Returns:
True if the executor is not currently processing jobs
"""
with self.jobs_processed.get_lock():
return self.jobs_processed.value == self.jobs_queued
def reset(self):
"""
Reset the input and output queues as well as job session IDs.
The results of the jobs that have not yet been calculated will be discarded
"""
# here we could clear the queues for a faster implementation.
# Unfortunately, this is not an easy task to properly
# counts all the jobs processed or discarded due to the
# multi-threading. Instead, all tasks queued are executed
# and we use a `job_session_id` to figure out the jobs to be
# discarded
"""
# empty the various queues
try:
for input_queue in self.worker_input_queues:
while not input_queue.empty():
input_queue.get()
except EOFError: # in case the other process was already terminated
pass
try:
for output_queue in self.worker_output_queues:
while not output_queue.empty():
output_queue.get()
except EOFError: # in case the other process was already terminated
pass
with self.jobs_processed.get_lock():
self.jobs_processed.value = 0
self.jobs_queued = 0
"""
# empty the current queue results, they are not valid anymore!
try:
while not self.pin_memory_queue.empty():
self.pin_memory_queue.get()
except EOFError: # in case the other process was already terminated
pass
# discard the results of the jobs that will not have the
# current `job_session_id`
with self.job_session_id.get_lock():
self.job_session_id.value += 1
def __del__(self):
#logging.debug(f'JobExecutor={self}: destructor called')
self.close()
class AmpDecorator(Amplifier):
"""This class 'decorates' the Low-Level Amplifier classes with
Network-Marker and Save-To-File functionality.
You use it by decorating (not as in Python-Decorator, but in the GoF
sense) the low level amplifier class you want to use::
import libmushu
from libmushu.ampdecorator import AmpDecorator
from libmushu.driver.randomamp import RandomAmp
amp = Ampdecorator(RandomAmp)
Waring: The network marker timings on Windows have a resolution of
10ms-15ms. On Linux the resolution is 1us. This is due to
limitations of Python's time.time method, or rather a Windows
specific issue.
There exists currently no precise timer, providing times which are
comparable between two running processes on Windows. The performance
counter provided on Windows, has a much better resolution but is
relative to the processes start time and it drifts (1s per 100s), so
it is only precise for a relatively short amount of time.
If a higher precision is needed one has to replace the time.time
calls with something which provides a better precision. For example
one could create a third process which provides times or regularly
synchronize both processes with the clock synchronization algorithm
as described here:
http://en.wikipedia.org/wiki/Network_Time_Protocol
Alternatively one could use `timeGetTime` from Windows' Multi Media
library, which is tunable via `timeBeginPeriod` and provides a
precision of 1-2ms. Apparently this is the way Chrome and many
others do it.::
from __future__ import division
from ctypes import windll
import time
timeBeginPeriod = windll.winmm.timeBeginPeriod
timeEndPeriod = windll.winmm.timeEndPeriod
timeGetTime = windll.winmm.timeGetTime
if __name__ == '__main__':
# wrap the code that needs high precision in timeBegin- and
# timeEndPeriod with the same parameter. The parameter is
# the interval in ms you want as precision. Usually the
# minimum value allowed is 1 (best).
timeBeginPeriod(1)
times = []
t_start = time.time()
while time.time() < (time.time() + 1):
times.append(timeGetTime())
times = sorted(list(set(times)))
print(1000 / len(times))
timeEndPeriod(1)
"""
def __init__(self, ampcls):
self.amp = ampcls()
self.write_to_file = False
@property
def presets(self):
return self.amp.presets
def start(self, filename=None):
# prepare files for writing
self.write_to_file = False
if filename is not None:
self.write_to_file = True
filename_marker = filename + '.marker'
filename_eeg = filename + '.eeg'
filename_meta = filename + '.meta'
for filename in filename_marker, filename_eeg, filename_meta:
if os.path.exists(filename):
logger.error('A file "%s" already exists, aborting.' %
filename)
raise Exception
self.fh_eeg = open(filename_eeg, 'wb')
self.fh_marker = open(filename_marker, 'w')
self.fh_meta = open(filename_meta, 'w')
# write meta data
meta = {
'Channels': self.amp.get_channels(),
'Sampling Frequency': self.amp.get_sampling_frequency(),
'Amp': str(self.amp)
}
json.dump(meta, self.fh_meta, indent=4)
# start the marker server
self.marker_queue = Queue()
self.tcp_reader_running = Event()
self.tcp_reader_running.set()
tcp_reader_ready = Event()
self.tcp_reader = Process(target=marker_reader,
args=(self.marker_queue,
self.tcp_reader_running,
tcp_reader_ready))
self.tcp_reader.start()
logger.debug('Waiting for marker server to become ready...')
tcp_reader_ready.wait()
logger.debug('Marker server is ready.')
# zero the sample counter
self.received_samples = 0
# start the amp
self.amp.start()
def stop(self):
# stop the amp
self.amp.stop()
# stop the marker server
self.tcp_reader_running.clear()
logger.debug('Waiting for marker server process to stop...')
self.tcp_reader.join()
logger.debug('Marker server process stopped.')
# close the files
if self.write_to_file:
logger.debug('Closing files.')
for fh in self.fh_eeg, self.fh_marker, self.fh_meta:
fh.close()
def configure(self, **kwargs):
self.amp.configure(**kwargs)
def get_data(self):
"""Get data from the amplifier.
This method is supposed to get called as fast as possible (i.e
hundreds of times per seconds) and returns the data and the
markers.
Returns
-------
data : 2darray
a numpy array (time, channels) of the EEG data
markers : list of (float, str)
a list of markers. Each element is a tuple of timestamp and
string. The timestamp is the time in ms relative to the
onset of the block of data. Note that negative values are
*allowed* as well as values bigger than the length of the
block of data returned. That is to be interpreted as a
marker from the last block and a marker for a future block
respectively.
"""
# get data and marker from underlying amp
data, marker = self.amp.get_data()
t = time.time()
# length in sec of the new block according to #samples and fs
block_duration = len(data) / self.amp.get_sampling_frequency()
# abs time of start of the block
t0 = t - block_duration
# duration of all blocks in ms except the current one
duration = 1000 * self.received_samples / self.amp.get_sampling_frequency(
)
# merge markers
tcp_marker = []
while not self.marker_queue.empty():
m = self.marker_queue.get()
m[0] = (m[0] - t0) * 1000
tcp_marker.append(m)
marker = sorted(marker + tcp_marker)
# save data to files
if self.write_to_file:
for m in marker:
self.fh_marker.write("%f %s\n" % (duration + m[0], m[1]))
self.fh_eeg.write(struct.pack("f" * data.size, *data.flatten()))
self.received_samples += len(data)
if len(data) == 0 and len(marker) > 0:
logger.error(
'Received marker but no data. This is an error, the amp should block on get_data until data is available. Marker timestamps will be unreliable.'
)
return data, marker
def get_channels(self):
return self.amp.get_channels()
def get_sampling_frequency(self):
return self.amp.get_sampling_frequency()
class EyeTracker:
"""Generic class that is intended to act as a parent class for other
implementations. The general concept of processing images to find eyes
in them is the same whether the input is a webcam stream, a video, or a
series of images.
"""
def __init__(self, logfile=u'default', facedetect=True, eyedetect=True, \
pupthreshold=50, glintthreshold=200, glintdetect=True, \
pupsizemode=u'diameter', minfacesize=(30,30), Lexpect=(0.7,0.4), \
Rexpect=(0.3,0.4), maxpupdist=0.2, maxpupsize=0.3, maxcpu=6, \
**kwargs):
"""Initialises an EyeTracker class.
Keyword Arguments
logfile - A string that indicates the path to the log
file. An extension will be added
automatically. Default = 'default'.
facedetect - A Boolean that indicates whether face
detection should be attempted before further
processing (eye detection, and pupil/glint
detection). Set this to False if you will
be using the EyeTracker from close to an
eye, in which cases faces need and could not
be detected. Default = True.
pupthreshold - An integer that indicates what the highest
luminance value is that is still considered
to be part of the pupil. This value needs to
be between 0 and 255. Default = 50.
glintthreshold - An integer that indicates what the lowest
luminance value is that is still considered
to be part of the glint. This value needs to
be between 0 and 255. Default = 200.
glintdetect - A Boolean that indicates whether the glint
(the corneal reflection) should also be
detected. Default = True.
pupsizemode - A string that indicates how the pupil size
should be reported.
'diameter' reports the width of the rect in
which the thresholded pupil fits.
'surface' reports the number of thresholded
pixels that are assumed to be the pupil.
minfacesize - A (w,h) tuple that indicates what size a
face should minimally be. Default = (30,30)
Lexpect - A (x,y) tuple that indicates where the left
eye is expected to be. Note that the
coordinates are in relative space, where
(0,0) is the top-left of the image, (0,1)
is the bottom-left, and (1,1) is the
bottom-right. Also note that the left eye is
likely to be on the right side of the image,
and the right eye is likely to be in the
left part of the image. Default = (0.7,0.4)
Rexpect - A (x,y) tuple that indicates where the right
eye is expected to be. Note that the
coordinates are in relative space, where
(0,0) is the top-left of the image, (0,1)
is the bottom-left, and (1,1) is the
bottom-right. Also note that the left eye is
likely to be on the right side of the image,
and the right eye is likely to be in the
left part of the image. Default = (0.3,0.4)
maxpupdist - A float that indicates what the maximal
allowable distance is between the expected
eye position, and the position of detected
potential eye. The maximal distance is
defined as a proportion of the image height.
It can also be set to None. Default = (0.2)
maxpupsize - A float that indicates what the maximal
allowable width is of the detected eyes. The
maximal size is defined as a proportion of
the image width. It can also be set to None.
Default = (0.3)
maxcpu - Integer indicating the maximum amount of
parallel processes that will be doing all
of the image processing. This happens in
parallel to speed things up; the processing
time on one CPU can't keep up with the
camera's sampling rate. Default = 6.
"""
# DEBUG message.
_message(u'debug', u'generic.EyeTracker.__init__', \
u"Initialising a new EyeTracker.")
# GENERAL SETTINGS
# Face detection yes/no, and from what size.
self._facedetect = facedetect
self._minfacesize = minfacesize
# Face eye yes/no.
self._eyedetect = eyedetect
# Eye detection settings. These are relative positions of where
# each eye is expected to be in a frame, how far away detected eyes
# are allowed to be from the expected locations, and how large the
# detected eyes are allowed to be. (All defined as proportions of
# the frame's width and height.)
self._Lexpect = Lexpect
self._Rexpect = Rexpect
self._maxpupdist = maxpupdist
self._maxpupsize = maxpupsize
# Pupil detection thresholds (dark for pupil, bright for glint),
# and additional options that determine whether glints should be
# detected, and how the pupil size should be reported.
self._pupt = pupthreshold
self._glit = glintthreshold
self._glintdetect = glintdetect
self._pupsizemode = pupsizemode
# ALIVE EVENT
# This event signals whether the tracker is still alive. It should
# only be cleared when closing the connection to the tracker!
self._alive = Event()
self._alive.set()
# FRAME OBTAINING THREAD
# Boolean that turns to True when a connection with the source of
# frames has been established.
self._connected = False
# We need a Queue for frames that are generated in the obtainer
# Thread. The Queue is read out by the parallel processes.
self._framequeue = Queue()
# We need a lock to prevent potential simultaneous attempts to
# access the image source at the same time. This shouldn't actually
# be possible in the current implementation, but may be added in
# the future.
self._sourcelock = Lock()
# Start the frame obtaining Thread
_message(u'debug', u'generic.EyeTracker.__init__', \
u"Starting a Thread to obtain frames.")
self._frame_obtainer_thread = Thread(target=self._frame_obtainer, \
args=[self._alive, self._framequeue])
self._frame_obtainer_thread.name = u'frame_obtainer'
self._frame_obtainer_thread.daemon = True
self._frame_obtainer_thread.start()
# PARALLEL PROCESSING
# We need a Queue for samples that are generated in the parallel
# processes that are simultaneously processing new frames.
self._samplequeue = Queue()
# Check how many CPUs we can use.
cpus = cpu_count()
if cpus > maxcpu:
cpus = maxcpu
# Start parallel processes to do image processing.
_message(u'debug', u'generic.EyeTracker.__init__', \
u"Starting %d parallel processes to process frames into samples." \
% (cpus-1))
self._frame_processes = []
for i in range(1, cpus):
p = Process(target=_frame_processer, \
args=[self._alive, self._framequeue, self._samplequeue, \
self._pupt, self._glit, self._facedetect, self._eyedetect, \
self._minfacesize, self._Lexpect, self._Rexpect, \
self._maxpupdist, self._maxpupsize, self._glintdetect, \
self._pupsizemode])
p.name = u'frame_processor_%d' % (i)
p.daemon = True
p.start()
self._frame_processes.append(p)
# SAMPLE WRITING
# Variable that keeps track of the latest sample.
self._latest_sample = [0, numpy.zeros((2, 5)) * numpy.NaN]
# Boolean that signals whether the recording Thread should be
# active or not.
self._recording = False
# Lock to prevent simultaneous access to the log file.
self._loglock = Lock()
# The log file is an open text file. It will be opened when
# self._start_recording is called, and it will be closed when
# self._stop_recording is called. Between calling those two
# functions, samples will be appended to the log. To prevent
# samples from being appended to an existing log file, here we
# open a new logfile with in 'w' mode, thereby erasing any existing
# content of a previous log file. This means users need to be
# careful when naming their files, to prevent overwriting.
self._logfilename = u'%s.tsv' % (logfile)
_message(u'debug', u'generic.EyeTracker.__init__', \
u"Creating new logfile '%s'." \
% (self._logfilename))
# Create a header for the log file.
l = [u'time']
l.extend([u'Lpx', u'Lpy', u'Lps', u'Lgx', u'Lgy'])
l.extend([u'Rpx', u'Rpy', u'Rps', u'Rgx', u'Rgy'])
line = u'\t'.join(map(unicode, l)) + u'\n'
# Create a new log file.
self._loglock.acquire(True)
self._logfile = open(self._logfilename, u'w')
self._logfile.write(line)
self._logfile.close()
self._loglock.release()
# Start the sample logging Thread
_message(u'debug', u'generic.EyeTracker.__init__', \
u"Starting a Thread to log samples to file '%s'." \
% (self._logfilename))
self._sample_logging_thread = Thread(target=self._sample_logger, \
args=[self._alive, self._samplequeue])
self._sample_logging_thread.name = u'sample_logger'
self._sample_logging_thread.daemon = True
self._sample_logging_thread.start()
# CUSTOM INITIALISATION
# Run the custom initialisation procedure.
self.connect(**kwargs)
def close(self, **kwargs):
"""Use this function to implement the ending of a specific type
of eye tracking.
"""
# Only close the connection EyeTracker if it isn't closed yet.
if self._connected:
# Signal the _frame_obtainer that the tracker isn't connected
# anymore.
self._connected = False
# Signal all Threads and Processes that they should stop.
self._alive.clear()
# Wait for all frame-processing Processes to join.
for p in self._frame_processes:
p.join()
# Wait for the frame-obtaining Thread to join.
self._frame_obtainer_thread.join()
# Wait for the logging Thread to join.
self._sample_logging_thread.join()
# Call the custom closing implementation.
self._close(**kwargs)
def connect(self, **kwargs):
"""Use this function to implement the initialisation of a specific
type of eye tracking.
"""
# Only intialise the EyeTracker if it isn't initialised yet.
if not self._connected:
# Signal the frame obtainer that we are connected!
self._connected = True
# CUSTOM IMPLEMENTATION HERE
def is_connected(self):
"""Tells you whether the EyeTracker is still connected with its
source of images.
Returns
connected - A Boolean that indicates whether the tracker
is still connected.
"""
return self._connected
def sample(self):
"""Returns the newest sample.
Returns
t, [L, R] - t is a timestamp associated with the latest
sample.
[L, R] is a NumPy array that contains the
following:
L = numpy.array([px, py, ps, gx, gy])
R = numpy.array([px, py, ps, gx, gy])
px is an integer indicating the pupil's
likely horizontal position within the image.
py is an integer indicating the pupil's
likely vertical position within the image.
ps is an integer indicating the pupil's
size in pixels. The size reflects the width
of the rect in which the pupil falls when
mode=='diameter', or the total amount of
thresholded pixels when mode'surface'.
gx is an integer indicating the pupil's
likely horizontal position within the image.
gy is an integer indicating the pupil's
likely vertical position within the image.
All of these values can be None if they
could not be obtained.
"""
# Return the latest sample.
# TODO: Potential issue here is that this sample is at least five
# samples old. Maybe we could peek ahead in the Queue, and actually
# get the latest?
return self._latest_sample
def log(self, msg):
"""Logs a message to the log file.
Arguments
msg - A string that should be added to the log
file.
"""
# Make sure that the message is in a unicode format.
msg = msg.decode(u'utf-8')
# A message is a time-stamped sample, with a slightly different
# content than normal samples. In essence, the log message sample
# is pretending to be a real sample, with a list for the 'left'
# and the 'right' eye. It seems a bit ridiculous, but the nested
# lists are there to prevent the strings from being pulled apart
# when self._log_sample tries to turn them into lists.
sample = [time.time(), [[u'MSG'], [msg]]]
# Put the sample in the Queue.
self._samplequeue.put(sample)
def start_recording(self):
"""Starts the writing of samples to the log file.
"""
# Only start recording if it isn't currently active.
if not self._recording:
_message(u'debug', u'generic.EyeTracker.__init__', \
u"Starting recording, and re-opening logfile '%s'." \
% (self._logfilename))
# Signal the recording thread to start.
self._recording = True
# Re-open the logfile.
self._loglock.acquire(True)
self._logfile = open(self._logfilename, u'a')
self._loglock.release()
def stop_recording(self):
"""Pauses the writing of samples to the log file.
"""
# Only pause recording if recording is currently active.
if self._recording:
# Signal the recording Thread to stop what it's doing.
self._recording = False
# Wait for a bit, to allow the emptying of the local queue.
time.sleep(0.2)
# Close the logfile.
self._loglock.acquire(True)
self._logfile.close()
self._loglock.release()
_message(u'debug', u'generic.EyeTracker.__init__', \
u"Stopped recording, and closed logfile '%s'" \
% (self._logfilename))
def _get_frame(self):
"""Use this function to implement how the EyeTracker should obtain
a frame. A frame is supposed to be a two-dimensional image, usually
an image from a webcam or a video that was converted to greyscale.
Note that a frame should be a numpy.ndarray with shape=(w,h), and
dtype='uint8'. In addition to the frame, a success Boolean should
be returned by this function. It tells the functions that call
_get_frame whether a new frame was available. (See below what the
returned values should be exactly.)
IMPORTANT: This function should not have any keyword arguments.
Any settings should be handled through properties of self.
Returns
success, frame - success is a Boolean that indicates whether
a frame could be obtained.
frame is a numpy.ndarray with unsigned,
8-bit integers that reflect the greyscale
values of the image. If no frame could be
obtained, None will be returned.
"""
# Obtain a frame.
_message(u'message', u'generic.EyeTracker._get_frame', \
u"Implement your own _get_frame functionality")
return False, None
def _close(self, **kwargs):
"""Use this function to implement the specifics of closing a
connection in your eye-tracking implementation. You could, for
example, use it to close the connection to a webcam. This function
is automatically called when the close() method is passed, and this
setup allows you to pass your own keyword arguments to close (which
will then be passed on to _close).
"""
# CUSTOM IMPLEMENTATION HERE
_message(u'message', u'generic.EyeTracker._close', \
u"Implement your own _close functionality")
# # # # # #
# HELPERS #
# # # # # #
def _log_sample(self, sample):
"""Writes a sample to the log file.
Arguments
sample - A (t, [L, R]) tuple/list. Explanation below:
t, [L, R] - t is a timestamp associated with the sample.
[L, R] is a NumPy array that contains the
following:
L = numpy.array([px, py, ps, gx, gy])
R = numpy.array([px, py, ps, gx, gy])
px is an integer indicating the pupil's
likely horizontal position within the image.
py is an integer indicating the pupil's
likely vertical position within the image.
ps is an integer indicating the pupil's
size in pixels. The size reflects the width
of the rect in which the pupil falls when
mode=='diameter', or the total amount of
thresholded pixels when mode'surface'.
gx is an integer indicating the pupil's
likely horizontal position within the image.
gy is an integer indicating the pupil's
likely vertical position within the image.
All of these values can be None if they
could not be obtained.
"""
# Compose a string that can be written to the log file. This
# contains the timestamp in milliseconds (hence *1000), the left
# eye's values, and the right eye's values, all separated by tabs.
l = [int(sample[0] * 1000)]
l.extend(list(sample[1][0]))
l.extend(list(sample[1][1]))
line = u'\t'.join(map(unicode, l)) + u'\n'
# Write to the log file. Acquire the log lock first, and release it
# after writing, to prevent simultaneous writing to the same log.
self._loglock.acquire(True)
self._logfile.write(line)
self._loglock.release()
# # # # # # # # # # #
# RUNNING PROCESSES #
# # # # # # # # # # #
def _frame_obtainer(self, event, framequeue):
"""Continuously tries to get new frames using self._get_frame, and
puts the obtained frames in the framequeue.
"""
# Continuously run.
while event.is_set():
# Only try to get frames if a connection is established.
if self._connected:
# Obtain a new frame, and an associated timestamp.
# Make sure to lock the source while obtaining the
# frame, to prevent simultaneous access. (This
# shouldn't actually happen for the moment, but might
# become a feature in the future.)
self._sourcelock.acquire(True)
success, frame = self._get_frame()
t = time.time()
self._sourcelock.release()
# Put the obtained timestamp and frame in the queue.
framequeue.put([t, frame])
# If the tracker is not connected, wait for a bit to avoid
# wasting processing resources on continuously checking
# whether the tracker is already connected.
else:
# Pause for 10 milliseconds.
time.sleep(0.01)
def _sample_logger(self, event, samplequeue):
"""Continuously monitors the Queue, and writes samples to the log
file whenever over five are still in the Queue.
"""
# Create a list to keep track of samples.
timestamps = []
samplelist = []
# Continuously run.
while event.is_set():
# Only process samples if the tracker is recording.
if self._recording:
# Obtain a new sample if the Queue isn't empty, and lock
# the Queue while we're using it.
if not samplequeue.empty():
# if samplequeue.qsize() > 0:
# Get the oldest sample in the Queue.
sample = samplequeue.get()
# Add the sample to the list.
timestamps.append(sample[0])
samplelist.append(sample[1])
# Store the sample locally, but only if it's not
# a message.
if sample[1][0][0] != 'MSG':
self._latest_sample = sample
# Write the oldest samples from the list, but make sure
# there are always at least five samples still in the
# list. We do this, because the sampling happens
# asynchronously in several parallel processes. This
# might result in newer samples being processed and
# becoming available before older samples. Obviously,
# we want to log the samples in chronological order of
# obtaining them, not of them becoming available. So
# we keep a buffer of five samples, which should
# hopefully be enough to allow slightly older samples
# to come in.
while len(timestamps) > 5:
# Find the oldest timestamp.
i = numpy.argmin(timestamps)
t = timestamps.pop(i)
LR = samplelist.pop(i)
# Log the sample.
self._log_sample([t, LR])
# If we're not recording anymore, but there are samples left
# in the list, then we need to process those first.
elif not self._recording and len(timestamps) > 0:
# Empty out the sample buffer.
while len(timestamps) > 0:
# Find the oldest timestamp.
i = numpy.argmin(timestamps)
t = timestamps.pop(i)
LR = samplelist.pop(i)
# Log the sample.
self._log_sample([t, LR])
# If the tracker is not recording, wait for a bit to avoid
# wasting processing resources on continuously checking
# whether the tracker is recording.
else:
# Pause for 10 milliseconds.
time.sleep(0.01)
class AthSpectralScanner(object):
def __init__(self, interface):
# Set interface, phy, driver and debugfs directory
self.interface = interface
self.phy = None
with open('/sys/class/net/%s/phy80211/name' % interface) as f:
self.phy = f.read().strip()
self.driver = None
self.debugfs_dir = None
for dirname, subd, files in os.walk('/sys/kernel/debug/ieee80211'):
if 'spectral_scan_ctl' in files:
phy = dirname.split(os.path.sep)[-2]
if phy == self.phy:
self.driver = dirname.split(os.path.sep)[-1]
self.debugfs_dir = dirname
with open(
os.path.sep.join(dirname.split(os.path.sep)[:-1]) +
os.path.sep + 'ht40allow_map') as f:
self.ht40allow_map_raw = f.read()
f.close()
break
if self.debugfs_dir is None:
raise Exception(
"Unable to access 'spectral_scan_ctl' file for interface '%s'. "
"Maybe you need to adjust the access rights of /sys/kernel/debug/ieee80211 ?"
% interface) # Fixme: sudo chmod -R 777 /sys/kernel/debug
logger.debug("interface '%s' is on '%s' via %s. debugfs found at %s" %
(self.interface, self.phy, self.driver, self.debugfs_dir))
# hardware capabilities: channels to tune to
self.channels = []
self._get_supported_channels()
logger.debug("interface '%s' supports the channels: %s" %
(self.interface, self.channels))
logger.debug("ht40allow map: %s" %
self.ht40allow_map_raw.replace('\n', ","))
self.ht40allow_map = dict()
for line in self.ht40allow_map_raw.split(
'\n'): # 2412 HT40 + \n2432 HT40 -+ \n2484 Disabled
freq_mode_allowmap = line.split()
if len(freq_mode_allowmap) == 3:
self.ht40allow_map[
freq_mode_allowmap[0]] = freq_mode_allowmap[2]
# chanscan mode triggers on changed channels. Use Process to run "iw scan" to tune to all channels
self.chanscan_process = None
self.chanscan_process_exit = Event()
# Store current state of the config files
self.cfg_files = {}
self.cfg_filenames = ("spectral_count", "spectral_fft_period",
"spectral_period", "spectral_scan_ctl",
"spectral_short_repeat")
self._store_former_config()
self.mode = self.cfg_files['spectral_scan_ctl']['former_value']
self.need_tear_down = True # fixme
self.running = False
self.current_freq = 0 # Fixme: read from interface
self.current_ht_mode = "HT20" # Fixme: read from interface
self.set_channel(1) # Fixme: read from interface
def __del__(self):
#self.stop() # FIXME
pass
# suger for set_mode(mode)
def set_mode_chanscan(self):
self.set_mode("chanscan")
def set_mode_background(self):
self.set_mode("background")
def set_mode_manual(self):
self.set_mode("manual")
def set_mode_disable(self):
self.set_mode("disable")
def set_mode(self, mode, skip_interface_config=False):
if mode not in ["chanscan", "background", "manual", "disable"]:
raise Exception("Unknown mode requested: '%s'" % mode)
if mode is "chanscan" and self.mode is not "chanscan":
self.mode = mode
if not skip_interface_config:
logger.debug(
"enter 'chanscan' mode: set dev type to 'managed'")
os.system("sudo ifconfig %s down" % self.interface)
os.system("sudo iw dev %s set type managed" % self.interface)
os.system(
"sudo ifconfig %s up" %
self.interface) # FIXME: does the interface need to be up?
self._set_spectral_cfg('spectral_scan_ctl', "chanscan")
#self._start_scan_process() -> start()
self.need_tear_down = True
return
self._stop_scan_process(
) # all other modes: kill external "iw scan" (if any)
if mode is "background" and self.mode is not "background":
self.mode = mode
if not skip_interface_config:
logger.debug(
"enter 'background' mode: set dev type to 'monitor'")
os.system("sudo ifconfig %s down" % self.interface)
os.system("sudo iw dev %s set monitor fcsfail" % self.interface
) # fcsfail = also report frames with corrupt FCS
os.system("sudo ifconfig %s up" %
self.interface) # need to be up
self._set_spectral_cfg('spectral_scan_ctl', "background")
#self._set_spectral_cfg('spectral_scan_ctl', "trigger") -> start()
self.need_tear_down = True
return
if mode is "manual" and self.mode is not "manual":
self.mode = mode
if not skip_interface_config:
logger.debug("enter 'manual' mode: set dev type to 'monitor'")
os.system("sudo ifconfig %s down" % self.interface)
os.system("sudo iw dev %s set monitor fcsfail" % self.interface
) # fcsfail = also report frames with corrupt FCS
os.system("sudo ifconfig %s up" %
self.interface) # need to be up
self._set_spectral_cfg('spectral_scan_ctl', "manual")
self.need_tear_down = True
return
if mode is "disable" and self.mode is not "disable":
self.mode = mode
if not skip_interface_config:
os.system("sudo ifconfig %s down" % self.interface)
os.system("sudo iw dev %s set type managed" % self.interface)
self._set_spectral_cfg('spectral_scan_ctl', "disable")
# need to trigger() here? ? -> not needed. ath9k_cmn_spectral_scan_config() calls
# ath9k_hw_ops(ah)->spectral_scan_config() which unset the AR_PHY_SPECTRAL_SCAN_ENABLE flag if needed
self.need_tear_down = True
return
def trigger(self):
self._set_spectral_cfg('spectral_scan_ctl', "trigger")
def retrigger(self):
if self.mode == "background":
self._set_spectral_cfg('spectral_scan_ctl', "trigger")
def get_mode(self):
return self.mode
def get_debugfs_directory(self):
return self.debugfs_dir
def get_data_filename(self):
return self.debugfs_dir + os.path.sep + "spectral_scan0"
def get_config(self):
cfg = {}
for fn in self.cfg_filenames:
path = self.debugfs_dir + os.path.sep + fn
with open(path, 'r') as f:
cfg[fn] = f.read().strip()
f.close()
cfg['driver'] = self.driver
cfg['frequency'] = self.current_freq
return cfg
def set_channel(self, channel):
self._tune(channel=channel)
def set_frequency(self, frequency):
self._tune(frequency=frequency)
def get_supported_freqchan(self):
return self.channels
def set_HT_mode(self, ht_mode):
if ht_mode == "HT20":
self.current_ht_mode = ht_mode
elif "HT40" in ht_mode:
self._set_ht40_mode_for_freq(self.current_freq)
else:
raise Exception(
"unknown value for HT mode: '%s'. valid: HT20, HT40" % count)
self.set_channel(self.current_chan) # set new HT mode
def _set_ht40_mode_for_freq(self, freq):
if not str(freq) in self.ht40allow_map.keys():
self.current_ht_mode = "HT20"
return
if "+" in self.ht40allow_map[str(freq)]:
self.current_ht_mode = "HT40+"
else:
self.current_ht_mode = "HT40-"
# Source of min/max values for parameters: ath9k/spectral-common.c
def set_spectral_count(self, count):
if count > 255 or count < 0:
logger.error(
"invalid value for 'spectral_count' of %d. valid: 0-255" %
count)
return
self._set_spectral_cfg('spectral_count', count)
def get_spectral_count(self):
return int(self._get_spectral_cfg('spectral_count'))
def set_spectral_fft_period(self, period):
if period > 15 or period < 0:
logger.error(
"invalid value for 'spectral_fft_period' of %d. valid: 0-15" %
period)
return
self._set_spectral_cfg('spectral_fft_period', period)
def get_spectral_fft_period(self):
return int(self._get_spectral_cfg('spectral_fft_period'))
def set_spectral_period(self, period):
if period > 255 or period < 0:
logger.error(
"invalid value for 'spectral_period' of %d. valid: 0-255" %
period)
return
self._set_spectral_cfg('spectral_period', period)
def get_spectral_period(self):
return int(self._get_spectral_cfg('spectral_period'))
def set_spectral_short_repeat(self, repeat):
if repeat > 1 or repeat < 0:
logger.error(
"invalid value for 'spectral_short_repeat' of %d. valid: 0-1" %
repeat)
return
self._set_spectral_cfg('spectral_short_repeat', repeat)
def get_spectral_short_repeat(self):
return int(self._get_spectral_cfg('spectral_short_repeat'))
def _set_spectral_cfg(self, filenname, value):
logger.debug("set '%s' to '%s'" % (filenname, value))
with open(self.cfg_files[filenname]['path'], 'w') as f:
f.write("%s" % value)
f.close()
def _get_spectral_cfg(self, filenname):
with open(self.cfg_files[filenname]['path']) as f:
cfg = f.read()
f.close()
return cfg
def start(self):
self.running = True
if self.mode is "chanscan":
self._start_scan_process()
else:
self._set_spectral_cfg('spectral_scan_ctl', "trigger")
def stop(self):
self.running = False
self.set_mode("disable")
if self.need_tear_down:
self._restore_former_config()
self.need_tear_down = False
self._stop_scan_process()
def _tune(self, channel=None, frequency=None):
if channel is None and frequency is None:
raise Exception("need channel or frequency")
if self.mode is "chanscan":
logger.warning(
"Manual set of channel/frequency gets probably overwritten in chanscan mode."
)
for i in range(0, len(self.channels)):
(freq, chan) = self.channels[i]
if chan == channel or freq == frequency:
self.current_freq = freq
self.current_chan = chan
if self.current_ht_mode != "HT20":
self._set_ht40_mode_for_freq(freq)
logger.debug("set freq to %d in mode %s" %
(freq, self.current_ht_mode))
os.system("sudo iw dev %s set freq %d %s" %
(self.interface, freq, self.current_ht_mode))
if self.running:
self._set_spectral_cfg(
'spectral_scan_ctl', "trigger"
) # need to trigger again after switch channel
return
logger.warning(
"can not tune to unsupported channel %d / frequency %d. "
"Supported channels: %s" % channel, frequency, self.channels)
# FIXME: add interface config, use with open
def _store_former_config(self):
for fn in self.cfg_filenames:
path = self.debugfs_dir + os.path.sep + fn
f = open(path, 'r')
val = f.read().strip()
f.close()
self.cfg_files[fn] = {'path': path, 'former_value': val}
logger.debug("read config for '%s': '%s'" % (path, val))
def _restore_former_config(self):
#for fn, stored_cfg in self.cfg_files.iteritems(): python2 # FIXME remove
for fn, stored_cfg in self.cfg_files.items():
val = stored_cfg['former_value']
path = stored_cfg['path']
f = open(path, 'w')
f.write(val)
f.close()
logger.debug("restore '%s' to: '%s'" % (path, val))
def _get_supported_channels(self):
# parses the supported channels as (channel, freq) from 'iw phy'
iw_phy_output = subprocess.check_output(["iw", "phy"]).decode('UTF-8')
found_device = False
for line in iw_phy_output.split('\n'):
line = line.strip()
if "Wiphy" in line:
if self.phy in line:
found_device = True
else:
found_device = False
continue
if found_device:
if "*" in line and "MHz" in line and "[" in line and "]" in line:
try:
freq = int(line.split("MHz")[0].split("*")[1].strip())
chan = int(line.split("[")[1].split("]")[0].strip())
self.channels.append((freq, chan))
except Exception as e:
raise Exception(
"Cant parse freq/channel from line '%s': '%s'" %
(line, e))
def _start_scan_process(self):
if self.chanscan_process is None:
logger.debug("start process chanscan")
self.chanscan_process_exit.clear()
self.chanscan_process = Process(target=self._scan, args=())
self.chanscan_process.start()
def _stop_scan_process(self):
if self.chanscan_process is not None:
logger.debug("stop process chanscan")
#self.chanscan_process.terminate() # dont work
self.chanscan_process_exit.set()
self.chanscan_process.join()
self.chanscan_process = None
def _scan(self):
while not self.chanscan_process_exit.is_set():
cmd = 'iw dev %s scan' % self.interface
os.system('%s >/dev/null 2>/dev/null' % cmd) # call blocks
class TestRestFirmware(TestAcceptanceBase):
def setUp(self):
super().setUp()
self.analysis_finished_event = Event()
self.elements_finished_analyzing = Value('i', 0)
self.db_backend_service = BackEndDbInterface(config=self.config)
self._start_backend(post_analysis=self._analysis_callback)
self.test_container_uid = '418a54d78550e8584291c96e5d6168133621f352bfc1d43cf84e81187fef4962_787'
time.sleep(2) # wait for systems to start
def tearDown(self):
self._stop_backend()
self.db_backend_service.shutdown()
super().tearDown()
def _analysis_callback(self, fo):
self.db_backend_service.add_analysis(fo)
self.elements_finished_analyzing.value += 1
if self.elements_finished_analyzing.value == 4 * 3: # container including 3 files times 3 plugins
self.analysis_finished_event.set()
def _rest_upload_firmware(self):
data = get_firmware_for_rest_upload_test()
rv = self.test_client.put('/rest/firmware',
data=json.dumps(data),
follow_redirects=True)
self.assertIn(b'"status": 0', rv.data, 'rest upload not successful')
self.assertIn(self.test_container_uid.encode(), rv.data,
'uid not found in rest upload reply')
def _rest_get_analysis_result(self):
rv = self.test_client.get('/rest/firmware/{}'.format(
self.test_container_uid),
follow_redirects=True)
self.assertIn(b'analysis_date', rv.data,
'rest analysis download not successful')
self.assertIn(b'software_components', rv.data,
'rest analysis not successful')
self.assertIn(b'"device_part": "test_part"', rv.data,
'device part not present')
def _rest_search(self):
rv = self.test_client.get('/rest/firmware?query={}'.format(
urllib.parse.quote('{"device_class": "test_class"}')),
follow_redirects=True)
self.assertIn(self.test_container_uid.encode(), rv.data,
'test firmware not found in rest search')
def _rest_search_fw_only(self):
query = json.dumps({'sha256': self.test_container_uid.split('_')[0]})
rv = self.test_client.get('/rest/firmware?query={}'.format(
urllib.parse.quote(query)),
follow_redirects=True)
self.assertIn(self.test_container_uid.encode(), rv.data,
'test firmware not found in rest search')
def _rest_update_analysis_bad_analysis(self):
rv = self.test_client.put('/rest/firmware/{}?update={}'.format(
self.test_container_uid, urllib.parse.quote('["unknown_system"]')),
follow_redirects=True)
self.assertIn(
'Unknown analysis system'.encode(), rv.data,
"rest analysis update should break on request of non existing system"
)
def _rest_update_analysis_success(self):
rv = self.test_client.put('/rest/firmware/{}?update={}'.format(
self.test_container_uid,
urllib.parse.quote(json.dumps(['crypto_material']))),
follow_redirects=True)
self.assertNotIn(b'error_message', rv.data, 'Error on update request')
def _rest_check_new_analysis_exists(self):
rv = self.test_client.get('/rest/firmware/{}'.format(
self.test_container_uid),
follow_redirects=True)
response_data = json.loads(rv.data.decode())
assert response_data['firmware']['analysis']['crypto_material']
assert response_data['firmware']['analysis']['crypto_material'][
'analysis_date'] > response_data['firmware']['analysis'][
'software_components']['analysis_date']
def test_run_from_upload_to_show_analysis_and_search(self):
self._rest_upload_firmware()
self.analysis_finished_event.wait(timeout=15)
self.elements_finished_analyzing.value = 4 * 2 # only one plugin to update so we offset with 4 times 2 plugins
self.analysis_finished_event.clear()
self._rest_get_analysis_result()
self._rest_search()
self._rest_search_fw_only()
self._rest_update_analysis_bad_analysis()
self._rest_update_analysis_success()
self.analysis_finished_event.wait(timeout=10)
self._rest_check_new_analysis_exists()
def main():
args = parse_args()
timeout = args.timeout
## start crawler
# crawler_noticer = Event()
# crawler_noticer.clear()
# result_noticer = Event()
# result_noticer.clear()
# qreader, qwriter = Pipe()
# stdreader, stdwriter = Pipe()
# crawler = multiprocessing.Process(
# target=crawler_daemon,
# args=(crawler_noticer, qreader, result_noticer, stdwriter)
# )
# crawler.daemon = True
# crawler.start()
adb_bin = get_adb_tool()
if use_monitor:
os.system("{0} connect 127.0.0.1:62001".format(adb_bin))
check_screenshot(filename="screenshot.png", directory=data_directory)
if enable_chrome:
closer = Event()
noticer = Event()
closer.clear()
noticer.clear()
reader, writer = Pipe()
browser_daemon = multiprocessing.Process(
target=run_browser, args=(closer, noticer, reader,))
browser_daemon.daemon = True
browser_daemon.start()
def __inner_job():
start = time.time()
text_binary = analyze_current_screen_text(
directory=data_directory,
compress_level=image_compress_level[0],
crop_area=crop_areas[game_type],
use_monitor=use_monitor
)
keywords = get_text_from_image(
image_data=text_binary,
)
if not keywords:
print("text not recognize")
return
true_flag, real_question, question, answers = parse_question_and_answer(
keywords)
## notice crawler to work
# qwriter.send(real_question.strip("?"))
# crawler_noticer.set()
print('-' * 72)
print(real_question)
print('-' * 72)
print("\n".join(answers))
# notice browser
if enable_chrome:
writer.send(question)
noticer.set()
search_question = pre_process_question(question)
summary = baidu_count(search_question, answers, timeout=timeout)
summary_li = sorted(
summary.items(), key=operator.itemgetter(1), reverse=True)
data = [("选项", "同比")]
for a, w in summary_li:
data.append((a, w))
table = AsciiTable(data)
print(table.table)
print("*" * 72)
if true_flag:
print("肯定回答(**)： ", summary_li[0][0])
print("否定回答( )： ", summary_li[-1][0])
else:
print("肯定回答( )： ", summary_li[0][0])
print("否定回答(**)： ", summary_li[-1][0])
print("*" * 72)
# try crawler
# retry = 4
# while retry:
# if result_noticer.is_set():
# print("~" * 60)
# print(stdreader.recv())
# print("~" * 60)
# break
# retry -= 1
# time.sleep(1)
# result_noticer.clear()
print("~" * 60)
print(kwquery(real_question.strip("?")))
print("~" * 60)
end = time.time()
print("use {0} 秒".format(end - start))
save_screen(
directory=data_directory
)
print("""
请选择答题节目:
1. 百万英雄
2. 冲顶大会
3. 芝士超人
""")
game_type = input("输入节目序号: ")
if game_type == "1":
game_type = '百万英雄'
elif game_type == "2":
game_type = '冲顶大会'
elif game_type == "3":
game_type = "芝士超人"
else:
game_type = '百万英雄'
while True:
print("""
请在答题开始前就运行程序，
答题开始的时候按Enter预测答案
""")
print("当前选择答题游戏: {}\n".format(game_type))
enter = input("按Enter键开始，按ESC键退出...")
if enter == chr(27):
break
try:
__inner_job()
except Exception as e:
print(str(e))
print("欢迎下次使用")
if enable_chrome:
reader.close()
writer.close()
closer.set()
time.sleep(3)
class IWebCamStreamProcessing:
"""
Webcam stream processing interface.
Takes images from webcam, process it via NDL API and visualize results
:param service: NDL API service
"""
def __init__(self, service):
self.service = service
# Assign streaming function to service. This function have to yield images and will be called from NDL API core
self.service.set_streaming_function(self.iterate_webcam_images,
'image')
# Create images and result sources for communication between several processes
self.images_queue, self.result_queue = Queue(), Queue()
# Create event for stopping the system
self.stop_event = Event()
self.stop_event.clear()
def iterate_webcam_images(self):
"""
Iterates over images stream.
Takes images from input source (i.e. queue) and send it to NDL API
Raises Empty if there are no images to send for at least 5 seconds
"""
try:
# Do until somebody stop event
while not self.stop_event.is_set():
# Get image from input source
image = self.images_queue.get(timeout=5)
# Yield image for NDL API
yield image
except Empty:
print('Webcam timeout exceeded. Aborting.')
raise
except:
print('Exception in iterating webcam images. Aborting')
raise
finally:
print('Webcam iterating was successfully stopped')
def iterate_api_responses(self):
"""
Iterates over NDL API responses
Takes response from NDL API, translates it to the correct format and puts to the target source (i.e. queue)
Raises BaseException in case of postprocessing error
"""
try:
# Do until service stop processing
for response in self.service.process_stream():
# Check response status
if response[1] is None:
print(response[2])
continue
# Get result and translate it to the correct format
result = {
i: json.loads(image_res.result)
for i, image_res in enumerate(response[1])
}
processed_result = self.service._postprocess_result(result)
# Put result to target source
self.result_queue.put(processed_result)
# Check system is not stopped
if self.stop_event.is_set():
break
except:
print("Exception while iterating API response")
raise
finally:
self.stop_event.set()
def start_streaming(self):
"""
Start webcam streaming.
Open webcam stream and process images from it
Creates additional process to iterate over NDL API responses
:return:
"""
# Create and start parallel process for iterating over NDL API response
response_iterating_process = Process(target=self.iterate_api_responses)
response_iterating_process.daemon = True
response_iterating_process.start()
# Last processed result
last_result, wait_for_result = None, False
# Open webcam stream
cap = cv2.VideoCapture(0)
# Get first frame from webcam stream
ret, frame = cap.read()
# Check that webcam works normally and nobody stopped the system
while ret and not self.stop_event.is_set():
# Check if last putted images already processed
if self.images_queue.empty() and not wait_for_result:
self.images_queue.put(frame)
wait_for_result = True
# Check if there is new processing result
if not self.result_queue.empty():
last_result = self.result_queue.get()
wait_for_result = False
# Visualize result on frame
vis_frame = self.visualize_result(frame, last_result)
# Show frame
cv2.imshow('webcam', vis_frame)
# Wait 50ms for any key pressed. Stops system if 'q' was pressed.
if cv2.waitKey(50) & 0xff == ord('q'):
break
# Read next frame
ret, frame = cap.read()
# Stop system
self.stop_event.set()
# Release webcam
cap.release()
print('Webcam capture released')
def visualize_result(self, image, res=None):
"""
Visualizes data on image
:param image: image from webcam stream
:param res: result from NDL API
:return: image with visualized data
"""
raise NotImplementedError
class Core(Process):
__SESSIONFILENAME = 'session.ini'
__CONFIGFILENAME = 'config.ini'
DEFAULT_LANGUAGE = 'english'
DEFAULT_USERNAME = '******'
DEFAULT_PASSWORD = '******'
DEFAULT_STORAGEDIRNAME = 'downloads'
DEFAULT_LOGDIRNAME = 'logs'
DEFAULT_LOGFILENAME = 'log.txt'
def _init_consolelogger(self):
if False and self.config.get('log',
'color_console') and ismodule('colorlog'):
fmt = "%(label)s %(levelname)-8s %(reset)s %(log_color)s%(asctime)s %(message)s"
datefmt = "%Y-%m-%d %H:%M:%S"
primary_colors = {
'DEBUG': "bold,cyan",
'WARNING': "bold,yellow",
'ERROR': "bold,red",
'CRITICAL': "bold,purple",
}
secondary_colors = {
'label': {
'DEBUG': "bold,white,bg_cyan",
'INFO': "bold,white,bg_green",
'WARNING': "bold,white,bg_yellow",
'ERROR': "bold,white,bg_red",
'CRITICAL': "bold,white,bg_purple",
}
}
consoleform = colorlog.ColoredFormatter(
fmt,
datefmt,
log_colors=primary_colors,
secondary_log_colors=secondary_colors)
else:
fmt = "%(asctime)s %(levelname)-8s %(message)s"
datefmt = "%Y-%m-%d %H:%M:%S"
consoleform = logging.Formatter(fmt, datefmt)
consolehdlr = logging.StreamHandler(sys.stdout)
consolehdlr.setFormatter(consoleform)
self.log.addHandler(consolehdlr)
def _init_syslogger(self):
# try to mimic to normal syslog messages
fmt = "%(asctime)s %(name)s: %(message)s"
datefmt = "%b %e %H:%M:%S"
syslogform = logging.Formatter(fmt, datefmt)
syslogaddr = None
syslog = self.config.get('log', 'syslog')
if syslog == 'remote':
syslog_host = self.config.get('log', 'syslog_host')
syslog_port = self.config.get('log', 'syslog_port')
syslogaddr = (syslog_host, syslog_port)
else:
syslog_folder = self.config.get('log', 'syslog_folder')
if syslogaddr:
syslogaddr = syslog_folder
elif sys.platform == 'darwin':
syslogaddr = '/var/run/syslog'
elif os.name != 'nt':
syslogaddr = '/dev/log'
sysloghdlr = logging.handlers.SysLogHandler(syslogaddr)
sysloghdlr.setFormatter(syslogform)
self.log.addHandler(sysloghdlr)
def _init_filelogger(self):
fmt = "%(asctime)s %(levelname)-8s %(message)s"
datefmt = "%Y-%m-%d %H:%M:%S"
fileform = logging.Formatter(fmt, datefmt)
logfile_folder = self.config.get('log', 'logfile_folder')
if not logfile_folder:
logfile_folder = self.DEFAULT_LOGDIRNAME
makedirs(logfile_folder, exist_ok=True)
logfile_name = self.config.get('log', 'logfile_name')
if not logfile_name:
logfile_name = self.DEFAULT_LOGFILENAME
logfile = os.path.join(logfile_folder, logfile_name)
if self.config.get('log', 'rotate'):
logfile_size = self.config.get('log', 'logfile_size') << 10
max_logfiles = self.config.get('log', 'max_logfiles')
filehdlr = logging.handlers.RotatingFileHandler(
logfile,
maxBytes=logfile_size,
backupCount=max_logfiles,
encoding=locale.getpreferredencoding(do_setlocale=False))
else:
filehdlr = logging.FileHandler(
logfile,
encoding=locale.getpreferredencoding(do_setlocale=False))
filehdlr.setFormatter(fileform)
self.log.addHandler(filehdlr)
# TODO: Extend `logging.Logger` like `..plugin.Log`
def _init_logger(self):
level = logging.DEBUG if self.debug else logging.INFO
# Init logger
self.log = logging.getLogger()
self.log.setLevel(level)
# Set console handler
self._init_consolelogger()
# Set syslog handler
if self.config.get('log', 'syslog') != 'no':
self._init_syslogger()
# Set file handler
if self.config.get('log', 'logfile'):
self._init_filelogger()
def _setup_permissions(self):
if os.name == 'nt':
return None
change_group = self.config.get('permission', 'change_group')
change_user = self.config.get('permission', 'change_user')
if change_group:
try:
group = self.config.get('permission', 'group')
set_process_group(group)
except Exception as e:
self.log.error(self._("Unable to change gid: %s"), str(e))
if change_user:
try:
user = self.config.get('permission', 'user')
set_process_user(user)
except Exception as e:
self.log.error(self._("Unable to change uid: %s"), str(e))
def set_language(self, lang):
localedir = resource_filename(__package__, 'locale')
lc = locale.locale_alias[lang.lower()].split('_', 1)[0]
trans = get_translation('core', localedir, (lc, ))
try:
self._ = trans.ugettext
except AttributeError:
self._ = trans.gettext
def _setup_language(self):
self.log.debug("Loading language ...")
lang = self.config.get('general', 'language')
default = self.DEFAULT_LANGUAGE
if not lang:
code = locale.getlocale()[0] or locale.getdefaultlocale()[0]
lang = default if code is None else code.lower().split('_', 1)[0]
try:
self.set_language(lang)
except Exception as e:
if lang == default:
raise
self.log.warning(
self._(
"Unable to load `%(lang)s` language, using default "
"`%(default)s`: %(error)s", ),
{
'lang': lang,
'default': default,
'error': str(e)
},
)
self.set_language(default)
def _setup_debug(self):
if self.__debug is None:
debug_log = self.config.get('log', 'debug')
verbose_log = self.config.get('log', 'verbose')
self.__debug = 0 if not debug_log else 2 if verbose_log else 1
def _start_interface(self, webui=None, rpc=None):
# if webui is None:
# webui = self.__webui
# if rpc is None:
# rpc = self.__rpc
# TODO: Parse `remote`
# if rpc or self.config.get('rpc', 'activated'):
# self.log.debug("Activating RPC interface ...")
# self.rem.start()
# elif not webui:
if not webui:
webui = True
# TODO: Parse remote host:port
if isinstance(webui, str):
host, port = map(str.strip, webui.rsplit(':', 1))
webui = True
else:
host, port = (None, None)
kwgs = {
'server': self.config.get('webui', 'server'),
'host': host or self.config.get('webui', 'host'),
'port': port or self.config.get('webui', 'port'),
'key': self.config.get('ssl', 'key'),
'cert': self.config.get('ssl', 'cert'),
'ssl': self.config.get('ssl', 'activated')
}
if webui or self.config.get('webui', 'activated'):
from pyload.webui.webserver import WebServer
# from .thread.webserver import WebServer
self.webserver = WebServer(self)
self.webserver.start()
time.sleep(1000)
# self.svm.add('webui', **kwgs)
# self.svm.start()
def _init_api(self):
from .api import Api
self.api = Api(self)
def _init_database(self):
from .database import DatabaseBackend
from .datatype import Permission, Role
# TODO: Move inside DatabaseBackend
newdb = not os.path.isfile(DatabaseBackend.DB_FILE)
self.db = DatabaseBackend(self)
self.db.setup()
if self.__restore or newdb:
self.db.add_user(self.DEFAULT_USERNAME, self.DEFAULT_PASSWORD,
Role.Admin, Permission.All)
if self.__restore:
self.log.warning(
self._("Restored default login credentials `admin|pyload`"))
def _init_managers(self):
from .manager import (AccountManager, AddonManager, EventManager,
ExchangeManager, FileManager, InfoManager,
PluginManager, TransferManager, RemoteManager)
self.scheduler = sched.scheduler(time.time, time.sleep)
self.filemanager = self.files = FileManager(self)
self.pluginmanager = self.pgm = PluginManager(self)
self.exchangemanager = self.exm = ExchangeManager(self)
self.eventmanager = self.evm = EventManager(self)
self.accountmanager = self.acm = AccountManager(self)
self.infomanager = self.iom = InfoManager(self)
self.transfermanager = self.tsm = TransferManager(self)
self.addonmanager = self.adm = AddonManager(self)
self.remotemanager = self.rem = RemoteManager(self)
# self.servermanager = self.svm = ServerManager(self)
self.db.manager = self.files # ugly?
def _init_requests(self):
from .network.factory import RequestFactory
self.request = self.req = RequestFactory(self)
def _init_config(self):
session = ConfigParser(self.__SESSIONFILENAME, session_defaults)
flags = portalocker.LOCK_EX | portalocker.LOCK_NB
#with session.fp:
portalocker.lock(session.fp, flags)
profiledir = os.path.join(self.configdir, self.profile)
psp = psutil.Process()
session.set('current', 'id', time.time())
session.set('current', 'profile', 'path', profiledir)
session.set('current', 'profile', 'pid', psp.pid)
session.set('current', 'profile', 'ctime', psp.create_time())
self.config = ConfigParser(self.__CONFIGFILENAME, config_defaults)
self.session = session
def _init_cache(self):
# Re-use cache
tempdir = self.__tempdir
if tempdir is None:
tempdir = self.session.get('previous', 'cache', 'path')
if tempdir is None or not os.path.isdir(tempdir):
pydir = os.path.join(TMPDIR, __namespace__)
makedirs(pydir, exist_ok=True)
tempdir = tempfile.mkdtemp(dir=pydir)
self.session.set('current', 'cache', 'path', tempdir)
self.cachedir = tempdir
# if tempdir not in sys.path:
# sys.path.append(tempdir)
def _register_signals(self):
shutfn = lambda s, f: self.shutdown()
quitfn = lambda s, f: self.terminate()
try:
if os.name == 'nt':
# signal.signal(signal.CTRL_C_EVENT, shutfn)
signal.signal(signal.CTRL_BREAK_EVENT, shutfn)
else:
signal.signal(signal.SIGTERM, shutfn)
# signal.signal(signal.SIGINT, shutfn)
signal.signal(signal.SIGQUIT, quitfn)
# signal.signal(signal.SIGTSTP, lambda s, f: self.stop())
# signal.signal(signal.SIGCONT, lambda s, f: self.run())
except Exception:
pass
def __init__(self,
profiledir=None,
tempdir=None,
debug=None,
restore=None):
self.__running = Event()
self.__do_restart = False
self.__do_shutdown = False
self.__debug = debug if debug is None else int(debug)
self.__restore = bool(restore)
self.__tempdir = tempdir
self._ = lambda x: x
self._init_profile(profiledir)
# if refresh:
# cleanpy(PACKDIR)
Process.__init__(self)
@property
def version(self):
return __version__
@property
def version_info(self):
return __version_info__
@property
def running(self):
return self.__running.is_set()
@property
def debug(self):
return self.__debug
def _init_profile(self, profiledir):
profiledir = fullpath(profiledir)
os.chdir(profiledir)
self.configdir, self.profile = os.path.split(profiledir)
def _setup_process(self):
try:
set_process_name('pyLoad')
except AttributeError:
pass
niceness = self.config.get('general', 'niceness')
renice(niceness=niceness)
ioniceness = int(self.config.get('general', 'ioniceness'))
ionice(niceness=ioniceness)
def _setup_storage(self):
storage_folder = self.config.get('general', 'storage_folder')
if not storage_folder:
storage_folder = os.path.join(USERDIR, self.DEFAULT_STORAGEDIRNAME)
self.config.set('general', 'storage_folder', storage_folder)
self.log.debug("Storage: {0}".format(storage_folder))
makedirs(storage_folder, exist_ok=True)
avail_space = format.size(availspace(storage_folder))
self.log.info(
self._("Available storage space: {0}").format(avail_space))
def _workloop(self):
self.__running.set()
self.tsm.pause = False # NOTE: Recheck...
while True:
time.sleep(1) # temporary fix, hopefully
self.__running.wait()
self.tsm.work()
self.iom.work()
self.exm.work()
if self.__do_restart:
raise Restart
if self.__do_shutdown:
raise Shutdown
self.scheduler.run()
def _start_plugins(self):
# TODO: Move to accountmanager
self.log.info(self._("Activating accounts ..."))
# self.acm.get_account_infos()
# self.scheduler.enter(0, 0, self.acm.get_account_infos)
self.adm.activate_addons()
def _show_info(self):
self.log.info(self._("Welcome to pyLoad v{0}").format(self.version))
self.log.info(self._("Profile: {0}").format(self.profile))
self.log.info(self._("Config directory: {0}").format(self.configdir))
self.log.debug("Cache directory: {0}".format(self.cachedir))
def run(self):
self._init_config()
self._init_cache()
self._setup_debug()
self._init_logger()
try:
self.log.debug("Running pyLoad ...")
self._setup_language()
self._setup_permissions()
self._init_database()
self._init_managers()
self._init_requests()
self._init_api()
self._show_info()
self._setup_storage()
self._start_plugins()
self._setup_process()
self._start_interface()
self.log.info(self._("pyLoad is up and running"))
self.evm.fire('pyload:started')
# # some memory stats
# from guppy import hpy
# hp=hpy()
# print(hp.heap())
# import objgraph
# objgraph.show_most_common_types(limit=30)
# import memdebug
# memdebug.start(8002)
# from meliae import scanner
# scanner.dump_all_objects(os.path.join(PACKDIR, 'objs.json'))
self._workloop()
except Restart:
self.restart()
except Shutdown:
self.shutdown()
except (KeyboardInterrupt, SystemExit):
self.shutdown()
except Exception as e:
self.log.critical(str(e))
self.terminate()
raise
else:
self.shutdown()
def _remove_loggers(self):
for handler in self.log.handlers:
with closing(handler) as hdlr:
self.log.removeHandler(hdlr)
def restart(self):
self.stop()
self.log.info(self._("Restarting pyLoad ..."))
self.evm.fire('pyload:restarting')
self.start()
def _register_instance(self):
profiledir = os.path.join(self.configdir, self.profile)
if profiledir in _pmap:
raise RuntimeError("A pyLoad instance using profile `{0}` "
"is already running".format(profiledir))
_pmap[profiledir] = self
def _unregister_instance(self):
profiledir = os.path.join(self.configdir, self.profile)
_pmap.pop(profiledir, None)
def _close_session(self):
id = str(
self.session.get('previous', 'id')
) # TODO: added str(), is previous still needed at all? maybe delete this and the next line
self.session[id] = self.session['previous']
self.session['previous'] = self.session['current']
self.session['current'].reset()
self.session.close()
def terminate(self):
try:
self.log.debug("Killing pyLoad ...")
self._unregister_instance()
self._close_session()
finally:
# TODO: remove try catch; used to remove debug output as _popen is None
try:
Process.terminate(self)
except Exception:
pass
def shutdown(self):
try:
self.stop()
self.log.info(self._("Exiting pyLoad ..."))
self.tsm.shutdown()
self.db.shutdown() # NOTE: Why here?
self.config.close()
self._remove_loggers()
# if cleanup:
# self.log.info(self._("Deleting temp files ..."))
# remove(self.tempdir, ignore_errors=True)
finally:
self.terminate()
def start(self):
if not self.is_alive():
self._register_instance()
self._register_signals()
Process.start(self)
elif not self.running:
self.log.info(self._("Starting pyLoad ..."))
self.evm.fire('pyload:starting')
self.__running.set()
def stop(self):
if not self.running:
return None
try:
self.log.info(self._("Stopping pyLoad ..."))
self.evm.fire('pyload:stopping')
self.adm.deactivate_addons()
self.api.stop_all_downloads()
finally:
self.files.sync_save()
self.__running.clear()
self.evm.fire('pyload:stopped')
q.put(w)
if __name__ == "__main__":
buffer_size = int(1e+6)
env_dict = {"obs": {"shape": 4},
"act": {},
"rew": {},
"next_obs": {"shape": 4},
"done": {}}
n_explorer = 4
global_rb = MPPrioritizedReplayBuffer(buffer_size,env_dict)
is_training_done = Event()
is_training_done.clear()
qs = [SimpleQueue() for _ in range(n_explorer)]
ps = [Process(target=explorer,
args=[global_rb,env_dict,is_training_done,q])
for q in qs]
for p in ps:
p.start()
learner(global_rb,qs)
is_training_done.set()
for p in ps:
p.join()
class ExposureMonitoring(Process):
def __init__(self):
super().__init__()
self.pool = futures.ThreadPoolExecutor(max_workers=1)
self.process = None
self.exit = Event()
self.running = Event()
self.ics_last_exposure = {}
self.ics = get_qlf_interface()
self.process_id = Value('i', 0)
def run(self):
""" """
while not self.exit.is_set():
time.sleep(1.5)
if not self.process or not self.process.running():
self.running.clear()
last_exposure = self.ics.last_exposure()
try:
exposure = last_exposure['exposure']
fibermap = last_exposure['fibermap']
except:
logger.debug('No exposure available')
continue
fibermap = last_exposure.get('fibermap', None)
ics_last_expid = self.ics_last_exposure.get('exposure_id', None)
if exposure.get('exposure_id') == ics_last_expid:
logger.debug('Exposure {} has already been processed'.format(
ics_last_expid))
continue
self.ics_last_exposure = exposure
logger.debug('Exposure {} obtained'.format(
exposure.get('exposure_id')))
# records exposure in database
exposure_obj = QLFModels().insert_exposure(**exposure)
if exposure_obj:
fibermap['exposure'] = exposure_obj
QLFModels().insert_fibermap(**fibermap)
if self.process and self.process.running():
logger.debug('Process {} is running.'.format(
str(self.process_id.value)))
continue
if isinstance(exposure.get('time'), str):
exposure['time'] = datetime.datetime.strptime(
exposure.get('time'), "%Y-%m-%dT%H:%M:%S.%f")
delay = datetime.datetime.utcnow() - exposure.get('time')
delay = delay.total_seconds()
if delay > allowed_delay:
logger.debug(
('The delay in the acquisition of the exposure '
'went from {} seconds'.format(str(allowed_delay))))
continue
logger.info('Exposure {} ({} {}) available.'.format(
exposure.get('exposure_id'),
exposure.get('program').capitalize(), exposure.get('flavor')))
del self.process
gc.collect()
self.process = self.pool.submit(run_process, exposure,
self.process_id)
self.process.add_done_callback(future_callback)
self.running.set()
logger.debug("Bye!")
def shutdown(self):
""" Turn off monitoring """
self.exit.set()
self.running.clear()
self.pool.shutdown()
del self.process
gc.collect()
self.process = None
class H5FileWatcher:
def __init__(self, h5_filepath, row_handler, contraints_dict=None):
self.handle_event = Event()
self.h5_filepath = h5_filepath
self.handle_count = 0
self.job_frames = {}
self.last_handle_count = None
self.row_handler = row_handler
self.contraints_clause = '' if contraints_dict is None else ' '.join(
'and {}={}'.format(k, v) for k, v in contraints_dict.items())
Log.d('cc: {}', self.contraints_clause)
assert row_handler
def handle_change(self):
self.handle_count += 1
Log.d('file change event {}', self.handle_count)
self.handle_event.set()
def ensure_strictly_increasing_index(self, df):
p = None
for i, r in df.iterrows():
assert p is None or p < i, '{} < {}'.format(p, i)
p = i
def process_h5(self):
with pd.HDFStore(self.h5_filepath, mode='r') as h5:
for jobuid in h5:
is_first_encounter = jobuid not in self.job_frames
if is_first_encounter == True:
self.job_frames[jobuid] = pd.read_hdf(
h5, jobuid, start=0,
stop=1) # fetch first row to get the first index/epoch
job_df = self.job_frames[jobuid]
latest_epoch = job_df.index.values[
-1] # will ensure we don't 'miss' any rows in case the handle count jumps more than once
where_clause = 'index > {} {}'.format(latest_epoch,
self.contraints_clause)
new_df = pd.read_hdf(h5, jobuid, where=where_clause)
if new_df.empty:
Log.w('dataset was empty for key {} and index > {}',
jobuid, latest_epoch)
else:
assert new_df.index.values[0] > latest_epoch
new_first_index = 0 if is_first_encounter == True else len(
job_df)
joined = pd.concat([job_df, new_df])
self.job_frames[jobuid] = joined
if len(joined) > 100000:
Log.w(
'holding a dataset of significant length ({} rows, {:.1f}mb): {}',
len(joined),
joined.memory_usage().sum() / 1_000_000, jobuid)
assert joined.shape[0] == len(job_df) + len(new_df)
self.ensure_strictly_increasing_index(
joined) # TODO: remove once this is confirmed
self.row_handler(jobuid, joined, new_first_index)
def trigger(self):
self.handle_event.set()
def __run(self):
Log.d('Watching file: {}', self.h5_filepath)
thread = FileWatcher(self.h5_filepath,
modified=self.handle_change).run_async()
try:
while self.handle_event.wait():
if self.last_handle_count is not None:
jump_count = self.handle_count - self.last_handle_count
if jump_count > 1:
Log.w(
'handle count has jumped {} times than once since the last processing',
jump_count)
self.last_handle_count = self.handle_count
self.process_h5()
self.handle_event.clear()
finally:
Log.w('run loop broken, unwatching file: {}', self.h5_filepath)
thread.stop()
thread.join()
def run_async(self):
thread = Thread(target=self.__run)
thread.start()
return thread
class SpectrumIdentificationWorkerBase(Process, SpectrumEvaluatorBase):
verbose = False
def __init__(self, input_queue, output_queue, producer_done_event,
consumer_done_event, scorer_type, evaluation_args,
spectrum_map, mass_shift_map, log_handler, solution_packer):
Process.__init__(self)
if evaluation_args is None:
evaluation_args = dict()
self.daemon = True
self.input_queue = input_queue
self.output_queue = output_queue
self.producer_done_event = producer_done_event
self.consumer_done_event = consumer_done_event
self.scorer_type = scorer_type
self.evaluation_args = evaluation_args
self.solution_packer = solution_packer
self.spectrum_map = spectrum_map
self.mass_shift_map = mass_shift_map
self.local_scan_map = LRUMapping(1000)
self.local_mass_shift_map = dict({Unmodified.name: Unmodified})
self.solution_map = dict()
self._work_complete = Event()
self._work_complete.clear()
self.log_handler = log_handler
self.token = uid()
self.items_handled = 0
self.result_buffer = []
self.buffer_size = 1000
self.last_sent_result = time.time()
def log(self, message):
"""Send a normal logging message via :attr:`log_handler`
Parameters
----------
message : str
The message to log
"""
if self.log_handler is not None:
self.log_handler(message)
def debug(self, message):
"""Send a debugging message via :attr:`log_handler`
Parameters
----------
message : str
The message to log
"""
if self.verbose:
self.log_handler("DEBUG::%s" % message)
def fetch_scan(self, key):
try:
return self.local_scan_map[key]
except KeyError:
serialized_scan = self.spectrum_map[key]
scan = pickle.loads(serialized_scan)
self.local_scan_map[key] = scan
return scan
def fetch_mass_shift(self, key):
try:
return self.local_mass_shift_map[key]
except KeyError:
mass_shift = self.mass_shift_map[key]
self.local_mass_shift_map[key] = mass_shift
return mass_shift
def all_work_done(self):
"""A helper method to encapsulate :attr:`_work_complete`'s ``is_set``
method.
Returns
-------
bool
"""
return self._work_complete.is_set()
def _append_to_result_buffer(self, payload):
self.result_buffer.append(payload)
if len(self.result_buffer) > self.buffer_size:
self._flush_result_buffer()
def _flush_result_buffer(self):
if self.result_buffer:
self.output_queue.put(self.result_buffer)
self.result_buffer = []
self.last_sent_result = time.time()
def pack_output(self, target):
"""Transmit the completed identifications for a given target
structure.
Rather than returning these values directly, this implementation uses
the output queue to send the target and its scores along an IPC queue.
Parameters
----------
target : object
The structure being identified.
"""
if self.solution_map:
target_id = getattr(target, 'id', target)
self._append_to_result_buffer(
(target_id, self.solution_map, self.token))
self.solution_map = dict()
def evaluate(self,
scan,
structure,
evaluation_context=None,
*args,
**kwargs):
raise NotImplementedError()
def cleanup(self):
"""Send signals indicating the worker process is finished and do any
final shared resource cleanup needed on the worker's side.
This will set the :attr:`_work_complete` event and join :attr:`output_queue`
"""
self.debug(
"... Process %s Setting Work Complete Flag. Processed %d structures"
% (self.name, self.items_handled))
try:
self._work_complete.set()
except (RemoteError, KeyError):
self.log("An error occurred while cleaning up worker %r" %
(self, ))
self._flush_result_buffer()
self.output_queue.put(SentinelToken(self.token))
self.consumer_done_event.wait()
# joining the queue may not be necessary if we depend upon consumer_event_done
self.debug("... Process %s Queue Joining" % (self.name, ))
self.output_queue.join()
self.debug("... Process %s Finished" % (self.name, ))
def before_task(self):
'''A method to be overriden by subclasses that want to do something before
starting the task loop.
'''
pass
def task(self):
"""The worker process's main loop where it will poll for new work items,
process incoming work items and send them back to the master process.
"""
has_work = True
self.items_handled = 0
strikes = 0
while has_work:
try:
payload = self.input_queue.get(True, 5)
self.input_queue.task_done()
strikes = 0
except QueueEmptyException:
if self.producer_done_event.is_set():
has_work = False
break
else:
strikes += 1
self._flush_result_buffer()
if strikes % 1000 == 0:
self.log("... %d iterations without work for %r" %
(strikes, self))
continue
if self.items_handled % 100 == 0:
if time.time() - self.last_sent_result > 60:
self._flush_result_buffer()
# Handling a group of work items
if isinstance(payload, dict):
work_order = payload
self.items_handled += 1
try:
self.handle_group(work_order)
except Exception:
message = "An error occurred while processing %r on %r:\n%s" % (
work_order, self, traceback.format_exc())
self.log(message)
break
else: # Handling a single work item
structure, scan_ids = payload
self.items_handled += 1
try:
self.handle_item(structure, scan_ids)
except Exception:
message = "An error occurred while processing %r on %r:\n%s" % (
structure, self, traceback.format_exc())
self.log(message)
break
self.cleanup()
def run(self):
new_name = getattr(self, 'process_name', None)
if new_name is not None:
TaskBase().try_set_process_name(new_name)
try:
self.before_task()
except Exception:
self.log("An exception occurred during before_task for %r.\n%s" %
(self, traceback.format_exc()))
try:
self.task()
except Exception:
self.log("An exception occurred while executing %r.\n%s" %
(self, traceback.format_exc()))
self.cleanup()
