class SuggestionsThread(Thread): def __init__(self, suggestionsService): Thread.__init__(self) self.suggestionsService = suggestionsService self.value = None self.running = True self.condition = Condition() def run(self): while self.running: self.condition.acquire() if self.value is None: self.condition.wait() value = self.value self.value = None self.condition.release() if value is not None: self.suggestionsService.getSuggestions(value) def stop(self): self.running = False self.condition.acquire() self.condition.notify() self.condition.release() self.join() def getSuggestions(self, value): self.condition.acquire() self.value = value self.condition.notify() self.condition.release()
class DBPool(object):
def __init__(self, dbconfig, maxconn):
self.lock = ThreadLock()
self.cond = ThreadCondition(self.lock)
self.dbs = []
for _ in range(0, maxconn):
master_conn = None
slave_conn = None
if 'master' in dbconfig:
master_conn = Connection(dbconfig['master'])
if 'slave' in dbconfig:
slave_conn = Connection(dbconfig['slave'])
if master_conn is None and slave_conn is None:
master_conn = Connection(dbconfig)
self.dbs.append(ConnectionProxy(master_conn, slave_conn))
def close_idle(self):
pass
def acquire(self):
with self.lock:
if len(self.dbs) == 0:
self.cond.wait()
return self.dbs.pop()
def release(self, conn):
with self.lock:
old_cnt = len(self.dbs)
self.dbs.append(conn)
if not old_cnt:
self.cond.notify()
class MultithreadConnection(sqlite3.Connection):
def __init__(self, *args, **kwargs):
super(MultithreadConnection, self).__init__(
check_same_thread=False, *args, **kwargs)
self.waiting_list = []
self.mutex = Lock()
self.empty = Condition(self.mutex)
def _write_execute(self, method, sql, parameters=()):
self.waiting_list.append(None)
with self.mutex:
cur = method(sql, parameters)
self.waiting_list.pop()
if not self.waiting_list:
try:
self.empty.notify()
except RuntimeError:
pass
return cur
def commit(self):
with self.empty:
while self.waiting_list:
self.empty.wait()
super(MultithreadConnection, self).commit()
def write_execute(self, sql, parameters=()):
return self._write_execute(self.execute, sql, parameters)
def write_executemany(self, sql, parameters=()):
return self._write_execute(self.executemany, sql, parameters)
def writeable_cursor(self):
return WriteableCursor(self)
class GridGenerationUpdateThread(Thread):
disconnect = False
elecgen = None
sleeper = None
def __init__(self, parent):
Thread.__init__(self)
self.disconnect = False
self.elecgen = parent
self.ngdata = ElectricityGenerationDataSource()
self.sleeper = Condition()
def stopUpdates(self):
self.disconnect = True
self.sleeper.acquire()
self.sleeper.notify()
self.sleeper.release()
def run(self):
while self.disconnect == False:
emxml = self.ngdata.DownloadRealtimeXML()
try:
self.elecgen.energyMix = self.ngdata.ParseRealtimeXML(emxml)
except Exception, exc:
trc.Error("failed to parse realtime xml")
trc.Error(str(emxml))
trc.Error(repr(exc))
trc.Error(repr(exc.message))
# go to sleep for a while - the realtime data doesn't update very
# often, so no need to download it constantly!
self.sleeper.acquire()
self.sleeper.wait(180)
self.sleeper.release()
class FileTrigger(Thread):
"""Wait for a file.
"""
def __init__(self, filename):
Thread.__init__(self)
self.filename = filename
self.loop = True
self.cond = Condition()
# self.start()
self.exit_status = 0
def run(self):
while self.loop:
self.cond.acquire()
# Polling is ugly, this should be replaced when possible by inotify.
if os.path.exists(self.filename):
# Could we use yield instead ?
self.cond.release()
return
self.cond.wait(1)
self.cond.release()
def cancel(self):
self.loop = False
self.exit_status = 1
self.cond.acquire()
self.cond.notify()
self.cond.release()
def __repr__(self):
return "File trigger " + self.filename
class TimeTrigger(Thread):
"""Wait for a given time.
"""
def __init__(self, time):
Thread.__init__(self)
self.time = time
self.loop = True
self.cond = Condition()
# self.start()
self.exit_status = 0
def run(self):
while self.loop and self.time > datetime.utcnow():
self.cond.acquire()
now = datetime.utcnow()
diff = (self.time - now).seconds + (self.time - now).microseconds / 1000000.0
self.cond.wait(diff)
self.cond.release()
def cancel(self):
self.loop = False
self.exit_status = 1
self.cond.acquire()
self.cond.notify()
self.cond.release()
def __repr__(self):
return "Time trigger " + str(self.time)
class AutoFetcher(Thread):
_interval = 60 * 60 # = 1 hour
_inactive_block = None
def __init__(self):
super(AutoFetcher, self).__init__()
self._inactive_block = Condition()
self.daemon = True
def trigger(self):
with self._inactive_block:
self._inactive_block.notify()
def run(self):
while True:
self.launch_task()
time.sleep(self._interval)
if not center.main_win.win.isActiveWindow():
logging.debug('AutoFetcher paused.')
with self._inactive_block:
self._inactive_block.wait()
logging.debug('AutoFetcher resumed.')
@qt.run_in_qt
def launch_task(self):
tasks.run_task(tasks.FetchTask())
if center.settings['update_notify'] and '-dev' not in center.VERSION:
tasks.run_task(tasks.CheckUpdateTask())
class Semaphore(object):
def __init__(self, count=1):
if count < 1:
raise ValueError('Semaphore count should be at least 1.')
self._count = count
self._cond = Condition(Lock())
def P(self):
'''Acquire a lock.
Block until acquired successfully.
'''
with self._cond:
while self._count <= 0:
self._cond.wait() # block and wait for the lock.
else:
self._count = self._count - 1
def V(self):
'''Release a lock.'''
with self._cond:
self._count = self._count + 1
self._cond.notify() # wake up a waiting thread.
class MasterSlaveLock: def __init__(self): self.__master_lock = Lock() self.__slave_locks_counter_condition = Condition() self.__slave_locks_count = 0 def acquire_master(self): self.__master_lock.acquire() # wait for all slavelocks to unlock self.__slave_locks_counter_condition.acquire() while self.__slave_locks_count != 0: self.__slave_locks_counter_condition.wait() self.__slave_locks_counter_condition.release() def release_master(self): self.__master_lock.release() def acquire_slave(self): # stop further slave locks here, when master-process is waiting for a full stop of slave processes self.__master_lock.acquire() self.__master_lock.release() self.__slave_locks_counter_condition.acquire() self.__slave_locks_count += 1 self.__slave_locks_counter_condition.release() def release_slave(self): self.__slave_locks_counter_condition.acquire() self.__slave_locks_count -= 1 if not self.__slave_locks_count: # we released the last lock, notify (maybe) waiting master-process self.__slave_locks_counter_condition.notify() self.__slave_locks_counter_condition.release()
class _Timer(object):
def __init__(self, event_engine, seconds=1):
# 计时器，用于触发计时器事件
self._timer = Thread(target = self._run_timer)
self._timer.daemon = True
self._timer_active = False
self._timer_sleep = seconds
self._timer_pause_condition = Condition(Lock())
self._event_engine = event_engine
def set_timer(self, seconds):
self._timer_sleep = seconds
def start_timer(self):
self._timer_active = True
self._timer.start()
def pause_timer(self):
self._timer_active = False
self._timer_pause_condition.acquire()
def resume_timer(self):
self._timer_active = True
self._timer_pause_condition.notify()
self._timer_pause_condition.release()
def _run_timer(self):
event = Event(route=EVENT_TIMER)
while True:
with self._timer_pause_condition:
if not self._timer_active:
self._timer_pause_condition.wait()
self._event_engine.emit(event)
# 等待
sleep(self._timer_sleep)
class Club:
def __init__(self):
self.clubLock = Lock()
self.redditCond = Condition(self.clubLock)
self.fourchannerCond = Condition(self.clubLock)
self.fourchannerCount = 0
self.redditorCount = 0
def redditor_enter(self):
with self.clubLock:
while self.fourchannerCount > 0 :
self.redditCond.wait()
self.redditorCount = self.redditorCount + 1
def redditor_exit(self):
with self.clubLock:
self.redditorCount = self.redditorCount - 1
if self.redditorCount == 0 :
self.fourchannerCond.notify()
def fourchanner_enter(self):
with self.clubLock:
while self.redditorCount > 0 :
self.fourchannerCond.wait()
self.fourchannerCount = self.fourchannerCount + 1
def fourchanner_exit(self):
with self.clubLock:
self.fourchannerCount = self.fourchannerCount - 1
if self.fourchannerCount == 0:
self.redditCond.notify()
class Rtp_proxy_client_local(object):
is_local = True
wi_available = None
wi = None
nworkers = None
workers = None
def __init__(self, global_config, address = '/var/run/rtpproxy.sock', \
bind_address = None, nworkers = 1):
self.address = address
self.is_local = True
self.wi_available = Condition()
self.wi = []
self.nworkers = nworkers
self.workers = []
for i in range(0, self.nworkers):
self.workers.append(_RTPPLWorker(self))
def send_command(self, command, result_callback = None, *callback_parameters):
self.wi_available.acquire()
self.wi.append((command, result_callback, callback_parameters))
self.wi_available.notify()
self.wi_available.release()
def reconnect(self, address, bind_address = None):
self.shutdown()
self.address = address
self.workers = []
for i in range(0, self.nworkers):
self.workers.append(_RTPPLWorker(self))
def shutdown(self):
for rworker in self.workers:
rworker.shutdown()
self.workers = None
class _InterruptableLock(object):
def __init__(self, lock):
"""Creates a new lock."""
self.__lock = lock
self.__condition = Condition(Lock())
def __enter__(self):
return self.acquire()
def __exit__(self, exc_type, exc_value, traceback):
self.release()
def acquire(self, blocking=True, timeout=-1):
"""Acquire a lock, blocking or non-blocking.
Blocks until timeout, if timeout a positive float and blocking=True. A timeout
value of -1 blocks indefinitely, unless blocking=False."""
if not isinstance(timeout, (int, float)):
raise TypeError("a float is required")
if blocking:
# blocking indefinite
if timeout == -1:
with self.__condition:
while not self.__lock.acquire(False):
# condition with timeout is interruptable
self.__condition.wait(60)
return True
# same as non-blocking
elif timeout == 0:
return self.__lock.acquire(False)
elif timeout < 0:
raise ValueError("timeout value must be strictly positive (or -1)")
# blocking finite
else:
start = time()
waited_time = 0
with self.__condition:
while waited_time < timeout:
if self.__lock.acquire(False):
return True
else:
self.__condition.wait(timeout - waited_time)
waited_time = time() - start
return False
elif timeout != -1:
raise ValueError("can't specify a timeout for a non-blocking call")
else:
# non-blocking
return self.__lock.acquire(False)
def release(self):
"""Release a lock."""
self.__lock.release()
with self.__condition:
self.__condition.notify()
class Servermonitor:
def __init__(self):
self.serverlock = Lock()
self.numberofwaitingthreads = 0
self.numberofwaitingclients = 0
self.waitingthreadcond = Condition(self.serverlock)
def thread_start_serving(self,id):
with self.serverlock:
self.numberofwaitingthreads +=1
while self.numberofwaitingclients == 0:
self.waitingthreadcond.wait();
self.numberofwaitingclients -=1
#User by server to find if it can serve request or not
def checkifcanserveclient(self):
retval = False
with self.serverlock:
if(self.numberofwaitingthreads > 0):
retval = True
return retval
#This should only be called if there are waiting threads on the condition variable
def wakeupthread(self):
with self.serverlock:
self.numberofwaitingclients += 1
self.numberofwaitingthreads -=1
if(self.numberofwaitingthreads <0):
print "SHITsssssssssssssssssssssssssssssssssssssss"
self.waitingthreadcond.notify()
class Future(object):
def __init__(self, func, *args, **kwargs):
self.__done = False
self.__result = None
self.__cond = Condition()
self.__func = func
self.__args = args
self.__kwargs = kwargs
self.__except = None
def __call__(self):
with self.__cond:
try:
self.__result = self.__func(*self.__args, **self.__kwargs)
except:
self.__result = None
self.__except = sys.exc_info()
self.__done = True
self.__cond.notify()
def result(self):
with self.__cond:
while not self.__done:
self.__cond.wait()
if self.__except:
exc = self.__except
reraise(exc[0], exc[1], exc[2])
result = self.__result
return copy.deepcopy(result)
class MessageQueue(object):
"""The message queue used internally by Gossip."""
def __init__(self):
self._queue = deque()
self._condition = Condition()
def pop(self):
self._condition.acquire()
try:
while len(self._queue) < 1:
self._condition.wait()
return self._queue.pop()
finally:
self._condition.release()
def __len__(self):
return len(self._queue)
def __deepcopy__(self, memo):
newmq = MessageQueue()
newmq._queue = copy.deepcopy(self._queue, memo)
return newmq
def appendleft(self, msg):
self._condition.acquire()
try:
self._queue.appendleft(msg)
self._condition.notify()
finally:
self._condition.release()
class DataQueue ( Queue ):
def __init__ ( self, count ):
Queue.__init__(self)
self._count = count
self._cond = Condition()
def get ( self, block = True, timeout = None ):
self._cond.acquire()
if self.empty():
if not self._count:
raise Empty()
elif block:
self._cond.wait(timeout)
if self.empty() and not self._count:
self._cond.release()
raise Empty()
self._cond.release()
return Queue.get(self, block, timeout)
def put ( self, data ):
self._cond.acquire()
self._count = self._count - 1
Queue.put(self, data)
if self._count:
self._cond.notify()
else:
self._cond.notifyAll()
self._cond.release()
def remain ( self ):
return self._count + self.unfinished_tasks
class ZEventLoopThread(object):
def __init__(self, cb=None):
self._event_loop = None
self._thr = Thread(target=self._thread_func)
self._guard = Condition(Lock())
self._thr_callback = cb
def start_loop(self):
self._thr.start()
self._guard.acquire()
while self._event_loop is None:
self._guard.wait()
return self._event_loop
def _thread_func(self):
# Note: create event loop in the new thread
# instead in the constructor
event_loop = ZEventLoop()
if self._thr_callback:
self._thr_callback(event_loop)
with self._guard:
self._event_loop = event_loop
self._guard.notify()
event_loop.loop()
class External_command(object):
work_available = None
work = None
def __init__(self, command, max_workers = _MAX_WORKERS):
self.work_available = Condition()
self.work = []
for i in range(0, max_workers):
_Worker(command, self)
def process_command(self, data, result_callback, *callback_parameters):
wi = Work_item(tuple(data), result_callback, callback_parameters)
self.work_available.acquire()
self.work.append(wi)
self.work_available.notify()
self.work_available.release()
return wi
def shutdown(self):
self.work_available.acquire()
self.work.append(None)
self.work_available.notify()
self.work_available.release()
def process_result(self, result_callback, result, *callback_parameters):
try:
result_callback(result, *callback_parameters)
except Exception as ex:
if isinstance(ex, SystemExit):
raise
dump_exception('External_command: unhandled exception in external command results callback')
class SharedCell(object):
"""Shared data for the producer/consumer problem."""
def __init__(self):
self._data = -1
self._writeable = True
self._condition = Condition()
def setData(self, data):
"""Producer's method to write to shared data."""
self._condition.acquire()
while not self._writeable:
self._condition.wait()
print "%s setting data to %d" % \
(currentThread().getName(), data)
self._data = data
self._writeable = False
self._condition.notify()
self._condition.release()
def getData(self):
"""Consumer's method to read from shared data."""
self._condition.acquire()
while self._writeable:
self._condition.wait()
print "%s accessing data %d" % \
(currentThread().getName(), self._data)
self._writeable = True
self._condition.notify()
self._condition.release()
return self._data
class External_command(object):
work_available = None
work = None
def __init__(self, command, max_workers = _MAX_WORKERS):
self.work_available = Condition()
self.work = []
for i in range(0, max_workers):
_Worker(command, self)
def process_command(self, data, result_callback, *callback_parameters):
wi = Work_item(tuple(data), result_callback, callback_parameters)
self.work_available.acquire()
self.work.append(wi)
self.work_available.notify()
self.work_available.release()
return wi
def shutdown(self):
self.work_available.acquire()
self.work.append(None)
self.work_available.notify()
self.work_available.release()
def process_result(self, result_callback, result, *callback_parameters):
try:
result_callback(result, *callback_parameters)
except:
print datetime.now(), 'External_command: unhandled exception in external command results callback'
print '-' * 70
print_exc(file = stdout)
print '-' * 70
stdout.flush()
class Queue(object):
def __init__(self):
self.lock = Lock()
self.cond = Condition(self.lock)
self.data = []
self.live = True
def put(self, values):
self.cond.acquire()
self.data.extend(values)
self.cond.notify()
self.cond.release()
def get(self):
if not self.cond.acquire(False):
return []
self.cond.wait()
result = self.data
self.data = []
self.cond.release()
return result
def close(self):
self.cond.acquire()
self.live = False
self.cond.notify()
self.cond.release()
class RepceJob(object):
"""class representing message status we can use
for waiting on reply"""
def __init__(self, cbk):
"""
- .rid: (process-wise) unique id
- .cbk: what we do upon receiving reply
"""
self.rid = (os.getpid(), thread.get_ident(), time.time())
self.cbk = cbk
self.lever = Condition()
self.done = False
def __repr__(self):
return ':'.join([str(x) for x in self.rid])
def wait(self):
self.lever.acquire()
if not self.done:
self.lever.wait()
self.lever.release()
return self.result
def wakeup(self, data):
self.result = data
self.lever.acquire()
self.done = True
self.lever.notify()
self.lever.release()
class ConnectorPool:
""" A thread-safe connection pool.
TODO: Make this an actual queue, not a stack. Nomenclature is imporant
sometimes.
"""
def __init__(self, Connector, count=20, user=None, password=None, db=None):
self.pool = [ Connector(user, password, db) for _ in range(count) ]
self.C_pool = Condition()
def dequeue(self):
""" Get connector.
:rtype: seaice.SeaIceConnector.SeaIceConnector
"""
self.C_pool.acquire()
while len(self.pool) == 0:
self.C_pool.wait()
db_con = self.pool.pop()
self.C_pool.release()
return db_con
def enqueue(self, db_con):
""" Release connector.
:param db_con: The connector.
:type db_con: seaice.SeaIceConnector.SeaIceConnector
"""
self.C_pool.acquire()
self.pool.append(db_con)
self.C_pool.notify()
self.C_pool.release()
class NotificationQueue(Queue):
""" A queue class that notifies waiting threads when the queue is available for
reading."""
def __init__(self):
""" Constructor."""
Queue.__init__(self)
self.lock = Condition()
def get(self):
""" Get an item from the queue. This method is thread-safe.
Method must be called on a non-empty queue.
Return : the first item from the queue
Raises: a EmptyQueue exeception
"""
return Queue.get(self, False)
def put(self, item):
""" Add a new item to the queue. This method is thread-safe.
Threads that are waiting for get an item are notified.
item: new item to add to the queue.
Return : None
"""
self.lock.acquire()
Queue.put(self, item)
self.lock.notify()
self.lock.release()
def acquire(self):
self.lock.acquire()
def release(self):
self.lock.release()
def wait(self):
self.lock.wait()
class SSHProtocol(Protocol):
def __init__(self,proxy):
print "SShProtocol %s"%self.rules
Protocol.__init__(self,self.rules)
self.proxy=proxy
self.cv=Condition()
def init(self):
print "OK initialized"
self.state=2
def handle(self,msg):
print "handle message %s"%msg
if msg['type'] == "send":
thread.start_new_thread(self.send, (msg,))
def send(self,message):
print "Sending Message %s"%message
self.release()
def release(self):
self.cv.acquire()
self.cv.notify()
self.cv.release()
def wait(self):
self.cv.acquire()
self.cv.wait()
self.cv.release()
rules={1:init,2:handle}
class TmThread(Thread):
def __init__(self, target, args): # name, event_for_wait, event_for_set
Thread.__init__(self) # , target=target, args=args)
self.__target = target
self.__args = args
self.__kwargs = None
self.priority = 0
self.time = 0
#flag to Finish thread
self.Finishd = False
# Explicitly using Lock over RLock since the use of self.Finishd
# break reentrancy anyway, and I believe using Lock could allow
# one thread to Finish the worker, while another resumes; haven't
# checked if Condition imposes additional limitations that would
# prevent that. In Python 2, use of Lock instead of RLock also
# boosts performance.
self.Finish_cond = Condition(Lock())
def Finish(self):
self.Finishd = True
# If in sleep, we acquire immediately, otherwise we wait for thread
# to release condition. In race, worker will still see self.Finishd
# and begin waiting until it's set back to False
self.Finish_cond.acquire()
#should just resume the thread
def resume(self):
self.Finishd = False
# Notify so thread will wake after lock released
self.Finish_cond.notify()
# Now release the lock
self.Finish_cond.release()
class DirtyBit:
def __init__(self):
self._mon = RLock()
self._rc = Condition(self._mon)
self._dirty = 0
def clear(self):
self._mon.acquire()
self._dirty = 0
#self._rc.notify() only interested in knowing when we are dirty
self._mon.release()
def set(self):
self._mon.acquire()
self._dirty = 1
self._rc.notify()
self._mon.release()
def value(self):
return self._dirty
def wait(self):
self._mon.acquire()
self._rc.wait()
self._mon.release()
class Future:
def __init__(self, correlation_id, request_type=None):
self.correlation_id = correlation_id
self._result = None
self._condition = Condition()
self._request_type = request_type
def done(self):
return self._result is not None
def result(self, timeout=None):
with self._condition:
if self._result is None:
if not self._condition.wait(timeout):
message_type = validator_pb2.Message.MessageType.Name(
self._request_type) if self._request_type else None
raise FutureTimeoutError(
'Future timed out waiting for response to {}'.format(
message_type))
return self._result
def set_result(self, result):
with self._condition:
self._result = result
self._condition.notify()
class Mailboxmonitor:
def __init__(self):
self.mailboxlock = Lock()
self.mailidentifier = 0
self.fileHandle = open("mailbox", "w")
self.backupthreadactive = 0
self.backupthreadswaiting = 0
self.waitingforbackuptocomplete = Condition(self.mailboxlock)
self.backupneededcond = Condition(self.mailboxlock)
def delivermail(self, clientname, mailsender, mailreceiver, mailcontent):
with self.mailboxlock:
#if back up is happening stop writes everywhere
while self.backupthreadactive == 1:
self.waitingforbackuptocomplete.wait()
mail = "Received from " + str(clientname) + " by pm489 (CS4410MP3)\n"
self.mailidentifier += 1
mail += "Number: " + str(self.mailidentifier) + "\n"
mail += "From: " + str(mailsender) + "\n"
for i in range(0, len(mailreceiver)):
mail += "To: " + str(mailreceiver[i]) + "\n"
mail += "\n"
mail += mailcontent + "\n"
mail += "\n"
while self.backupthreadactive == 1: #if back up is happening stop writes everywhere
print "worker sleeping"
self.waitingforbackuptocomplete.wait()
self.fileHandle.write(mail)
self.fileHandle.flush()
if self.mailidentifier%32 == 0 and self.backupthreadswaiting > 0:
self.backupthreadactive = 1
self.backupthreadswaiting = 0
self.backupneededcond.notify()
return self.mailidentifier
def startbackupthread(self):
with self.mailboxlock:
while self.backupthreadactive==0:
#print "backup slept"
self.backupthreadswaiting = 1
self.backupneededcond.wait()
def initiatebackup(self, lastbackedmessage):
with self.mailboxlock:
if( self.mailidentifier%32 != 0):
print "Something went very horribly wrong"
newfilename = "mailbox." + str(lastbackedmessage+1)+"-"+ str(lastbackedmessage+32)
self.fileHandle.close()
os.rename("mailbox", newfilename)
self.fileHandle = open("mailbox", "w")
lastbackedmessage += 32
self.backupthreadactive = 0
self.waitingforbackuptocomplete.notifyAll() #wake up all waiting threads to write
return lastbackedmessage
class Runner:
def __init__(self):
logger.info('Starting performances node')
self.robot_name = rospy.get_param('/robot_name')
self.running = False
self.paused = False
self.pause_time = 0
self.start_time = 0
self.start_timestamp = 0
self.lock = Lock()
self.run_condition = Condition()
self.queue = Queue.Queue()
self.ids = []
self.running_nodes = []
# References to event subscribing node callbacks
self.observers = {}
self.worker = Thread(target=self.worker)
self.worker.setDaemon(True)
rospy.init_node('performances')
self.services = {
'head_pau_mux':
rospy.ServiceProxy('/' + self.robot_name + '/head_pau_mux/select',
MuxSelect),
'neck_pau_mux':
rospy.ServiceProxy('/' + self.robot_name + '/neck_pau_mux/select',
MuxSelect)
}
self.topics = {
'look_at':
rospy.Publisher('/blender_api/set_face_target',
Target,
queue_size=1),
'gaze_at':
rospy.Publisher('/blender_api/set_gaze_target',
Target,
queue_size=1),
'head_rotation':
rospy.Publisher('/blender_api/set_head_rotation',
Float32,
queue_size=1),
'emotion':
rospy.Publisher('/blender_api/set_emotion_state',
EmotionState,
queue_size=3),
'gesture':
rospy.Publisher('/blender_api/set_gesture',
SetGesture,
queue_size=3),
'expression':
rospy.Publisher('/' + self.robot_name + '/make_face_expr',
MakeFaceExpr,
queue_size=3),
'kfanimation':
rospy.Publisher('/' + self.robot_name + '/play_animation',
PlayAnimation,
queue_size=3),
'interaction':
rospy.Publisher('/behavior_switch', String, queue_size=1),
'bt_control':
rospy.Publisher('/behavior_control', Int32, queue_size=1),
'events':
rospy.Publisher('~events', Event, queue_size=1),
'chatbot':
rospy.Publisher('/' + self.robot_name + '/speech',
ChatMessage,
queue_size=1),
'speech_events':
rospy.Publisher('/' + self.robot_name + '/speech_events',
String,
queue_size=1),
'soma_state':
rospy.Publisher("/blender_api/set_soma_state",
SomaState,
queue_size=2),
'tts': {
'en':
rospy.Publisher('/' + self.robot_name + '/tts_en',
String,
queue_size=1),
'zh':
rospy.Publisher('/' + self.robot_name + '/tts_zh',
String,
queue_size=1),
'default':
rospy.Publisher('/' + self.robot_name + '/tts',
String,
queue_size=1),
}
}
rospy.Service('~load', srv.Load, self.load_callback)
rospy.Service('~load_nodes', srv.LoadNodes, self.load_nodes_callback)
rospy.Service('~load_sequence', srv.LoadSequence,
self.load_sequence_callback)
rospy.Service('~run', srv.Run, self.run_callback)
rospy.Service('~run_by_name', srv.RunByName, self.run_by_name_callback)
rospy.Service('~resume', srv.Resume, self.resume_callback)
rospy.Service('~pause', srv.Pause, self.pause_callback)
rospy.Service('~stop', srv.Stop, self.stop)
rospy.Service('~current', srv.Current, self.current_callback)
# Shared subscribers for nodes
rospy.Subscriber('/' + self.robot_name + '/speech_events', String,
lambda msg: self.notify('speech_events', msg))
self.worker.start()
rospy.spin()
def load_callback(self, request):
return srv.LoadResponse(success=True,
nodes=json.dumps(
self.load_sequence([request.id])))
def load_nodes_callback(self, request):
self.load_nodes(json.loads(request.nodes))
return srv.LoadNodesResponse(True)
def load_sequence_callback(self, request):
return srv.LoadSequenceResponse(success=True,
nodes=json.dumps(
self.load_sequence(request.ids)))
def run_by_name_callback(self, request):
self.stop()
nodes = self.load_sequence([request.id])
if not nodes:
return srv.RunByNameResponse(False)
return srv.RunByNameResponse(self.run(0.0))
def load_sequence(self, ids):
nodes = []
offset = 0
for id in ids:
robot_name = rospy.get_param('/robot_name')
path = os.path.join(rospack.get_path('robots_config'), robot_name,
'performances', id + '.yaml')
duration = 0
if os.path.isfile(path):
with open(path, 'r') as f:
data = yaml.load(f.read())
if 'nodes' in data and isinstance(data['nodes'], list):
for node in data['nodes']:
if not 'start_time' in node:
node['start_time'] = 0
if node['name'] == 'pause':
node['duration'] = 0.1
duration = max(
duration,
(node['duration'] if 'duration' in node else 0) +
node['start_time'])
node['start_time'] += offset
offset += duration
nodes += data['nodes']
self.load_nodes(nodes, ids)
return nodes
def load_nodes(self, nodes, ids=None):
self.stop()
if ids is None:
ids = []
for node in nodes:
node.pop('id', None)
with self.lock:
self.ids = ids
self.running_nodes = nodes
def run_callback(self, request):
return srv.RunResponse(self.run(request.startTime))
def run(self, start_time):
self.stop()
# Wait for worker to stop performance and enter waiting before proceeding
self.run_condition.acquire()
with self.lock:
rospy.logerr(start_time)
if len(self.running_nodes) > 0:
self.running = True
self.start_time = start_time
self.start_timestamp = time.time()
# notify worker thread
self.run_condition.notify()
self.run_condition.release()
return True
else:
return False
def resume_callback(self, request):
success = self.resume()
with self.lock:
run_time = self.get_run_time()
return srv.ResumeResponse(success, run_time)
def resume(self):
success = False
with self.lock:
if self.running and self.paused:
run_time = self.get_run_time()
self.paused = False
self.start_timestamp = time.time() - run_time
self.start_time = 0
self.topics['events'].publish(Event('resume', run_time))
success = True
return success
def stop(self, request=None):
stop_time = 0
with self.lock:
if self.running:
stop_time = self.get_run_time()
self.running = False
self.paused = False
return srv.StopResponse(True, stop_time)
def pause_callback(self, request):
if self.pause():
with self.lock:
return srv.PauseResponse(True, self.get_run_time())
else:
return srv.PauseResponse(False, 0)
# Pauses current
def pause(self):
with self.lock:
if self.running and not self.paused:
self.pause_time = time.time()
self.paused = True
self.topics['events'].publish(
Event('paused', self.get_run_time()))
return True
else:
return False
# Returns current performance
def current_callback(self, request):
with self.lock:
current_time = self.get_run_time()
running = self.running and not self.paused
return srv.CurrentResponse(ids=self.ids,
nodes=json.dumps(self.running_nodes),
current_time=current_time,
running=running)
def worker(self):
self.run_condition.acquire()
while True:
with self.lock:
self.paused = False
self.topics['events'].publish(Event('idle', 0))
self.run_condition.wait()
with self.lock:
nodes = [
Node.createNode(node, self, self.start_time)
for node in self.running_nodes
]
self.topics['events'].publish(Event('running', self.start_time))
if len(nodes) == 0:
continue
running = True
while running:
with self.lock:
run_time = self.get_run_time()
if not self.running:
self.topics['events'].publish(
Event('finished', run_time))
break
elif self.paused:
continue
running = False
# checks if any nodes still running
for node in nodes:
running = node.run(run_time) or running
self.run_condition.release()
def get_run_time(self):
"""
Must acquire self.lock in order to safely use this method
:return:
"""
run_time = 0
if self.running:
run_time = self.start_time
if self.paused:
run_time += self.pause_time - self.start_timestamp
else:
run_time += time.time() - self.start_timestamp
return run_time
# Notifies register nodes on the events from ROS.
def notify(self, event, msg):
if event not in self.observers.keys():
return
for i in xrange(len(self.observers[event]) - 1, -1, -1):
try:
self.observers[event][i](msg)
except TypeError:
# Remove dead methods
del self.observers[event][i]
# Registers callbacks for specific events. Uses weak reference to allow nodes cleanup after finish.
def register(self, event, cb):
if not event in self.observers:
self.observers[event] = []
m = WeakMethod(cb)
self.observers[event].append(m)
return m
# Allows nodes to unsubscribe from events
def unregister(self, event, ref):
if event in self.observers:
if ref in self.observers[event]:
self.observers[event].remove(ref)
class PooledDB:
"""Pool for DB-API 2 connections.
After you have created the connection pool, you can use
connection() to get pooled, steady DB-API 2 connections.
"""
version = __version__
def __init__(self,
creator,
mincached=0,
maxcached=0,
maxshared=0,
maxconnections=0,
blocking=False,
maxusage=None,
setsession=None,
reset=True,
failures=None,
ping=1,
*args,
**kwargs):
"""Set up the DB-API 2 connection pool.
creator: either an arbitrary function returning new DB-API 2
connection objects or a DB-API 2 compliant database module
mincached: initial number of idle connections in the pool
(0 means no connections are made at startup)
maxcached: maximum number of idle connections in the pool
(0 or None means unlimited pool size)
maxshared: maximum number of shared connections
(0 or None means all connections are dedicated)
When this maximum number is reached, connections are
shared if they have been requested as shareable.
maxconnections: maximum number of connections generally allowed
(0 or None means an arbitrary number of connections)
blocking: determines behavior when exceeding the maximum
(if this is set to true, block and wait until the number of
connections decreases, otherwise an error will be reported)
maxusage: maximum number of reuses of a single connection
(0 or None means unlimited reuse)
When this maximum usage number of the connection is reached,
the connection is automatically reset (closed and reopened).
setsession: optional list of SQL commands that may serve to prepare
the session, e.g. ["set datestyle to ...", "set time zone ..."]
reset: how connections should be reset when returned to the pool
(False or None to rollback transcations started with begin(),
True to always issue a rollback for safety's sake)
failures: an optional exception class or a tuple of exception classes
for which the connection failover mechanism shall be applied,
if the default (OperationalError, InternalError) is not adequate
ping: determines when the connection should be checked with ping()
(0 = None = never, 1 = default = whenever fetched from the pool,
2 = when a cursor is created, 4 = when a query is executed,
7 = always, and all other bit combinations of these values)
args, kwargs: the parameters that shall be passed to the creator
function or the connection constructor of the DB-API 2 module
"""
try:
threadsafety = creator.threadsafety
except AttributeError:
try:
if not callable(creator.connect):
raise AttributeError
except AttributeError:
threadsafety = 2
else:
threadsafety = 0
if not threadsafety:
raise NotSupportedError("Database module is not thread-safe.")
self._creator = creator
self._args, self._kwargs = args, kwargs
self._maxusage = maxusage
self._setsession = setsession
self._reset = reset
self._failures = failures
self._ping = ping
if mincached is None:
mincached = 0
if maxcached is None:
maxcached = 0
if maxconnections is None:
maxconnections = 0
if maxcached:
if maxcached < mincached:
maxcached = mincached
self._maxcached = maxcached
else:
self._maxcached = 0
if threadsafety > 1 and maxshared:
self._maxshared = maxshared
self._shared_cache = [] # the cache for shared connections
else:
self._maxshared = 0
if maxconnections:
if maxconnections < maxcached:
maxconnections = maxcached
if maxconnections < maxshared:
maxconnections = maxshared
self._maxconnections = maxconnections
else:
self._maxconnections = 0
self._idle_cache = [] # the actual pool of idle connections
self._condition = Condition()
if not blocking:
def wait():
raise TooManyConnections
self._condition.wait = wait
self._connections = 0
# Establish an initial number of idle database connections:
idle = [self.dedicated_connection() for i in range(mincached)]
while idle:
idle.pop().close()
def steady_connection(self):
"""Get a steady, unpooled DB-API 2 connection."""
return connect(self._creator, self._maxusage, self._setsession,
self._failures, self._ping, True, *self._args,
**self._kwargs)
def connection(self, shareable=True):
"""Get a steady, cached DB-API 2 connection from the pool.
If shareable is set and the underlying DB-API 2 allows it,
then the connection may be shared with other threads.
"""
if shareable and self._maxshared:
self._condition.acquire()
try:
while (not self._shared_cache and self._maxconnections
and self._connections >= self._maxconnections):
self._condition.wait()
if len(self._shared_cache) < self._maxshared:
# shared cache is not full, get a dedicated connection
try: # first try to get it from the idle cache
con = self._idle_cache.pop(0)
except IndexError: # else get a fresh connection
con = self.steady_connection()
else:
con._ping_check() # check this connection
con = SharedDBConnection(con)
self._connections += 1
else: # shared cache full or no more connections allowed
self._shared_cache.sort() # least shared connection first
con = self._shared_cache.pop(0) # get it
while con.con._transaction:
# do not share connections which are in a transaction
self._shared_cache.insert(0, con)
self._condition.wait()
self._shared_cache.sort()
con = self._shared_cache.pop(0)
con.con._ping_check() # check the underlying connection
con.share() # increase share of this connection
# put the connection (back) into the shared cache
self._shared_cache.append(con)
self._condition.notify()
finally:
self._condition.release()
con = PooledSharedDBConnection(self, con)
else: # try to get a dedicated connection
self._condition.acquire()
try:
while (self._maxconnections
and self._connections >= self._maxconnections):
self._condition.wait()
# connection limit not reached, get a dedicated connection
try: # first try to get it from the idle cache
con = self._idle_cache.pop(0)
except IndexError: # else get a fresh connection
con = self.steady_connection()
else:
con._ping_check() # check connection
con = PooledDedicatedDBConnection(self, con)
self._connections += 1
finally:
self._condition.release()
return con
def dedicated_connection(self):
"""Alias for connection(shareable=False)."""
return self.connection(False)
def unshare(self, con):
"""Decrease the share of a connection in the shared cache."""
self._condition.acquire()
try:
con.unshare()
shared = con.shared
if not shared: # connection is idle,
try: # so try to remove it
self._shared_cache.remove(con) # from shared cache
except ValueError:
pass # pool has already been closed
finally:
self._condition.release()
if not shared: # connection has become idle,
self.cache(con.con) # so add it to the idle cache
def cache(self, con):
"""Put a dedicated connection back into the idle cache."""
self._condition.acquire()
try:
if not self._maxcached or len(self._idle_cache) < self._maxcached:
con._reset(force=self._reset) # rollback possible transaction
# the idle cache is not full, so put it there
self._idle_cache.append(con) # append it to the idle cache
else: # if the idle cache is already full,
con.close() # then close the connection
self._connections -= 1
self._condition.notify()
finally:
self._condition.release()
def close(self):
"""Close all connections in the pool."""
self._condition.acquire()
try:
while self._idle_cache: # close all idle connections
con = self._idle_cache.pop(0)
try:
con.close()
except Exception:
pass
if self._maxshared: # close all shared connections
while self._shared_cache:
con = self._shared_cache.pop(0).con
try:
con.close()
except Exception:
pass
self._connections -= 1
self._condition.notifyAll()
finally:
self._condition.release()
def __del__(self):
"""Delete the pool."""
try:
self.close()
except Exception:
pass
class CairnsmoreHotplugWorker(BaseWorker):
version = "theseven.cairnsmore hotplug manager v0.1.0"
default_name = "Cairnsmore hotplug manager"
can_autodetect = True
settings = dict(
BaseWorker.settings, **{
"baudrate": {
"title": "Baud rate",
"type": "int",
"position": 1100
},
"jobinterval": {
"title": "Job interval",
"type": "float",
"position": 1200
},
"initialspeed": {
"title": "Initial clock frequency",
"type": "int",
"position": 2000
},
"maximumspeed": {
"title": "Maximum clock frequency",
"type": "int",
"position": 2100
},
"invalidwarning": {
"title": "Warning invalids",
"type": "int",
"position": 3200
},
"invalidcritical": {
"title": "Critical invalids",
"type": "int",
"position": 3300
},
"warmupstepshares": {
"title": "Shares per warmup step",
"type": "int",
"position": 3400
},
"speedupthreshold": {
"title": "Speedup threshold",
"type": "int",
"position": 3500
},
"scaninterval": {
"title": "Bus scan interval",
"type": "float",
"position": 4000
},
})
@classmethod
def autodetect(self, core):
try:
import serial
found = False
for port in glob(
"/dev/serial/by-id/usb-FTDI_Cairnsmore1_*-if0?-port0"):
try:
handle = serial.Serial(port, 115200, serial.EIGHTBITS,
serial.PARITY_NONE,
serial.STOPBITS_ONE, 1, False,
False, 5, False, None)
handle.close()
found = True
break
except:
pass
if found: core.add_worker(self(core))
except:
pass
# Constructor, gets passed a reference to the miner core and the saved worker state, if present
def __init__(self, core, state=None):
# Initialize bus scanner wakeup event
self.wakeup = Condition()
# Let our superclass do some basic initialization and restore the state if neccessary
super(CairnsmoreHotplugWorker, self).__init__(core, state)
# Validate settings, filling them with default values if neccessary.
# Called from the constructor and after every settings change.
def apply_settings(self):
# Let our superclass handle everything that isn't specific to this worker module
super(CairnsmoreHotplugWorker, self).apply_settings()
if not "scaninterval" in self.settings or not self.settings.scaninterval:
self.settings.scaninterval = 10
if not "baudrate" in self.settings or not self.settings.baudrate:
self.settings.baudrate = 115200
if not "jobinterval" in self.settings or not self.settings.jobinterval:
self.settings.jobinterval = 20
if not "initialspeed" in self.settings:
self.settings.initialspeed = 150
self.settings.initialspeed = min(max(self.settings.initialspeed, 4),
250)
if not "maximumspeed" in self.settings:
self.settings.maximumspeed = 200
self.settings.maximumspeed = min(max(self.settings.maximumspeed, 4),
300)
if not "invalidwarning" in self.settings:
self.settings.invalidwarning = 2
self.settings.invalidwarning = min(
max(self.settings.invalidwarning, 1), 10)
if not "invalidcritical" in self.settings:
self.settings.invalidcritical = 10
self.settings.invalidcritical = min(
max(self.settings.invalidcritical, 1), 50)
if not "warmupstepshares" in self.settings:
self.settings.warmupstepshares = 5
self.settings.warmupstepshares = min(
max(self.settings.warmupstepshares, 1), 10000)
if not "speedupthreshold" in self.settings:
self.settings.speedupthreshold = 100
self.settings.speedupthreshold = min(
max(self.settings.speedupthreshold, 50), 10000)
# Push our settings down to our children
fields = [
"baudrate", "jobinterval", "initialspeed", "maximumspeed",
"invalidwarning", "invalidcritical", "warmupstepshares",
"speedupthreshold"
]
for child in self.children:
for field in fields:
child.settings[field] = self.settings[field]
child.apply_settings()
# Rescan the bus immediately to apply the new settings
with self.wakeup:
self.wakeup.notify()
# Reset our state. Called both from the constructor and from self.start().
def _reset(self):
# Let our superclass handle everything that isn't specific to this worker module
super(CairnsmoreHotplugWorker, self)._reset()
# These need to be set here in order to make the equality check in apply_settings() happy,
# when it is run before starting the module for the first time. (It is called from the constructor.)
# Start up the worker module. This is protected against multiple calls and concurrency by a wrapper.
def _start(self):
# Let our superclass handle everything that isn't specific to this worker module
super(CairnsmoreHotplugWorker, self)._start()
# Initialize child map
self.childmap = {}
# Reset the shutdown flag for our threads
self.shutdown = False
# Start up the main thread, which handles pushing work to the device.
self.mainthread = Thread(None, self.main, self.settings.name + "_main")
self.mainthread.daemon = True
self.mainthread.start()
# Shut down the worker module. This is protected against multiple calls and concurrency by a wrapper.
def _stop(self):
# Let our superclass handle everything that isn't specific to this worker module
super(CairnsmoreHotplugWorker, self)._stop()
# Set the shutdown flag for our threads, making them terminate ASAP.
self.shutdown = True
# Trigger the main thread's wakeup flag, to make it actually look at the shutdown flag.
with self.wakeup:
self.wakeup.notify()
# Wait for the main thread to terminate.
self.mainthread.join(10)
# Shut down child workers
while self.children:
child = self.children.pop(0)
try:
self.core.log(
self,
"Shutting down worker %s...\n" % (child.settings.name),
800)
child.stop()
except Exception as e:
self.core.log(
self, "Could not stop worker %s: %s\n" %
(child.settings.name, traceback.format_exc()), 100, "rB")
# Main thread entry point
# This thread is responsible for scanning for boards and spawning worker modules for them
def main(self):
import serial
# Loop until we are shut down
while not self.shutdown:
try:
boards = {}
for port in glob(
"/dev/serial/by-id/usb-FTDI_Cairnsmore1_*-if0?-port0"):
available = False
try:
handle = serial.Serial(port, 115200, serial.EIGHTBITS,
serial.PARITY_NONE,
serial.STOPBITS_ONE, 1, False,
False, 5, False, None)
handle.close()
available = True
except:
pass
boards[port] = available
kill = []
for port, child in self.childmap.items():
if not port in boards:
kill.append((port, child))
for port, child in kill:
try:
self.core.log(
self, "Shutting down worker %s...\n" %
(child.settings.name), 800)
child.stop()
except Exception as e:
self.core.log(
self, "Could not stop worker %s: %s\n" %
(child.settings.name, traceback.format_exc()), 100,
"rB")
childstats = child.get_statistics()
fields = [
"ghashes", "jobsaccepted", "jobscanceled",
"sharesaccepted", "sharesrejected", "sharesinvalid"
]
for field in fields:
self.stats[field] += childstats[field]
try:
self.child.destroy()
except:
pass
del self.childmap[port]
try:
self.children.remove(child)
except:
pass
for port, available in boards.items():
if port in self.childmap or not available: continue
child = CairnsmoreWorker(self.core)
child.settings.name = "Cairnsmore1 board %s FPGA%s" % (
re.match(
"/dev/serial/by-id/usb-FTDI_Cairnsmore1_([0-9A-Z]+)-if0([0-3])-port0",
port).group(1, 2))
child.settings.port = port
fields = [
"jobinterval", "baudrate", "initialspeed",
"maximumspeed", "invalidwarning", "invalidcritical",
"warmupstepshares", "speedupthreshold"
]
for field in fields:
child.settings[field] = self.settings[field]
child.apply_settings()
self.childmap[port] = child
self.children.append(child)
try:
self.core.log(
self, "Starting up worker %s...\n" %
(child.settings.name), 800)
child.start()
except Exception as e:
self.core.log(
self, "Could not start worker %s: %s\n" %
(child.settings.name, traceback.format_exc()), 100,
"rB")
except:
self.core.log(
self, "Caught exception: %s\n" % traceback.format_exc(),
100, "rB")
with self.wakeup:
self.wakeup.wait(self.settings.scaninterval)
class GAThread(Thread):
"""
separate thread to run Genetic Algorithm while not blocking
the main thread.
"""
def __init__(self):
Thread.__init__(self)
# flag if start method has been called
self.start_triggered = False
# thread pause condition
self.pause_cond = Condition(Lock())
# flag to pause thread
self.__pause_now = False
# flag to state thread state
self.paused = True
# flag to stop thread
self.__stop_now = False
def run(self):
# initializing the population
pop = Population(g_pop_size, g_genes_num)
# fittest fitness of the previous generation, used to send deviation value
prv_fitness = pop.fittest().fitness()
# started signal to the renderer process
to_json({
"started": True,
"genesNum": pop.genes_num,
"fitness": prv_fitness,
"first-step": self.__pause_now,
})
# first generated solutions (generation 0)
to_json({
"generation": pop.generation,
"genes": pop.fittest().genes,
"fitness": prv_fitness,
})
while g_max_gen is False or g_max_gen < 0 or pop.generation < g_max_gen:
Evolve.evolve_population(pop)
# takes the current generation fitness
cur_fitness = pop.fittest().fitness()
# if g_del_rate is 0 or False than just ignore it
if g_del_rate:
sleep(g_del_rate)
if g_pause_gen is not False and pop.generation == g_pause_gen + 1:
self.pause()
to_json({'forced-pause': True})
# pause check, moved down to avoid another iteration if stop event
# was triggered after a pause event
self.__pause_check()
# stopped event, separating finished naturally (if there is valid
# solution) from being forcefully stopped
if self.__stop_now:
to_json({"stopped": True})
return
# moved down, so when GA is heavy (slow), user might stop it before the point is added
# the point must not be added, so ga stops before executing below code
to_json({
"prv-fitness": prv_fitness,
"fitness": cur_fitness,
"generation": pop.generation,
"genes": pop.fittest().genes,
})
# update prv_fitness
prv_fitness = cur_fitness
# finished event
to_json({"finished": True})
def __pause_check(self):
"""
pause if pause() is called
"""
if self.__pause_now:
# halt
self.pause_cond.acquire()
self.__pause_now = False
self.paused = True
with self.pause_cond:
while self.paused:
self.pause_cond.wait()
# should just resume the thread
def __resume(self):
"""
resume thread if paused
"""
# thread should be paused to resume
if self.paused:
# Notify so thread will wake after lock released
self.pause_cond.notify()
# Now release the lock
self.pause_cond.release()
self.paused = False
# notify app
to_json({"resumed": True})
# user triggered pause (through play button) through GUI and self.paused is still false means
# GA is too slow on generating the next generation, than when the user clicked play (for resume)
# it just turns self.__pause_now to false to prevent GA from pausing.
elif self.__pause_now:
self.__pause_now = False
def start(self):
"""
starts thread activity if start method was not called
before on this thread
"""
if not self.start_triggered:
self.start_triggered = True
self.paused = False
Thread.start(self)
else:
self.__resume()
def pause(self):
"""
pause thread if running
"""
# thread should be running to pause
if not self.paused:
self.__pause_now = True
# notify app of the pause
to_json({"paused": True})
def step_forward(self):
"""
move one iteration forward
"""
# start it if not started yet
if not self.start_triggered:
self.start_triggered = True
self.__pause_now = True
# fixes the blocking that happens when user clicks step_f multiple times on a heavy GA
self.paused = False
Thread.start(self)
return
# release if paused, it will lock automatically after one generation
# because __pause_now is set to True
elif not self.paused:
self.__pause_now = True
to_json({"paused": True})
elif not self.__pause_now:
# Notify so thread will wake after lock released
self.pause_cond.notify()
# Now release the lock
self.pause_cond.release()
# pause now
self.paused = False
self.__pause_now = True
def stop(self):
"""
if thread is alive, terminate it
"""
if self.is_alive():
self.__stop_now = True
# resume if paused to break out of running loop
if self.paused:
# Notify so thread will wake after lock released
self.pause_cond.notify()
# Now release the lock
self.pause_cond.release()
self.paused = False
# in case is going to pause but user pressed stop, GA should pass the pause check test to stop
else:
self.__pause_now = False
self.join(10)
class Explore(object):
def __init__(self, height):
self.motion_height = height # Height of the motion to the ground
# Create trajectory server
self.exploration_server = SimpleActionServer('assessment_server',
ExecuteAssesstAction,
self.exploreCallback,
False)
self.server_feedback = ExecuteAssesstFeedback()
self.server_result = ExecuteAssesstResult()
# Get client from trajectory server
self.trajectory_client = SimpleActionClient(
"approach_server", ExecuteDroneApproachAction)
self.trajectory_client.wait_for_server()
self.next_point = ExecuteDroneApproachGoal(
) # Message to define next position to look for victims
#Planning scene client
self.frontiers_client = rospy.ServiceProxy('frontiers_server/find',
Frontiers)
self.frontiers_client.wait_for_service()
self.frontiers_req = FrontiersRequest() #Frontiers request message
# Variables
self.sonar_me = Condition()
self.odometry_me = Condition()
self.current_height = None
self.odometry = None
# Subscribe to sonar_height
rospy.Subscriber("sonar_height",
Range,
self.sonar_callback,
queue_size=10)
# Subscribe to ground_truth to monitor the current pose of the robot
rospy.Subscriber("ground_truth/state", Odometry, self.poseCallback)
self.scene_req = GetPlanningSceneRequest()
# Start trajectory server
self.exploration_server.start()
def sonar_callback(self, msg):
'''
Function to update drone height
'''
self.sonar_me.acquire()
self.current_height = msg.range
self.sonar_me.release()
def poseCallback(self, odometry):
'''
Monitor the current position of the robot
'''
self.odometry_me.acquire()
self.odometry = odometry.pose.pose
self.odometry_me.notify()
self.odometry_me.release()
def trajectory_feed(self, msg):
'''
Verifies preemption requisitions
'''
print("\n\n\nASSESSMENT FEEDBACK")
if self.exploration_server.is_preempt_requested():
self.trajectory_client.cancel_goal()
def exploreCallback(self, pose):
'''
Execute a loop looking for frontiers and moving to points unvisited into the defined area
'''
# Wait till the robot pose is received
self.odometry_me.acquire()
while self.odometry == None:
self.odometry_me.wait()
self.next_point.goal = self.odometry
self.odometry_me.release()
self.frontiers_req.x_min = 0.0
self.frontiers_req.x_max = 50.0
self.frontiers_req.y_min = 0.0
self.frontiers_req.y_max = 50.0
trials = 0 # v_search trials
while not rospy.is_shutdown():
self.odometry_me.acquire()
self.server_result.last_pose = self.odometry
self.server_feedback.current_pose = self.odometry
self.odometry_me.release()
if self.exploration_server.is_preempt_requested():
self.exploration_server.set_preempted(self.server_result)
return
self.exploration_server.publish_feedback(self.server_feedback)
self.sonar_me.acquire()
# print("Current height from ground:\n\n{}".format(self.current_height)) # Current distance from ground
h_error = self.motion_height - self.current_height
self.sonar_me.release()
self.odometry_me.acquire()
self.next_point.goal.position.z = self.odometry.position.z + h_error # Desired z position
self.odometry_me.release()
self.trajectory_client.send_goal(self.next_point,
feedback_cb=self.trajectory_feed)
self.trajectory_client.wait_for_result() # Wait for the result
result = self.trajectory_client.get_state(
) # Get the state of the action
# print(result)
if result == GoalStatus.SUCCEEDED:
p = Pose()
self.odometry_me.acquire()
p = self.odometry
self.frontiers_req.explored.append(p)
self.odometry_me.release()
# Verify if all the area have been explored and find next frontier point if needed
# if 'all area explored':
# self.odometry_me.acquire()
# self.server_result.last_pose = self.odometry
# self.odometry_me.release()
# self.exploration_server.set_succeeded(self.server_result)
status = self.findFrontiers()
if not status:
self.exploration_server.set_succeeded(self.server_result)
return
self.odometry_me.acquire()
self.server_result.last_pose = self.odometry
self.odometry_me.release()
# self.next_point.goal.position.y =
# self.next_point.goal.position.x =
# theta =
# Convert desired angle
# q = quaternion_from_euler(0,0,theta,'ryxz')
# self.next_point.goal.orientation.x = q[0]
# self.next_point.goal.orientation.y = q[1]
# self.next_point.goal.orientation.z = q[2]
# self.next_point.goal.orientation.w = q[3]
elif result == GoalStatus.ABORTED:
trials += 1
if trials == 2:
self.exploration_server.set_aborted(self.server_result)
return
def findFrontiers(self):
'''
Return points not visited into the specified frontier
'''
rospy.loginfo("Looking for frontiers!")
frontiers = self.frontiers_client.call(self.frontiers_req)
if frontiers.frontiers:
self.next_point.goal.position.x = frontiers.frontiers[0].x
self.next_point.goal.position.y = frontiers.frontiers[0].y
self.next_point.goal.position.x = self.motion_height
return True
else:
return False
class SocketIORunnerClientNamespace(ClientNamespace):
def __init__(
self,
socketio_runner: SocketIORunner,
interactions: List[Interaction],
substitutes: dict = {},
):
super().__init__()
self.socketio_runner = socketio_runner
self._interaction_stack: List[Tuple[
bool, dict]] = _create_interaction_stack(
socketio_runner.interaction_loader, interactions, substitutes)
self._timeout_condition = Condition()
self._failed_interaction: Optional[FailedInteraction] = None
self._current_user_input: dict = {}
def session_request(self) -> None:
self.emit("session_request")
def on_bot_uttered(self, data: Any) -> None:
self._timeout_notify()
is_user_message, message = self._pop_interaction_stack()
while is_user_message:
self._send_user_input(message)
is_user_message, message = self._pop_interaction_stack()
json_diff = self.socketio_runner.comparator.compare(message, data)
if not json_diff.identical and self._failed_interaction is None:
self._failed_interaction = FailedInteraction(
self._current_user_input,
message,
data,
json_diff,
)
if self._next_is_user_message():
_, message = self._pop_interaction_stack()
self._send_user_input(message)
def _next_is_user_message(self) -> bool:
if len(self._interaction_stack) > 0:
is_user_message, _ = self._interaction_stack[0]
return is_user_message
return False
def _pop_interaction_stack(self) -> Tuple[bool, dict]:
return (self._interaction_stack.pop(0)
if len(self._interaction_stack) > 0 else (False, {}))
def _get_remaining_bot_messages(self) -> List[dict]:
return [
message for is_user_input, message in self._interaction_stack
if not is_user_input
]
def _timeout_notify(self):
with self._timeout_condition:
self._timeout_condition.notify()
def _timeout_await(self) -> bool:
with self._timeout_condition:
return self._timeout_condition.wait(BOT_RESPONSE_TIMEOUT)
def _send_user_input(self, message: dict) -> None:
self._timeout_notify()
self._current_user_input = {SESSION_ID_KEY: self.client.sid}
self._current_user_input.update(message)
self.emit(EVENT_USER_UTTERED, self._current_user_input)
def run(self) -> Optional[FailedInteraction]:
self.session_request()
is_user_input, message = self._pop_interaction_stack()
if is_user_input:
self._send_user_input(message)
while self._failed_interaction is None and self._timeout_await():
pass
remaining_messages = self._get_remaining_bot_messages()
json_diff = self.socketio_runner.comparator.compare({},
remaining_messages)
if not json_diff.identical and self._failed_interaction is None:
return FailedInteraction(
self._current_user_input,
{},
remaining_messages,
json_diff,
)
return self._failed_interaction
class MappingThread(Mapping):
def __init__(self, graph, params):
super().__init__(graph, params)
self._requests_cv = Condition()
self._requests = [False, False
] # requests: [LOCKWINDOW_REQUEST, PROCESS_REQUEST]
self._lock = Lock()
self.locked_window = set()
self.status = defaultdict(bool)
self._queue = Queue()
self.maintenance_thread = Thread(target=self.maintenance)
self.maintenance_thread.start()
def add_keyframe(self, keyframe, measurements):
self.graph.add_keyframe(keyframe)
self.create_points(keyframe)
for m in measurements:
self.graph.add_measurement(keyframe, m.mappoint, m)
self._queue.put(keyframe)
with self._requests_cv:
self._requests_cv.notify()
def maintenance(self):
stopped = False
while not stopped:
while not self._queue.empty():
keyframe = self._queue.get()
if keyframe is None:
stopped = True
self._requests[1] = True
break
else:
self.local_keyframes.append(keyframe)
if len(self.local_keyframes) >= 5:
self._requests[1] = True
break
with self._requests_cv:
if self._requests.count(True) == 0:
self._requests_cv.wait()
while not self._queue.empty():
keyframe = self._queue.get()
if keyframe is None:
stopped = True
self._requests[1] = True
break
else:
self.local_keyframes.append(keyframe)
if len(self.local_keyframes) >= 5:
self._requests[1] = True
requests = self._requests[:]
self._requests[0] = False
self._requests[1] = False
self.status['processing'] = True
if requests[1] and len(self.local_keyframes) > 0:
self.fill(self.local_keyframes, self.local_keyframes[-1])
if requests[0]:
with self._lock:
for kf in self.local_keyframes:
self.locked_window.add(kf)
for ck, n in kf.covisibility_keyframes().items():
if n > 0:
self.locked_window.add(ck)
self.status['window_locked'] = True
if requests[1] and len(self.local_keyframes) > 0:
completed = self.bundle_adjust(self.local_keyframes)
if completed:
self.points_culling(self.local_keyframes)
self.local_keyframes.clear()
self.status['processing'] = False
def stop(self):
with self._requests_cv:
self._requests_cv.notify()
while not self._queue.empty():
time.sleep(1e-4)
self._queue.put(None) # sentinel value
self.maintenance_thread.join()
print('mapping stopped')
def is_safe(self, keyframe):
with self._lock:
return not self.is_window_locked(
) or keyframe in self.locked_window
def is_processing(self):
return self.status['processing']
def lock_window(self):
with self._lock:
self.status['window_locked'] = False
self.locked_window.clear()
with self._requests_cv:
self._requests[0] = True
self._requests_cv.notify()
while not self.is_window_locked():
time.sleep(1e-4)
return self.locked_window
def free_window(self):
with self._lock:
self.status['window_locked'] = False
self.locked_window.clear()
def is_window_locked(self):
return self.status['window_locked']
def wait_until_empty_queue(self):
while not self._queue.empty():
time.sleep(1e-4)
def interrupt_ba(self):
self.optimizer.abort()
class Popen(object):
_next_id = 0
_scheduler = None
_redirector = None
_lock = RLock()
def __init__(self,
args,
bufsize=0,
executable=None,
stdin=None,
stdout=None,
stderr=None,
preexec_fn=None,
close_fds=None,
shell=False,
cwd=None,
env=None,
universal_newlines=False,
startupinfo=None,
creationflags=0,
cpus=None,
mem=None,
gpus=None):
kw = dict(list(locals().items()))
a = (args, )
kw.pop('self')
kw.pop('args')
close_fds = kw.pop('close_fds')
if close_fds is False:
logger.warning('Can not support `close_fds=False`, '
'no fds will be inherrited.')
stdin = kw.pop('stdin', None)
stdout = kw.pop('stdout', None)
stderr = kw.pop('stderr', None)
cpus = kw.pop('cpus', None)
mem = kw.pop('mem', None)
gpus = kw.pop('gpus', None)
kw['cwd'] = kw.get('cwd') or os.getcwd()
kw['env'] = kw.get('env') or dict(list(os.environ.items()))
self.id = self._new_id()
self.cpus = float(cpus or CONFIG.get('default_cpus', 1.0))
self.mem = float(mem or CONFIG.get('default_mem', 1024.0))
self.gpus = int(gpus or CONFIG.get('default_gpus', 0))
self.pid = None
self.returncode = None
self._returncode = None
self._a = a
self._kw = kw
self._exc = None
self._state = _STARTING
self._io_waiting = True
self._cond = Condition()
self._prepare_handlers(stdin, stdout, stderr)
self._submit()
with self._cond:
deadline = time.time() + _STARTING_TIMEOUT
while True:
if self._state != _STARTING:
break
delta = deadline - time.time()
if deadline <= 0:
raise RuntimeError('Too long to start!')
self._cond.wait(delta)
if self._exc:
raise self._exc
@classmethod
def _new_id(cls):
cls._next_id += 1
return cls._next_id
def _submit(self):
cls = self.__class__
with cls._lock:
if not cls._scheduler:
cls._scheduler = ProcScheduler()
cls._scheduler.start()
atexit.register(cls._scheduler.stop)
cls._scheduler.submit(self)
def _prepare_handlers(self, stdin, stdout, stderr):
def _dup_file(f, *a, **kw):
fd = f if isinstance(f, int) else f.fileno()
return os.fdopen(os.dup(fd), *a, **kw)
cls = self.__class__
if not cls._redirector:
cls._redirector = Redirector()
atexit.register(cls._redirector.stop)
if stdin == PIPE:
r, w = os.pipe()
_in = os.fdopen(r, 'rb', 0)
self.stdin = os.fdopen(w, 'wb', 0)
else:
self.stdin = None
if stdin is None:
_in = _dup_file(sys.stdin, 'rb', 0)
else:
_in = _dup_file(stdin, 'rb', 0)
if stdout == PIPE:
r, w = os.pipe()
_out = os.fdopen(w, 'wb', 0)
self.stdout = os.fdopen(r, 'rb', 0)
else:
self.stdout = None
if stdout is None:
_out = _dup_file(sys.stdout, 'wb', 0)
else:
_out = _dup_file(stdout, 'wb', 0)
if stderr == PIPE:
r, w = os.pipe()
_err = os.fdopen(w, 'wb', 0)
self.stderr = os.fdopen(r, 'rb', 0)
else:
self.stderr = None
if stderr is None:
_err = _dup_file(sys.stderr, 'wb', 0)
elif stderr == STDOUT:
_err = _dup_file(_out, 'wb', 0)
else:
_err = _dup_file(sys.stderr, 'wb', 0)
self._handlers = cls._redirector.register(self.id, _in, _out, _err,
self._io_complete)
def _io_complete(self):
with self._cond:
self._io_waiting = False
self._cond.notify()
def _kill(self):
cls = self.__class__
cls._redirector.unregister(self.id)
def __repr__(self):
args = self._a[0]
if isinstance(args, string_types):
cmd = args
else:
cmd = ' '.join(args)
return '%s: %s' % (self.__class__.__name__, cmd)
@property
def params(self):
return dict(
a=self._a,
kw=self._kw,
cpus=self.cpus,
mem=self.mem,
gpus=self.gpus,
handlers=self._handlers,
hostname=socket.gethostname(),
)
def _started(self):
logger.info('Started')
with self._cond:
self._state = _RUNNING
self._cond.notify()
def _finished(self, success, message, data):
logger.info('Sucess:%s message:%s', success, message)
if success:
self._returncode, self._exc = data
else:
self._returncode, self._exc = data or (UNKNOWN_ERROR, None)
self._kill()
with self._cond:
self._state = _STOPPED
self._cond.notify()
def poll(self):
with self._cond:
if self._state == _STOPPED and not self._io_waiting:
if self.stdin and not self.stdin.closed:
self.stdin.close()
self.stdin = None
if self.stdout and not self.stdout.closed:
self.stdout.close()
self.stdout = None
if self.stderr and not self.stderr.closed:
self.stderr.close()
self.stderr = None
self.returncode = self._returncode
return self.returncode
def wait(self):
with self._cond:
while self.poll() is None:
self._cond.wait()
return self.returncode
def communicate(self, input=None):
BUFFER_SIZE = PIPE_BUF
buf = input and input[:]
out = None
err = None
to_write = [_f for _f in [self.stdin] if _f]
to_read = [_f for _f in [self.stdout, self.stderr] if _f]
while True:
can_wait = True
readable, writeable, _ = select.select(to_read, to_write, [], 0)
if writeable:
if buf:
can_wait = False
size = os.write(self.stdin.fileno(), buf[:BUFFER_SIZE])
buf = buf[size:]
else:
if self.stdin and not self.stdin.closed:
self.stdin.close()
to_write = []
if readable:
can_wait = False
if self.stdout in readable:
if out is None:
out = self.stdout.read(BUFFER_SIZE)
else:
out += self.stdout.read(BUFFER_SIZE)
if self.stderr in readable:
if err is None:
err = self.stderr.read(BUFFER_SIZE)
else:
err += self.stderr.read(BUFFER_SIZE)
with self._cond:
if self.poll() is None:
if can_wait:
self._cond.wait(0.1)
else:
return (out, err)
def send_signal(self, signal):
cls = self.__class__
cls._scheduler.send_data(self.id, _TYPE_SIGNAL, signal)
def terminate(self):
self.send_signal(signal.SIGTERM)
def kill(self):
self.send_signal(signal.SIGKILL)
def cancel(self):
cls = self.__class__
cls._scheduler.cancel(self)
self._kill()
self.returncode = UNKNOWN_ERROR
class RPC:
def __init__(self, infile, outfile, onRequest, onNotification, onError):
self.infile = infile
self.outfile = outfile
self.onRequest = onRequest
self.onNotification = onNotification
self.onError = onError
self.mid = 0
self.queue = {}
self.cv = Condition()
self.result = None
def incMid(self) -> int:
mid = self.mid
self.mid += 1
return mid
def message(self, contentDict: Dict[str, Any]) -> None:
content = json.dumps(contentDict)
message = ("Content-Length: {}\r\n\r\n"
"{}".format(len(content), content))
logger.debug(' => ' + content)
self.outfile.write(message)
self.outfile.flush()
def call(self, method: str, params: Dict[str, Any], cb=None):
"""
@param cb: func. Callback to handle result. If None, turn to sync call.
"""
mid = self.incMid()
if cb is not None:
self.queue[mid] = cb
contentDict = {
"jsonrpc": "2.0",
"method": method,
"params": params,
"id": mid,
} # type: Dict[str, Any]
self.message(contentDict)
if cb is not None:
return
with self.cv:
if not self.cv.wait_for(lambda: self.result is not None, 3):
return None
result = self.result
self.result = None
return result
def notify(self, method: str, params: Dict[str, Any]) -> None:
contentDict = {
"jsonrpc": "2.0",
"method": method,
"params": params,
} # type: Dict[str, Any]
self.message(contentDict)
def serve(self):
contentLength = 0
while not self.infile.closed:
try:
line = self.infile.readline().strip()
if line:
header, value = line.split(":")
if header == "Content-Length":
contentLength = int(value)
else:
content = self.infile.read(contentLength)
logger.debug(' <= ' + content)
self.handle(json.loads(content))
except Exception as ex:
msg = "Error handling server output."
self.onError(msg)
logger.exception(msg)
break
def handle(self, message: Dict[str, Any]):
if "error" in message: # error
if "id" in message:
mid = message["id"]
del self.queue[mid]
self.onError(message["error"])
elif "result" in message: # result
mid = message["id"]
if isinstance(mid, str):
mid = int(mid)
result = message["result"]
if mid in self.queue: # async call
cb = self.queue[mid]
del self.queue[mid]
cb(result)
else: # sync call
with self.cv:
self.result = result
self.cv.notify()
elif "method" in message: # request/notification
if "id" in message: # request
self.onRequest(message)
else:
self.onNotification(message)
else:
logger.error('Unknown message.')
class Bouncer:
def __init__(self, max_capacity):
self.bouncerLock = Lock()
self.totalGreeksCapacity = max_capacity
self.totalGreeksInside = 0
self.nkegsInside = 0
self.ndogsInside = 0
self.ngirlsInside = 0
self.nkegsWaiting = 0
self.ndogsWaiting = 0
self.ngirlssWaiting = 0
self.kegsCond = Condition(self.bouncerLock)
self.dogsCond = Condition(self.bouncerLock)
self.girlsCond = Condition(self.bouncerLock)
def keg_enter(self):
with self.bouncerLock:
while (self.totalGreeksInside >= self.totalGreeksCapacity) \
or (self.totalGreeksInside > 0 and ((1.0*self.ndogsInside)/self.totalGreeksInside) > .75):
self.nkegsWaiting = self.nkegsWaiting + 1
self.kegsCond.wait()
self.nkegsWaiting = self.nkegsWaiting - 1
self.nkegsInside = self.nkegsInside + 1
self.totalGreeksInside = self.totalGreeksInside + 1
def keg_exit(self):
with self.bouncerLock:
self.nkegsInside = self.nkegsInside - 1
self.totalGreeksInside = self.totalGreeksInside - 1
if self.ndogsWaiting > 0 and (self.totalGreeksInside==0 \
or(((1.0*self.ndogsInside)/self.totalGreeksInside) > .75)):
self.dogsCond.notify()
elif self.ngirlssWaiting > 0:
self.girlsCond.notify()
elif self.nkegsWaiting > 0:
self.kegsCond.notify()
def dog_enter(self):
with self.bouncerLock:
while (self.totalGreeksInside >= self.totalGreeksCapacity) \
or (self.totalGreeksInside>0 and ((1.0*self.nkegsInside)/self.totalGreeksInside) > .50):
self.ndogsWaiting = self.ndogsWaiting + 1
self.dogsCond.wait()
self.ndogsWaiting = self.ndogsWaiting - 1
self.ndogsInside = self.ndogsInside + 1
self.totalGreeksInside = self.totalGreeksInside + 1
def dog_exit(self):
with self.bouncerLock:
self.ndogsInside = self.ndogsInside - 1
self.totalGreeksInside = self.totalGreeksInside - 1
if self.nkegsWaiting > 0 and (self.totalGreeksInside==0 \
or (((1.0*self.ndogsInside)/self.totalGreeksInside) > .75)):
self.kegsCond.notify()
elif self.ngirlssWaiting > 0:
self.girlsCond.notify()
elif self.ndogsWaiting > 0:
self.dogsCond.notify()
def girl_enter(self):
with self.bouncerLock:
while (self.totalGreeksInside >= self.totalGreeksCapacity) \
or (((1.0*self.totalGreeksInside)/self.totalGreeksCapacity)<.10):
self.ngirlssWaiting = self.ngirlssWaiting + 1
self.girlsCond.wait()
self.ngirlssWaiting = self.ngirlssWaiting - 1
self.ngirlsInside = self.ngirlsInside + 1
self.totalGreeksInside = self.totalGreeksInside + 1
def girl_exit(self):
with self.bouncerLock:
self.ngirlsInside = self.ngirlsInside - 1
self.totalGreeksInside = self.totalGreeksInside - 1
if self.nkegsWaiting > 0:
self.kegsCond.notify()
elif self.ndogsWaiting > 0:
self.dogsCond.notify()
elif self.ngirlssWaiting > 0:
self.girlsCond.notify()
class EVNotify:
""" Interface to EVNotify. """
def __init__(self, config, car):
self._log = logging.getLogger("EVNotiPi/EVNotify")
self._log.info("Initializing EVNotify")
self._car = car
self._config = config
self._poll_interval = config['interval']
self._running = False
self._thread = None
self._data = []
self._gps_data = []
self._data_lock = Condition()
def start(self):
""" Start submit thread. """
self._running = True
self._thread = Thread(target=self.submit_data,
name="EVNotiPi/EVNotify")
self._thread.start()
self._car.register_data(self.data_callback)
def stop(self):
""" Stop submit thread. """
self._car.unregister_data(self.data_callback)
self._running = False
with self._data_lock:
self._data_lock.notify()
self._thread.join()
def data_callback(self, data):
""" Callback to be called from 'car'. """
self._log.debug("Enqeue...")
with self._data_lock:
self._data.append(data)
self._data_lock.notify()
def submit_data(self):
""" Thread that submits data to EVNotify in regular intervals. """
log = self._log
evn = EVNotifyAPI.EVNotify(self._config['akey'], self._config['token'])
abort_notification = ARMED
charging_start_soc = 0
last_charging = time()
last_charging_soc = 0
last_evn_settings_poll = 0
is_charging = 0
is_connected = 0
settings = None
soc_notification = ARMED
soc_threshold = self._config.get('soc_threshold', 100)
log.info("Get settings from backend")
while self._running and settings is None:
try:
settings = evn.getSettings()
except EVNotifyAPI.CommunicationError as err:
log.info("Waiting for network connectivity (%s)", err)
sleep(3)
while self._running:
with self._data_lock:
log.debug('Waiting...')
self._data_lock.wait(max(10, self._poll_interval))
now = time()
# Detect aborted charge
if ((now - last_charging > ABORT_NOTIFICATION_INTERVAL
and charging_start_soc > 0
and 0 < last_charging_soc < soc_threshold
and abort_notification is ARMED)
or abort_notification is PENDING):
log.info(
"Aborted charge detected, send abort notification now-last_charging(%i) charging_start_soc(%i) last_charging_soc(%i) soc_threshold(%i) abort_notification(%i)",
now - last_charging, charging_start_soc,
last_charging_soc, soc_threshold, abort_notification)
try:
evn.sendNotification(True)
abort_notification = SENT
except EVNotifyAPI.CommunicationError as err:
log.error("Communication Error: %s", err)
abort_notification = PENDING
if len(self._data) == 0:
continue
new_data = self._data.copy()
self._data.clear()
log.debug("Transmit...")
avgs = {
'dcBatteryCurrent': [],
'dcBatteryPower': [],
'dcBatteryVoltage': [],
'speed': [],
'latitude': [],
'longitude': [],
'altitude': [],
}
for data in new_data:
for key, values in avgs.items():
if data.get(key, None) is not None:
values.append(data[key])
# Need to copy data here because we update it later
data = new_data[-1]
data.update(
{k: sum(v) / len(v)
for k, v in avgs.items() if len(v) > 0})
try:
if (data['SOC_DISPLAY'] is not None
or data['SOC_BMS'] is not None):
current_soc = data['SOC_DISPLAY'] or data['SOC_BMS']
is_charging = bool(data['charging'])
is_connected = bool(data['normalChargePort']
or data['rapidChargePort'])
if is_charging:
last_charging = now
last_charging_soc = current_soc
evn.setSOC(data['SOC_DISPLAY'], data['SOC_BMS'])
extended_data = {
a: round(data[a], EXTENDED_FIELDS[a])
for a in EXTENDED_FIELDS if data[a] is not None
}
log.debug(extended_data)
evn.setExtended(extended_data)
if data['fix_mode'] > 1 and not is_charging and not is_connected:
location = {
a: data[a]
for a in ('latitude', 'longitude', 'speed')
}
evn.setLocation({'location': location})
# Notification handling from here on
if is_charging and now - last_evn_settings_poll > EVN_SETTINGS_INTERVAL:
try:
settings = evn.getSettings()
last_evn_settings_poll = now
if 'soc' in settings:
new_soc = int(settings['soc'])
if new_soc != soc_threshold:
soc_threshold = new_soc
log.info("New notification threshold: %i",
soc_threshold)
except EVNotifyAPI.CommunicationError as err:
log.error("Communication error occured %s", err)
# track charging started
if is_charging and charging_start_soc == 0:
charging_start_soc = current_soc or 0
elif not is_connected: # Rearm abort notification
charging_start_soc = 0
abort_notification = ARMED
# SoC threshold notification
if ((is_charging
and 0 < last_charging_soc < soc_threshold <= current_soc)
or soc_notification is PENDING):
log.info("Notification threshold(%i) reached: %i",
soc_threshold, current_soc)
try:
evn.sendNotification()
soc_notification = ARMED
except EVNotifyAPI.CommunicationError as err:
log.info("Communication Error: %s", err)
soc_notification = PENDING
except EVNotifyAPI.CommunicationError as err:
log.info("Communication Error: %s", err)
# Prime next loop iteration
if self._running:
interval = self._poll_interval - (time() - now)
sleep(max(0, interval))
def check_thread(self):
""" Return running state of thread. """
return self._thread.is_alive()
class SrSender:
def __init__(self, addr):
self.__sock = socket(AF_INET, SOCK_DGRAM)
self.__sock.connect(addr)
self.__cache = []
self.__timer = []
self.__timer_lock = []
self.__wait_send = []
self.__win_size = 10
self.__interval = 4.
self.__notify_sender = Condition()
# 用于控制对缓存的访问
self.__lock = Lock()
self.__seq = 0
self.__min_seq = 0
self.__SEQ_SIZE = 255
t = Thread(target=self.__ack_thread)
t.start()
t = Thread(target=self.__send_thread)
t.start()
def __get_seq(self):
self.__seq += 1
if self.__seq == self.__SEQ_SIZE:
self.__seq = 0
return self.__seq
def __find_index(self, seq):
"""
通过seq查找index,在查找过程中会获得缓存的锁。
:param seq: seq
:return: 如果查找到了返回True，否则返回False
"""
self.__lock.acquire()
for i in range(len(self.__cache)):
if self.__cache[i][1] == seq:
self.__lock.release()
return i
self.__lock.release()
return None
def __send_thread(self):
"""
包发送线程，被发送方法send唤醒之后发送待发送缓存中数据（不一定是所有），并将数据发乳到窗口中。
:return:
"""
self.__notify_sender.acquire()
while True:
while len(self.__wait_send) == 0:
self.__notify_sender.wait()
num_send = min(len(self.__wait_send), self.__win_size - len(self.__cache))
for i in range(num_send):
# 构造分组
seq = self.__get_seq()
data = bytes([0, seq]) + len(self.__wait_send[i]).to_bytes(4, 'big') + self.__wait_send[i]
# 计算校验和
data += (lib.checksum(data) ^ 0xffff).to_bytes(2, 'big')
assert lib.checksum(data) == 0xffff
self.__sock.send(data)
self.__lock.acquire()
self.__cache.append(data)
t = Timer(4, self.__timer_thread, args=(seq,))
self.__timer.append(t)
self.__timer_lock.append(Lock())
self.__lock.release()
t.start()
for i in range(num_send):
self.__wait_send.pop(0)
def send(self, data):
"""
向用户提供的API接口，用于发送数据
:param data: 需要发送的数据，应该是bytes类型的；
:return:
"""
if len(data) % 2 == 1:
data += b'\x00'
self.__wait_send.append(data)
self.__notify_sender.acquire()
self.__notify_sender.notify()
self.__notify_sender.release()
def __timer_thread(self, seq):
"""
每一个数据包的定时器线程,在函数执行的过程中会获取数据包的锁。
:param seq: 数据包的seq
:return: 无
"""
index = self.__find_index(seq)
if index is None:
return
lock = self.__timer_lock[index]
lock.acquire()
self.__sock.send(self.__cache[index])
timer = Timer(self.__interval, self.__timer_thread, args=(seq, ))
timer.start()
self.__timer[index] = timer
lock.release()
def __ack_thread(self):
while True:
data = self.__sock.recv(1024)
seq = data[1]
index = self.__find_index(seq)
if index is None:
continue
lock = self.__timer_lock[index]
lock.acquire()
timer = self.__timer[index]
# timer 是 None 说明其ack在之前已经收到了
if timer is None:
continue
timer.cancel()
self.__timer[index] = None
lock.release()
self.__lock.acquire()
while len(self.__timer) > 0 and self.__timer[0] is None:
self.__timer.pop(0)
self.__cache.pop(0)
self.__timer_lock.pop(0)
self.__lock.release()
class worker(object):
"""
serialize main thread load/unload of PLC shared objects
"""
def __init__(self):
# Only one job at a time
self._finish = False
self._threadID = None
self.mutex = Lock()
self.todo = Condition(self.mutex)
self.done = Condition(self.mutex)
self.free = Condition(self.mutex)
self.job = None
self.enabled = False
def reraise(self, job):
"""
reraise exception happend in a job
@param job: job where original exception happend
"""
exc_type = job.exc_info[0]
exc_value = job.exc_info[1]
exc_traceback = job.exc_info[2]
six.reraise(exc_type, exc_value, exc_traceback)
def runloop(self, *args, **kwargs):
"""
meant to be called by worker thread (blocking)
"""
self._threadID = _thread.get_ident()
self.mutex.acquire()
self.enabled = True
if args or kwargs:
_job = job(*args, **kwargs)
_job.do()
# _job.success can't be None after do()
if not _job.success:
self.reraise(_job)
while not self._finish:
self.todo.wait()
if self.job is not None:
self.job.do()
self.done.notify()
else:
break
self.mutex.release()
def call(self, *args, **kwargs):
"""
creates a job, execute it in worker thread, and deliver result.
if job execution raise exception, re-raise same exception
meant to be called by non-worker threads, but this is accepted.
blocking until job done
"""
_job = job(*args, **kwargs)
if self._threadID == _thread.get_ident():
# if caller is worker thread execute immediately
_job.do()
else:
# otherwise notify and wait for completion
self.mutex.acquire()
if not self.enabled:
self.mutex.release()
raise EOFError("Worker is disabled")
while self.job is not None:
self.free.wait()
self.job = _job
self.todo.notify()
self.done.wait()
self.job = None
self.free.notify()
self.mutex.release()
if _job.success is None:
raise EOFError("Worker job was interrupted")
if _job.success:
return _job.result
else:
self.reraise(_job)
def quit(self):
"""
unblocks main thread, and terminate execution of runloop()
"""
# mark queue
self._finish = True
self.mutex.acquire()
self.enabled = False
self.job = None
self.todo.notify()
self.done.notify()
self.mutex.release()
class RndUploader:
#Esta clase tiene un único atributo que es enable y
#se inicializa como True. Miestras tenga este valor
#se seguiran subiendo datos cada 2 minutos.
#cuando se cambie su valor a Flase parara.
def __init__(self, flaskApp, handSQL=None, handBee=None, handMongo=None,\
tiempoSleep = 120, debug = False,handSSE=None):
#self.__enable -> __enable es privado gracias a '__'
#para acceder al valor de enable utilizaremos
#self.__enable.value
self.__enable = Value('b', True)
#modo debug
self.__debug = True
#tiempo a esperar entre inserciones
self.__tiempo = tiempoSleep
#Creo instancias PRIVADAS de las clases a utilizar
#para obtener los numeros aleatorios
self.__RndGen = web_fetcher.rnd_fetcher.Rnd_fetcher()
#manejar BBDD
#self.__SQLHand = sql_rnd.SQLHandler(flaskApp)
#self.__BeeHand = beebotte_rnd.BeeHandler()
if handSQL:
self.__SQLHand = handSQL
else:
logging.info("SQLHandler - Creando instancia Propia")
self.__SQLHand = sql_rnd.SQLHandler(flaskApp)
if handBee:
self.__BeeHand = handBee
else:
logging.info("BeeHandler - Creando instancia Propia")
self.__BeeHand = beebotte_rnd.BeeHandler()
#MongoClient opened before fork. Create MongoClient only after forking.
#See PyMongo's documentation for details:
#http://api.mongodb.org/python/current/faq.html#is-pymongo-fork-safe
#No es seguro que esta clase reciba la instancia de MongoDB del main
#Por lo que creare en esta clase mi propia instancia de MongoHandler
#Esto no debería importar ya que aunque sean dos instancias distintas
#leerán de la misma base de datos.
#La instancia en esta clase se dedicará principalmente a escribir y la
#que está en el main a leer.
#
#Debo crearla una vez dentro del subproceso creado, es decir,
#dentro de la función upload que es la que ejecuta el proceso.
#UPDATE: ya no es necesario, ya que ahora rnd_uploader crea un hilo
#en vez de un proceso, por lo que comparten memoria y no hace fork()
#de esta instancia recibida de MongoHandler.
#self.__MongoHand= handMongo
if handMongo:
self.__MongoHand = handMongo
else:
logging.info("MongoHandler - Creando instancia Propia")
self.__MongoHand = mongo_rnd.MongoHandler()
#self.__MongoHand= mongo_rnd.MongoHandler()
#Manejador de SSE
self.__SSEHand = handSSE
if handSSE:
self.__SSEHand = handSSE
else:
logging.info("SSEHandler - Creando instancia Propia")
self.__SSEHand = SSEHandler()
#inicio proceso para subir los datos a las BBDD
self.lanzar()
#Devuelve el manejador de MySQL
def getSQLHandler(self):
return self.__SQLHand
#Devuelve el manejador de Beebotte
def getBeeHandler(self):
return self.__BeeHand
#Devuelve el manejador de MongoDB
def getMongoHandler(self):
return self.__MongoHand
#Subira un número aleatorio cada 2 min
def upload(self):
#MongoClient opened before fork. Create MongoClient only after forking.
#See PyMongo's documentation for details:
#http://api.mongodb.org/python/current/faq.html#is-pymongo-fork-safe
#No es seguro que esta clase reciba la instancia de MongoDB del main
#Por lo que creare en esta clase mi propia instancia de MongoHandler
#Esto no debería importar ya que aunque sean dos instancias distintas
#leerán de la misma base de datos.
#La instancia en esta clase se dedicará principalmente a escribir y la
#que está en el main a leer.
#
#Debo crearla una vez dentro del subproceso creado, es decir,
#dentro de la función upload que es la que ejecuta el proceso.
#UPDATE: Ya no es necesario, porque la funcion upload() se
#lanza mediante un hilo y no un proceso, por lo que ya no
#se realiza ningún fork() que es sobre lo que va este aviso.
#self.__MongoHand= mongo_rnd.MongoHandler()
#Obtengo la condición de forma que este hilo pueda utlizar
#cond.wait() para esperar.
self.cond.acquire()
while self.__enable.value:
#obtenemos numero aleatorio a insertar
rnd = self.__RndGen.get_web_rnd()
if self.__debug:
logging.debug("num aleatorio a escribir: " + str(rnd))
"""
#BORRA ESTO!-----------------------------
#Este trozo de codigo sirve para que esta
#clase no suba numeros.
logging.warning("SUBIDA DE NUMEROS DESACTIVADA!!!")
self.__enable.value = False
self.__SQLHand.readDataDB()
self.__BeeHand.readRandom()
self.__MongoHand.readRandom()
#BORRA ESTO!-----------------------------
"""
"""
if self.__debug:
logging.debug("rnd_uploader - Las listas en rnd_uploader: ")
logging.debug("BeeHandler : " + str(self.__BeeHand.listaGlobalNumero))
logging.debug("SQLHandler : " + str(self.__SQLHand.listaGlobalNumero))
logging.debug("MongoHandler : " + str(self.__MongoHand.listaGlobalNumero))
"""
#Escribir
if (self.__enable.value):
#La funcion de obtener numeros aleatorios
#devuelve -1 en caso de no haber podido conectar
#con la web de donde obtenemos los numeros aleatorios.
#En este caso, rnd sera = a -1, por lo que no guardaremos
#este valor en las BDs.
if rnd > -1:
#escribo en Beebotte. 0 si bien. 1 si mal.
resBee = self.__BeeHand.writeRandom(rnd, self.__debug)
#solo necesario para la BD local, ya que Beebotte
#almacena automaticamente la fecha
fecha = str(date_handler.getDatetimeMs())
#escribo en MongoDB. 0 si bien. 1 si mal.
resMongo = self.__MongoHand.writeRandom(rnd, fecha)
#escribo en MySQL. 0 si bien. 1 si mal.
resSQL = self.__SQLHand.writeDataDB(
rnd, fecha, self.__debug)
#---
#envio SSE (notificación a los clientes con
#el número obtenido)
#FORMATO SSE (parseado mediante js en el cliente)
#NUM,FECHA#BD1,BD2,BD3,
msg = str(rnd) + "," + str(fecha) + "#"
if resBee == 0:
msg += "Beebotte,"
if resMongo == 0:
msg += "MongoDB,"
if resSQL == 0:
msg += "MySQL,"
res = self.__SSEHand.createSSE(str(msg))
#if self.__debug:
if True:
logging.info("ENVIANDO SSE: " + str(msg))
logging.debug(res)
"""
#ACTUALIZO LOS DATOS EN LAS LISTAS LOCALES
#DE LOS MANEJADORES
self.__SQLHand.readDataDB()
self.__BeeHand.readRandom()
self.__MongoHand.readRandom()
"""
"""
if self.__debug:
logging.debug("Tablas MySQL:")
logging.debug(self.__SQLHand.listaGlobalFecha)
logging.debug(self.__SQLHand.listaGlobalNumero)
logging.debug("Tablas Bee:")
logging.debug(self.__BeeHand.listaGlobalFecha)
logging.debug(self.__BeeHand.listaGlobalNumero)
logging.debug("Tablas Mongo:")
logging.debug(self.__MongoHand.listaGlobalFecha)
logging.debug(self.__MongoHand.listaGlobalNumero)
"""
#esperar entre escrituras
try:
#time.sleep(self.__tiempo)
#Utilizo la condicion para esperar
#el tiempo entre escrituras.
#Si la funcion finalizar() llama
#al método cond.notify() mientras
#hago wait(), la espera se
#interrumpe, al contrario que con
#time.sleep.
self.cond.wait(self.__tiempo)
except:
if self.__debug:
logging.debug("Uploader.upload(): sleep interrumpido!")
#Libero la condicion antes de salir.
self.cond.release()
logging.info("Uploader.upload(): saliendo...")
#Función que genera un proceso que ejecuta la función
#upload() de esta misma clase en segundo plano.
def lanzar(self):
#proceso que será el uploader.
#si solo pasamos un parametro como argumento (args),
#tendremos que poner una coma detras de el, que es la
#forma de decir que es una tupla de un solo elemento.
#Sin esta coma (',') la creación del proceso falla
#self.proceso = Process(target=self.upload, args=(self.__debug,) )
#self.proceso = Process(target=self.upload)
#Lo utilizara el proceso para esperar en vez de time.sleep()
self.cond = Condition()
self.proceso = Thread(target=self.upload)
#inicio proceso
logging.info("Soy un hilo (RndUploader.upload()).")
self.proceso.start()
#Marca y espera que los procesos en segundo plano terminen su ejecución.
def finalizar(self):
estavivo = self.proceso.isAlive()
logging.info("PROCESO VIVO: " + str(estavivo))
if estavivo:
logging.debug("PARANDO")
#Obtengo la condicion de forma que
#pueda utilizar el método notify()
self.cond.acquire()
#Hago notify de forma que si el hilo
#de lanzar() esta esperando con wait(),
#interrumpa su espera para terminar
self.cond.notify()
#Libero la condicion
self.cond.release()
self.__enable.value = False
if self.__debug:
logging.info("Bandera activada para finalizar: " +
str(self.__enable.value))
#Si el proceso no esta vivo no espero
#pues bloquerá el programa
if estavivo:
logging.info("Esperando para acabar - thread.join()")
self.proceso.join()
if self.__debug:
logging.info("el proceso ya ha acabado")
class HostConnectionPool(object):
"""
Used to pool connections to a host for v1 and v2 native protocol.
"""
host = None
host_distance = None
is_shutdown = False
open_count = 0
_scheduled_for_creation = 0
_next_trash_allowed_at = 0
_keyspace = None
def __init__(self, host, host_distance, session):
self.host = host
self.host_distance = host_distance
self._session = weakref.proxy(session)
self._lock = RLock()
self._conn_available_condition = Condition()
log.debug("Initializing new connection pool for host %s", self.host)
core_conns = session.cluster.get_core_connections_per_host(
host_distance)
self._connections = [
session.cluster.connection_factory(host.endpoint)
for i in range(core_conns)
]
self._keyspace = session.keyspace
if self._keyspace:
for conn in self._connections:
conn.set_keyspace_blocking(self._keyspace)
self._trash = set()
self._next_trash_allowed_at = time.time()
self.open_count = core_conns
log.debug("Finished initializing new connection pool for host %s",
self.host)
def borrow_connection(self, timeout):
if self.is_shutdown:
raise ConnectionException(
"Pool for %s is shutdown" % (self.host, ), self.host)
conns = self._connections
if not conns:
# handled specially just for simpler code
log.debug("Detected empty pool, opening core conns to %s",
self.host)
core_conns = self._session.cluster.get_core_connections_per_host(
self.host_distance)
with self._lock:
# we check the length of self._connections again
# along with self._scheduled_for_creation while holding the lock
# in case multiple threads hit this condition at the same time
to_create = core_conns - (len(self._connections) +
self._scheduled_for_creation)
for i in range(to_create):
self._scheduled_for_creation += 1
self._session.submit(self._create_new_connection)
# in_flight is incremented by wait_for_conn
conn = self._wait_for_conn(timeout)
return conn
else:
# note: it would be nice to push changes to these config settings
# to pools instead of doing a new lookup on every
# borrow_connection() call
max_reqs = self._session.cluster.get_max_requests_per_connection(
self.host_distance)
max_conns = self._session.cluster.get_max_connections_per_host(
self.host_distance)
least_busy = min(conns, key=lambda c: c.in_flight)
request_id = None
# to avoid another thread closing this connection while
# trashing it (through the return_connection process), hold
# the connection lock from this point until we've incremented
# its in_flight count
need_to_wait = False
with least_busy.lock:
if least_busy.in_flight < least_busy.max_request_id:
least_busy.in_flight += 1
request_id = least_busy.get_request_id()
else:
# once we release the lock, wait for another connection
need_to_wait = True
if need_to_wait:
# wait_for_conn will increment in_flight on the conn
least_busy, request_id = self._wait_for_conn(timeout)
# if we have too many requests on this connection but we still
# have space to open a new connection against this host, go ahead
# and schedule the creation of a new connection
if least_busy.in_flight >= max_reqs and len(
self._connections) < max_conns:
self._maybe_spawn_new_connection()
return least_busy, request_id
def _maybe_spawn_new_connection(self):
with self._lock:
if self._scheduled_for_creation >= _MAX_SIMULTANEOUS_CREATION:
return
if self.open_count >= self._session.cluster.get_max_connections_per_host(
self.host_distance):
return
self._scheduled_for_creation += 1
log.debug("Submitting task for creation of new Connection to %s",
self.host)
self._session.submit(self._create_new_connection)
def _create_new_connection(self):
try:
self._add_conn_if_under_max()
except (ConnectionException, socket.error) as exc:
log.warning("Failed to create new connection to %s: %s", self.host,
exc)
except Exception:
log.exception("Unexpectedly failed to create new connection")
finally:
with self._lock:
self._scheduled_for_creation -= 1
def _add_conn_if_under_max(self):
max_conns = self._session.cluster.get_max_connections_per_host(
self.host_distance)
with self._lock:
if self.is_shutdown:
return True
if self.open_count >= max_conns:
return True
self.open_count += 1
log.debug("Going to open new connection to host %s", self.host)
try:
conn = self._session.cluster.connection_factory(self.host.endpoint)
if self._keyspace:
conn.set_keyspace_blocking(self._session.keyspace)
self._next_trash_allowed_at = time.time() + _MIN_TRASH_INTERVAL
with self._lock:
new_connections = self._connections[:] + [conn]
self._connections = new_connections
log.debug(
"Added new connection (%s) to pool for host %s, signaling availablility",
id(conn), self.host)
self._signal_available_conn()
return True
except (ConnectionException, socket.error) as exc:
log.warning("Failed to add new connection to pool for host %s: %s",
self.host, exc)
with self._lock:
self.open_count -= 1
if self._session.cluster.signal_connection_failure(
self.host, exc, is_host_addition=False):
self.shutdown()
return False
except AuthenticationFailed:
with self._lock:
self.open_count -= 1
return False
def _await_available_conn(self, timeout):
with self._conn_available_condition:
self._conn_available_condition.wait(timeout)
def _signal_available_conn(self):
with self._conn_available_condition:
self._conn_available_condition.notify()
def _signal_all_available_conn(self):
with self._conn_available_condition:
self._conn_available_condition.notify_all()
def _wait_for_conn(self, timeout):
start = time.time()
remaining = timeout
while remaining > 0:
# wait on our condition for the possibility that a connection
# is useable
self._await_available_conn(remaining)
# self.shutdown() may trigger the above Condition
if self.is_shutdown:
raise ConnectionException("Pool is shutdown")
conns = self._connections
if conns:
least_busy = min(conns, key=lambda c: c.in_flight)
with least_busy.lock:
if least_busy.in_flight < least_busy.max_request_id:
least_busy.in_flight += 1
return least_busy, least_busy.get_request_id()
remaining = timeout - (time.time() - start)
raise NoConnectionsAvailable()
def return_connection(self, connection):
with connection.lock:
connection.in_flight -= 1
in_flight = connection.in_flight
if connection.is_defunct or connection.is_closed:
if not connection.signaled_error:
log.debug(
"Defunct or closed connection (%s) returned to pool, potentially "
"marking host %s as down", id(connection), self.host)
is_down = self._session.cluster.signal_connection_failure(
self.host, connection.last_error, is_host_addition=False)
connection.signaled_error = True
if is_down:
self.shutdown()
else:
self._replace(connection)
else:
if connection in self._trash:
with connection.lock:
if connection.in_flight == 0:
with self._lock:
if connection in self._trash:
self._trash.remove(connection)
log.debug("Closing trashed connection (%s) to %s",
id(connection), self.host)
connection.close()
return
core_conns = self._session.cluster.get_core_connections_per_host(
self.host_distance)
min_reqs = self._session.cluster.get_min_requests_per_connection(
self.host_distance)
# we can use in_flight here without holding the connection lock
# because the fact that in_flight dipped below the min at some
# point is enough to start the trashing procedure
if len(self._connections) > core_conns and in_flight <= min_reqs and \
time.time() >= self._next_trash_allowed_at:
self._maybe_trash_connection(connection)
else:
self._signal_available_conn()
def _maybe_trash_connection(self, connection):
core_conns = self._session.cluster.get_core_connections_per_host(
self.host_distance)
did_trash = False
with self._lock:
if connection not in self._connections:
return
if self.open_count > core_conns:
did_trash = True
self.open_count -= 1
new_connections = self._connections[:]
new_connections.remove(connection)
self._connections = new_connections
with connection.lock:
if connection.in_flight == 0:
log.debug(
"Skipping trash and closing unused connection (%s) to %s",
id(connection), self.host)
connection.close()
# skip adding it to the trash if we're already closing it
return
self._trash.add(connection)
if did_trash:
self._next_trash_allowed_at = time.time() + _MIN_TRASH_INTERVAL
log.debug("Trashed connection (%s) to %s", id(connection),
self.host)
def _replace(self, connection):
should_replace = False
with self._lock:
if connection in self._connections:
new_connections = self._connections[:]
new_connections.remove(connection)
self._connections = new_connections
self.open_count -= 1
should_replace = True
if should_replace:
log.debug("Replacing connection (%s) to %s", id(connection),
self.host)
connection.close()
self._session.submit(self._retrying_replace)
else:
log.debug("Closing connection (%s) to %s", id(connection),
self.host)
connection.close()
def _retrying_replace(self):
replaced = False
try:
replaced = self._add_conn_if_under_max()
except Exception:
log.exception("Failed replacing connection to %s", self.host)
if not replaced:
log.debug("Failed replacing connection to %s. Retrying.",
self.host)
self._session.submit(self._retrying_replace)
def shutdown(self):
with self._lock:
if self.is_shutdown:
return
else:
self.is_shutdown = True
self._signal_all_available_conn()
for conn in self._connections:
conn.close()
self.open_count -= 1
for conn in self._trash:
conn.close()
def ensure_core_connections(self):
if self.is_shutdown:
return
core_conns = self._session.cluster.get_core_connections_per_host(
self.host_distance)
with self._lock:
to_create = core_conns - (len(self._connections) +
self._scheduled_for_creation)
for i in range(to_create):
self._scheduled_for_creation += 1
self._session.submit(self._create_new_connection)
def _set_keyspace_for_all_conns(self, keyspace, callback):
"""
Asynchronously sets the keyspace for all connections. When all
connections have been set, `callback` will be called with two
arguments: this pool, and a list of any errors that occurred.
"""
remaining_callbacks = set(self._connections)
errors = []
if not remaining_callbacks:
callback(self, errors)
return
def connection_finished_setting_keyspace(conn, error):
self.return_connection(conn)
remaining_callbacks.remove(conn)
if error:
errors.append(error)
if not remaining_callbacks:
callback(self, errors)
self._keyspace = keyspace
for conn in self._connections:
conn.set_keyspace_async(keyspace,
connection_finished_setting_keyspace)
def get_connections(self):
return self._connections
def get_state(self):
in_flights = [c.in_flight for c in self._connections]
return {
'shutdown': self.is_shutdown,
'open_count': self.open_count,
'in_flights': in_flights
}
class Dispatcher(InstrumentedThread):
def __init__(self, timeout=10):
super().__init__(name='Dispatcher')
self._timeout = timeout
self._msg_type_handlers = {}
self._in_queue = queue.PriorityQueue()
self._send_message = {}
self._send_last_message = {}
self._message_information = {}
self._condition = Condition()
self._dispatch_timers = {}
self._priority = {}
def _get_dispatch_timer(self, tag):
if tag not in self._dispatch_timers:
self._dispatch_timers[tag] = COLLECTOR.timer(
'dispatch_execution_time',
tags={"handler": tag},
instance=self)
return self._dispatch_timers[tag]
def add_send_message(self, connection, send_message):
"""Adds a send_message function to the Dispatcher's
dictionary of functions indexed by connection.
Args:
connection (str): A locally unique identifier
provided by the receiver of messages.
send_message (fn): The method that should be called
by the dispatcher to respond to messages which
arrive via connection.
"""
self._send_message[connection] = send_message
LOGGER.debug("Added send_message function "
"for connection %s", connection)
def add_send_last_message(self, connection, send_last_message):
"""Adds a send_last_message function to the Dispatcher's
dictionary of functions indexed by connection.
Args:
connection (str): A locally unique identifier
provided by the receiver of messages.
send_last_message (fn): The method that should be called
by the dispatcher to respond to messages which
arrive via connection, when the connection should be closed
after the message has been sent.
"""
self._send_last_message[connection] = send_last_message
LOGGER.debug("Added send_last_message function "
"for connection %s", connection)
def remove_send_message(self, connection):
"""Removes a send_message function previously registered
with the Dispatcher.
Args:
connection (str): A locally unique identifier provided
by the receiver of messages.
"""
if connection in self._send_message:
del self._send_message[connection]
LOGGER.debug("Removed send_message function "
"for connection %s", connection)
else:
LOGGER.warning(
"Attempted to remove send_message "
"function for connection %s, but no "
"send_message function was registered", connection)
def remove_send_last_message(self, connection):
"""Removes a send_last_message function previously registered
with the Dispatcher.
Args:
connection (str): A locally unique identifier provided
by the receiver of messages.
"""
if connection in self._send_last_message:
del self._send_last_message[connection]
LOGGER.debug(
"Removed send_last_message function "
"for connection %s", connection)
else:
LOGGER.warning(
"Attempted to remove send_last_message "
"function for connection %s, but no "
"send_last_message function was registered", connection)
def dispatch(self, connection, message, connection_id):
if message.message_type in self._msg_type_handlers:
priority = self._priority.get(message.message_type, Priority.LOW)
message_id = _gen_message_id()
self._message_information[message_id] = (
connection, connection_id, message,
_ManagerCollection(
self._msg_type_handlers[message.message_type]))
self._in_queue.put_nowait((priority, message_id))
queue_size = self._in_queue.qsize()
if queue_size > 10:
LOGGER.debug("Dispatch incoming queue size: %s", queue_size)
else:
LOGGER.info(
"received a message of type %s "
"from %s but have no handler for that type",
get_enum_name(message.message_type), connection_id)
def add_handler(self, message_type, handler, executor, priority=None):
if not isinstance(handler, Handler):
raise TypeError("%s is not a Handler subclass" % handler)
if message_type not in self._msg_type_handlers:
self._msg_type_handlers[message_type] = [
_HandlerManager(executor, handler)
]
else:
self._msg_type_handlers[message_type].append(
_HandlerManager(executor, handler))
if priority is not None:
self._priority[message_type] = priority
def set_message_priority(self, message_type, priority):
self._priority[message_type] = priority
def _process(self, message_id):
_, connection_id, \
message, collection = self._message_information[message_id]
try:
handler_manager = next(collection)
except IndexError:
# IndexError is raised if done with handlers
del self._message_information[message_id]
return
timer_tag = type(handler_manager.handler).__name__
timer_ctx = self._get_dispatch_timer(timer_tag).time()
def do_next(result):
timer_ctx.stop()
try:
self._determine_next(message_id, result)
except Exception: # pylint: disable=broad-except
LOGGER.exception("Unhandled exception while determining next")
handler_manager.execute(connection_id, message.content, do_next)
def _determine_next(self, message_id, result):
if result is None:
LOGGER.debug('Ignoring None handler result, likely due to an '
'unhandled error while executing the handler')
return
if result.status == HandlerStatus.DROP:
del self._message_information[message_id]
elif result.status == HandlerStatus.PASS:
self._process(message_id)
elif result.status == HandlerStatus.RETURN_AND_PASS:
connection, connection_id, \
original_message, _ = self._message_information[message_id]
if result.message_out and result.message_type:
message = validator_pb2.Message(
content=result.message_out.SerializeToString(),
correlation_id=original_message.correlation_id,
message_type=result.message_type)
try:
self._send_message[connection](msg=message,
connection_id=connection_id)
except KeyError:
LOGGER.warning(
"Can't send message %s back to "
"%s because connection %s not in dispatcher",
get_enum_name(message.message_type), connection_id,
connection)
self._process(message_id)
else:
LOGGER.error("HandlerResult with status of RETURN_AND_PASS "
"is missing message_out or message_type")
elif result.status == HandlerStatus.RETURN:
connection, connection_id, \
original_message, _ = self._message_information[message_id]
del self._message_information[message_id]
if result.message_out and result.message_type:
message = validator_pb2.Message(
content=result.message_out.SerializeToString(),
correlation_id=original_message.correlation_id,
message_type=result.message_type)
try:
self._send_message[connection](msg=message,
connection_id=connection_id)
except KeyError:
LOGGER.warning(
"Can't send message %s back to "
"%s because connection %s not in dispatcher",
get_enum_name(message.message_type), connection_id,
connection)
else:
LOGGER.error("HandlerResult with status of RETURN "
"is missing message_out or message_type")
elif result.status == HandlerStatus.RETURN_AND_CLOSE:
connection, connection_id, \
original_message, _ = self._message_information[message_id]
del self._message_information[message_id]
if result.message_out and result.message_type:
message = validator_pb2.Message(
content=result.message_out.SerializeToString(),
correlation_id=original_message.correlation_id,
message_type=result.message_type)
try:
LOGGER.warning(
"Sending hang-up in reply to %s to connection %s",
get_enum_name(original_message.message_type),
connection_id)
self._send_last_message[connection](
msg=message, connection_id=connection_id)
except KeyError:
LOGGER.warning(
"Can't send last message %s back to "
"%s because connection %s not in dispatcher",
get_enum_name(message.message_type), connection_id,
connection)
else:
LOGGER.error("HandlerResult with status of RETURN_AND_CLOSE "
"is missing message_out or message_type")
with self._condition:
if not self._message_information:
self._condition.notify()
def run(self):
while True:
try:
_, msg_id = self._in_queue.get()
if msg_id == -1:
break
self._process(msg_id)
except Exception: # pylint: disable=broad-except
LOGGER.exception("Unhandled exception while dispatching")
def stop(self):
self._in_queue.put_nowait((Priority.HIGH, -1))
def block_until_complete(self):
"""Blocks until no more messages are in flight,
useful for unit tests.
"""
with self._condition:
if self._message_information:
self._condition.wait()
class NailgunConnection(object):
"""Stateful object holding the connection to the Nailgun server."""
def __init__(
self,
server_name,
server_port=None,
stdin=sys.stdin,
stdout=sys.stdout,
stderr=sys.stderr,
cwd=None,
heartbeat_interval_sec=DEFAULT_HEARTBEAT_INTERVAL_SEC,
):
self.transport = make_nailgun_transport(server_name, server_port, cwd)
self.stdin = stdin
self.stdout = stdout
self.stderr = stderr
self.recv_flags = 0
self.send_flags = 0
self.header_buf = ctypes.create_string_buffer(CHUNK_HEADER_LEN)
self.buf = ctypes.create_string_buffer(BUFSIZE)
self.exit_code = None
self.shutdown_event = Event()
self.error_lock = RLock()
self.error = None
self.error_traceback = None
self.stdin_condition = Condition()
self.stdin_thread = Thread(target=stdin_thread_main, args=(self, ))
self.stdin_thread.daemon = True
self.send_queue = Queue.Queue()
self.send_condition = Condition()
self.send_thread = Thread(target=send_thread_main, args=(self, ))
self.send_thread.daemon = True
self.heartbeat_interval_sec = heartbeat_interval_sec
self.heartbeat_condition = Condition()
self.heartbeat_thread = None
if heartbeat_interval_sec > 0:
self.heartbeat_thread = Thread(target=heartbeat_thread_main,
args=(self, ))
self.heartbeat_thread.daemon = True
def send_command(self,
cmd,
cmd_args=[],
filearg=None,
env=os.environ,
cwd=os.getcwd()):
"""
Sends the command and environment to the nailgun server, then loops forever
reading the response until the server sends an exit chunk.
Returns the exit value, or raises NailgunException on error.
"""
try:
return self._send_command_and_read_response(
cmd, cmd_args, filearg, env, cwd)
except socket.error as e:
re_raise(
NailgunException(
"Server disconnected unexpectedly: {0}".format(e),
NailgunException.CONNECTION_BROKEN,
))
def _send_command_and_read_response(self, cmd, cmd_args, filearg, env,
cwd):
self.stdin_thread.start()
self.send_thread.start()
try:
if filearg:
self._send_file_arg(filearg)
for cmd_arg in cmd_args:
self._send_chunk(cmd_arg, CHUNKTYPE_ARG)
self._send_env_var("NAILGUN_FILESEPARATOR", os.sep)
self._send_env_var("NAILGUN_PATHSEPARATOR", os.pathsep)
self._send_tty_format(self.stdin)
self._send_tty_format(self.stdout)
self._send_tty_format(self.stderr)
for k, v in env.items():
self._send_env_var(k, v)
self._send_chunk(cwd, CHUNKTYPE_DIR)
self._send_chunk(cmd, CHUNKTYPE_CMD)
if self.heartbeat_thread is not None:
self.heartbeat_thread.start()
while self.exit_code is None:
self._process_next_chunk()
finally:
self.shutdown_event.set()
with self.stdin_condition:
self.stdin_condition.notify()
with self.send_condition:
self.send_condition.notify()
if self.heartbeat_thread is not None:
with self.heartbeat_condition:
self.heartbeat_condition.notify()
self.heartbeat_thread.join(THREAD_TERMINATION_TIMEOUT_SEC)
self.stdin_thread.join(THREAD_TERMINATION_TIMEOUT_SEC)
self.send_thread.join(THREAD_TERMINATION_TIMEOUT_SEC)
return self.exit_code
def _process_next_chunk(self):
"""
Processes the next chunk from the nailgun server.
"""
readable, exceptional = self.transport.select(
SELECT_MAX_BLOCK_TIME_SEC)
if readable:
self._process_nailgun_stream()
if exceptional:
raise NailgunException("Server disconnected in select",
NailgunException.CONNECTION_BROKEN)
# if daemon thread threw, rethrow here
if self.shutdown_event.is_set():
e = None
e_tb = None
with self.error_lock:
e = self.error
e_tb = self.error_traceback
if e is not None:
re_raise(e, e_tb)
def _send_chunk(self, buf, chunk_type):
"""
Send chunk to the server asynchronously
"""
self.send_queue.put((chunk_type, buf))
with self.send_condition:
self.send_condition.notify()
def _send_env_var(self, name, value):
"""
Sends an environment variable in KEY=VALUE format.
"""
self._send_chunk("=".join((name, value)), CHUNKTYPE_ENV)
def _send_tty_format(self, f):
"""
Sends a NAILGUN_TTY_# environment variable.
"""
if not f or not hasattr(f, "fileno"):
return
try:
fileno = f.fileno()
isatty = os.isatty(fileno)
self._send_env_var("NAILGUN_TTY_" + str(fileno), str(int(isatty)))
except UnsupportedOperation:
return
def _send_file_arg(self, filename):
"""
Sends the contents of a file to the server.
"""
with open(filename) as f:
while True:
num_bytes = f.readinto(self.buf)
if not num_bytes:
break
self._send_chunk(self.buf.raw[:num_bytes], CHUNKTYPE_LONGARG)
def _recv_to_fd(self, dest_file, num_bytes):
"""
Receives num_bytes bytes from the nailgun socket and copies them to the specified file
object. Used to route data to stdout or stderr on the client.
"""
bytes_read = 0
while bytes_read < num_bytes:
bytes_to_read = min(len(self.buf), num_bytes - bytes_read)
bytes_received = self.transport.recv_into(self.buf, bytes_to_read)
if dest_file:
dest_file.write(bytes_to_str(self.buf[:bytes_received]))
bytes_read += bytes_received
def _recv_to_buffer(self, num_bytes, buf):
"""
Receives num_bytes from the nailgun socket and writes them into the specified buffer.
"""
# We'd love to use socket.recv_into() everywhere to avoid
# unnecessary copies, but we need to support Python 2.6. The
# only way to provide an offset to recv_into() is to use
# memoryview(), which doesn't exist until Python 2.7.
if HAS_MEMORYVIEW:
self._recv_into_memoryview(num_bytes, compat_memoryview(buf))
else:
self._recv_to_buffer_with_copy(num_bytes, buf)
def _recv_into_memoryview(self, num_bytes, buf_view):
"""
Receives num_bytes from the nailgun socket and writes them into the specified memoryview
to avoid an extra copy.
"""
bytes_read = 0
while bytes_read < num_bytes:
bytes_received = self.transport.recv_into(buf_view[bytes_read:],
num_bytes - bytes_read)
if not bytes_received:
raise NailgunException(
"Server unexpectedly disconnected in recv_into()",
NailgunException.CONNECTION_BROKEN,
)
bytes_read += bytes_received
def _recv_to_buffer_with_copy(self, num_bytes, buf):
"""
Receives num_bytes from the nailgun socket and writes them into the specified buffer.
"""
bytes_read = 0
while bytes_read < num_bytes:
recv_buf = self.transport.recv(num_bytes - bytes_read)
if not len(recv_buf):
raise NailgunException(
"Server unexpectedly disconnected in recv()",
NailgunException.CONNECTION_BROKEN,
)
buf[bytes_read:bytes_read + len(recv_buf)] = recv_buf
bytes_read += len(recv_buf)
def _process_exit(self, exit_len):
"""
Receives an exit code from the nailgun server and sets nailgun_connection.exit_code
to indicate the client should exit.
"""
num_bytes = min(len(self.buf), exit_len)
self._recv_to_buffer(num_bytes, self.buf)
self.exit_code = int(self.buf.raw[:num_bytes])
def _send_heartbeat(self):
"""
Sends a heartbeat to the nailgun server to indicate the client is still alive.
"""
self._send_chunk("", CHUNKTYPE_HEARTBEAT)
def _process_nailgun_stream(self):
"""
Processes a single chunk from the nailgun server.
"""
self._recv_to_buffer(len(self.header_buf), self.header_buf)
(chunk_len, chunk_type) = struct.unpack_from(">ic",
self.header_buf.raw)
if chunk_type == CHUNKTYPE_STDOUT:
self._recv_to_fd(self.stdout, chunk_len)
elif chunk_type == CHUNKTYPE_STDERR:
self._recv_to_fd(self.stderr, chunk_len)
elif chunk_type == CHUNKTYPE_EXIT:
self._process_exit(chunk_len)
elif chunk_type == CHUNKTYPE_SENDINPUT:
# signal stdin thread to get and send more data
with self.stdin_condition:
self.stdin_condition.notify()
else:
raise NailgunException(
"Unexpected chunk type: {0}".format(chunk_type),
NailgunException.UNEXPECTED_CHUNKTYPE,
)
def wait_termination(self, timeout):
"""
Wait for shutdown event to be signalled within specified interval
Return True if termination was signalled, False otherwise
"""
wait_time = timeout
start = monotonic_time_nanos()
with self.send_condition:
while True:
if self.shutdown_event.is_set():
return True
self.send_condition.wait(wait_time)
elapsed = (monotonic_time_nanos() - start) * 1.0 / NSEC_PER_SEC
wait_time = timeout - elapsed
if wait_time <= 0:
return False
return False
def __enter__(self):
return self
def __exit__(self, type, value, traceback):
try:
self.transport.close()
except socket.error:
pass
class HostConnection(object):
"""
When using v3 of the native protocol, this is used instead of a connection
pool per host (HostConnectionPool) due to the increased in-flight capacity
of individual connections.
"""
host = None
host_distance = None
is_shutdown = False
shutdown_on_error = False
_session = None
_connection = None
_lock = None
_keyspace = None
def __init__(self, host, host_distance, session):
self.host = host
self.host_distance = host_distance
self._session = weakref.proxy(session)
self._lock = Lock()
# this is used in conjunction with the connection streams. Not using the connection lock because the connection can be replaced in the lifetime of the pool.
self._stream_available_condition = Condition(self._lock)
self._is_replacing = False
if host_distance == HostDistance.IGNORED:
log.debug("Not opening connection to ignored host %s", self.host)
return
elif host_distance == HostDistance.REMOTE and not session.cluster.connect_to_remote_hosts:
log.debug("Not opening connection to remote host %s", self.host)
return
log.debug("Initializing connection for host %s", self.host)
self._connection = session.cluster.connection_factory(host.endpoint)
self._keyspace = session.keyspace
if self._keyspace:
self._connection.set_keyspace_blocking(self._keyspace)
log.debug("Finished initializing connection for host %s", self.host)
def borrow_connection(self, timeout):
if self.is_shutdown:
raise ConnectionException(
"Pool for %s is shutdown" % (self.host, ), self.host)
conn = self._connection
if not conn:
raise NoConnectionsAvailable()
start = time.time()
remaining = timeout
while True:
with conn.lock:
if conn.in_flight <= conn.max_request_id:
conn.in_flight += 1
return conn, conn.get_request_id()
if timeout is not None:
remaining = timeout - time.time() + start
if remaining < 0:
break
with self._stream_available_condition:
self._stream_available_condition.wait(remaining)
raise NoConnectionsAvailable("All request IDs are currently in use")
def return_connection(self, connection):
with connection.lock:
connection.in_flight -= 1
with self._stream_available_condition:
self._stream_available_condition.notify()
if connection.is_defunct or connection.is_closed:
if connection.signaled_error and not self.shutdown_on_error:
return
is_down = False
if not connection.signaled_error:
log.debug(
"Defunct or closed connection (%s) returned to pool, potentially "
"marking host %s as down", id(connection), self.host)
is_down = self._session.cluster.signal_connection_failure(
self.host, connection.last_error, is_host_addition=False)
connection.signaled_error = True
if self.shutdown_on_error and not is_down:
is_down = True
self._session.cluster.on_down(self.host,
is_host_addition=False)
if is_down:
self.shutdown()
else:
self._connection = None
with self._lock:
if self._is_replacing:
return
self._is_replacing = True
self._session.submit(self._replace, connection)
def _replace(self, connection):
with self._lock:
if self.is_shutdown:
return
log.debug("Replacing connection (%s) to %s", id(connection), self.host)
try:
conn = self._session.cluster.connection_factory(self.host.endpoint)
if self._keyspace:
conn.set_keyspace_blocking(self._keyspace)
self._connection = conn
except Exception:
log.warning("Failed reconnecting %s. Retrying." %
(self.host.endpoint, ))
self._session.submit(self._replace, connection)
else:
with self._lock:
self._is_replacing = False
self._stream_available_condition.notify()
def shutdown(self):
with self._lock:
if self.is_shutdown:
return
else:
self.is_shutdown = True
self._stream_available_condition.notify_all()
if self._connection:
self._connection.close()
self._connection = None
def _set_keyspace_for_all_conns(self, keyspace, callback):
if self.is_shutdown or not self._connection:
return
def connection_finished_setting_keyspace(conn, error):
self.return_connection(conn)
errors = [] if not error else [error]
callback(self, errors)
self._keyspace = keyspace
self._connection.set_keyspace_async(
keyspace, connection_finished_setting_keyspace)
def get_connections(self):
c = self._connection
return [c] if c else []
def get_state(self):
connection = self._connection
open_count = 1 if connection and not (connection.is_closed
or connection.is_defunct) else 0
in_flights = [connection.in_flight] if connection else []
return {
'shutdown': self.is_shutdown,
'open_count': open_count,
'in_flights': in_flights
}
@property
def open_count(self):
connection = self._connection
return 1 if connection and not (connection.is_closed
or connection.is_defunct) else 0
class Queue:
"""This class implements multi-producer, multi-consumer FIFO queue.
The main objective of Queue is to create a list of tasks for asynchronous processing
"""
def __init__(self, maxsize) -> None:
self._maxsize = maxsize
self._queue = OrderedDict()
self._lock = RLock()
self._not_full = Condition(self._lock)
self._not_empty = Condition(self._lock)
def put(self, id: str, content: Any, timeout: int = 10) -> None:
"""Put an message into the queue.
The 'id' and 'content' values are used to create a new QueueMessage and into the Queue.
If the queue is full it will try to put the message for 'timeout' seconds before raise a Full Exception.
"""
if timeout < 0:
raise ValueError
if id in self._queue:
raise RepeatedMessage
with self._not_full:
if not self.full() or self._not_full.wait(timeout):
queue_message = QueueMessage(id, content)
self._queue[id] = queue_message
self._not_empty.notify()
return
raise Full
def qsize(self) -> int:
"""Return the size of the queue."""
with self._lock:
return len(self._queue)
def full(self) -> bool:
"""Return True if the queue is full."""
return 0 < self._maxsize <= self.qsize()
def get(self,
timeout: int = 10,
acquire_timeout: int = 10) -> QueueMessage:
"""Return an message from the queue. This message is 'hidden' from the queue until its deletion or 'acquire_timeout' is reached.
'timeout' arg inform how long the Queue object should try to get the message if the queue is empty before raising the `Empty` error.
'acquire_timeout' arg value inform how long the thread wants to have this message for itself, if the message is not deleted it should be available in the queue after this time.
"""
if timeout < 0 or acquire_timeout < 0:
raise ValueError
with self._not_empty:
max_timeout = min_timeout = time.monotonic() + timeout
while max_timeout > time.monotonic():
if self.qsize() > 0:
for id in self._queue:
if (self._queue[id].timeout is None
or self._queue[id].timeout < time.monotonic()):
self._queue[id].timeout = time.monotonic(
) + acquire_timeout
return self._queue[id]
else:
if self._queue[id].timeout < min_timeout:
min_timeout = self._queue[id].timeout
self._not_empty.wait(min_timeout - time.monotonic())
raise Empty
def delete(self, id: str) -> None:
"""Removes the message from the queue"""
with self._lock:
if id in self._queue:
del self._queue[id]
self._not_full.notify()
return
raise DeleteAttemptToUnknownMessage
class ZtexBoardProxy(Process):
def __init__(self, rxconn, txconn, serial, takeover, firmware,
pollinterval):
super(ZtexBoardProxy, self).__init__()
self.rxconn = rxconn
self.txconn = txconn
self.serial = serial
self.takeover = takeover
self.firmware = firmware
self.pollinterval = pollinterval
def run(self):
signal.signal(signal.SIGINT, signal.SIG_IGN)
signal.signal(signal.SIGTERM, signal.SIG_IGN)
self.lock = RLock()
self.wakeup = Condition()
self.error = None
self.pollingthread = None
self.shutdown = False
self.job = None
self.checklockout = 0
self.lastnonce = 0
self.multiplier = 0
try:
# Listen for setup commands
while True:
data = self.rxconn.recv()
if data[0] == "connect": break
else: raise Exception("Unknown setup message: %s" % str(data))
# Connect to board and upload firmware if neccessary
self.device = ZtexDevice(self, self.serial, self.takeover,
self.firmware)
# Configure clock
self._set_multiplier(self.device.default_multiplier)
# Start polling thread
self.pollingthread = Thread(None, self.polling_thread,
"polling_thread")
self.pollingthread.daemon = True
self.pollingthread.start()
self.send("started_up")
# Listen for commands
while True:
if self.error: raise self.error
data = self.rxconn.recv()
if data[0] == "shutdown": break
elif data[0] == "ping": self.send("pong")
elif data[0] == "pong": pass
elif data[0] == "set_pollinterval":
self.pollinterval = data[1]
with self.wakeup:
self.wakeup.notify()
elif data[0] == "send_job":
self.checklockout = time.time() + 1
self.job = data[1]
with self.wakeup:
start = time.time()
self.device.send_job(data[1][64:76] + data[2])
end = time.time()
self.lastnonce = 0
self.checklockout = end + 0.5
self.respond(start, end)
else:
raise Exception("Unknown message: %s" % str(data))
except:
self.log("Exception caught: %s" % traceback.format_exc(), 100, "r")
finally:
self.shutdown = True
with self.wakeup:
self.wakeup.notify()
try:
self.pollingthread.join(2)
except:
pass
self.send("dying")
def send(self, *args):
with self.lock:
self.txconn.send(args)
def respond(self, *args):
self.send("response", *args)
def log(self, message, loglevel, format=""):
self.send("log", message, loglevel, format)
def polling_thread(self):
try:
lastshares = []
errorcount = [0] * (self.device.maximum_multiplier + 1)
errorweight = [0] * (self.device.maximum_multiplier + 1)
maxerrorrate = [0] * (self.device.maximum_multiplier + 1)
errorlimit = 0.05
errorhysteresis = 0.1
counter = 0
while not self.shutdown:
counter += 1
# Poll for nonces
now = time.time()
nonces = self.device.read_nonces()
exhausted = False
with self.wakeup:
if nonces[0][1] < self.lastnonce:
self.lastnonce = nonces[0][1]
exhausted = True
if exhausted: self.send("keyspace_exhausted")
for nonce in nonces:
if nonce[0] != -self.device.nonce_offset and not nonce[
0] in lastshares:
if self.job:
self.send("nonce_found", time.time(),
struct.pack("<I", nonce[0]))
lastshares.append(nonce[0])
while len(lastshares) > len(nonces):
lastshares.pop(0)
# Verify proper operation and adjust clocking if neccessary
if now > self.checklockout and self.job:
errorcount[self.multiplier] *= 0.995
errorweight[
self.
multiplier] = errorweight[self.multiplier] * 0.995 + 1
for nonce in nonces:
invalid = True
for offset in (0, 1, -1, 2, -2):
hash = Job.calculate_hash(
self.job[:76] +
struct.pack("<I", nonce[1] + offset))
if struct.unpack(
"!I",
hash[-4:])[0] == (nonce[2] +
0x5be0cd19) & 0xffffffff:
invalid = False
break
if invalid:
errorcount[self.multiplier] += 1. / len(nonces)
certainty = min(1, errorweight[self.multiplier] / 100)
errorrate = errorcount[self.multiplier] / errorweight[
self.multiplier]
maxerrorrate[self.multiplier] = max(
maxerrorrate[self.multiplier], errorrate * certainty)
for i in range(len(maxerrorrate) - 1):
if maxerrorrate[i + 1] * i < maxerrorrate[i] * (i +
20):
maxerrorrate[i +
1] = maxerrorrate[i] * (1 + 20.0 / i)
limit = 0
while limit < self.device.default_multiplier and maxerrorrate[
limit + 1] < errorlimit:
limit += 1
while limit < self.device.maximum_multiplier and errorweight[
limit] > 150 and maxerrorrate[limit +
1] < errorlimit:
limit += 1
multiplier = 0
best = 0
for i in range(limit + 1):
effective = (i + 1 +
(errorhysteresis if i == self.multiplier
else 0)) * (1 - maxerrorrate[i])
if effective > best:
best = effective
multiplier = i
self._set_multiplier(multiplier)
if counter >= 10:
counter = 0
try:
self.send(
"error_rate", errorcount[self.multiplier] /
errorweight[self.multiplier])
except:
pass
with self.wakeup:
self.wakeup.wait(self.pollinterval)
except Exception as e:
self.log("Exception caught: %s" % traceback.format_exc(), 100, "r")
self.error = e
# Unblock main thread
self.send("ping")
def _set_multiplier(self, multiplier):
multiplier = min(max(multiplier, 1), self.device.maximum_multiplier)
if multiplier == self.multiplier: return
self.device.set_multiplier(multiplier)
self.multiplier = multiplier
self.checklockout = time.time() + 2
self.send("speed_changed", (multiplier + 1) *
self.device.base_frequency * self.device.hashes_per_clock)
class TimedTaskQueue:
__single = None
def __init__(self, nameprefix="TimedTaskQueue"):
self.cond = Condition()
self.queue = []
self.thread = Thread(target=self.run)
self.thread.setDaemon(True)
self.thread.setName(nameprefix + self.thread.getName())
self.thread.start()
def add_task(self, task, t, id=None):
""" t parameter is now usable, unlike before.
If id is given, all the existing tasks with the same id will be removed
before inserting this task
"""
if task is None:
print_stack()
self.cond.acquire()
when = time() + t
if DEBUG:
print >> sys.stderr, "ttqueue: ADD EVENT", t, task
if id != None: # remove all redundant tasks
self.queue = filter(lambda item: item[2] != id, self.queue)
self.queue.append((when, task, id))
self.cond.notify()
self.cond.release()
def run(self):
""" Run by server thread """
while True:
task = None
timeout = None
flag = False
self.cond.acquire()
while True:
while len(self.queue) == 0 or flag:
flag = False
if timeout is None:
# Wait until something is queued
self.cond.wait()
else:
# Wait till first event is due
self.cond.wait(timeout)
# A new event was added or an event is due
self.queue.sort()
(when, task, id) = self.queue[0]
if DEBUG:
print >> sys.stderr, "ttqueue: EVENT IN QUEUE", when, task
now = time()
if now < when:
# Event not due, wait some more
if DEBUG:
print >> sys.stderr, "ttqueue: EVENT NOT TILL", when - now
timeout = when - now
flag = True
else:
# Event due, execute
if DEBUG:
print >> sys.stderr, "ttqueue: EVENT DUE"
self.queue.pop(0)
break
self.cond.release()
# Execute task outside lock
try:
task()
except:
print_exc()
class AntColony:
def __init__(self, graph, num_ants, num_iterations):
self.graph = graph
self.num_ants = num_ants
self.num_iterations = num_iterations
self.Alpha = 0.1
# condition var
self.cv = Condition()
self.reset()
def reset(self):
self.best_path_cost = sys.maxint
self.best_path_vec = None
self.best_path_mat = None
self.last_best_path_iteration = 0
def start(self):
self.ants = self.create_ants()
self.iter_counter = 0
while self.iter_counter < self.num_iterations:
self.iteration()
self.cv.acquire()
# wait until update calls notify()
self.cv.wait()
lock = self.graph.lock
lock.acquire()
self.global_updating_rule()
lock.release()
self.cv.release()
# one iteration involves spawning a number of ant threads
def iteration(self):
self.avg_path_cost = 0
self.ant_counter = 0
self.iter_counter += 1
print("iter_counter = %s" % (self.iter_counter, ))
for ant in self.ants:
print("starting ant = %s" % (ant.ID))
ant.start()
def num_ants(self):
return len(self.ants)
def num_iterations(self):
return self.num_iterations
def iteration_counter(self):
return self.iter_counter
# called by individual ants
def update(self, ant):
lock = Lock()
lock.acquire()
#outfile = open("results.dat", "a")
print("Update called by %s" % (ant.ID, ))
self.ant_counter += 1
self.avg_path_cost += ant.path_cost
# book-keeping
if ant.path_cost < self.best_path_cost:
self.best_path_cost = ant.path_cost
self.best_path_mat = ant.path_mat
self.best_path_vec = ant.path_vec
self.last_best_path_iteration = self.iter_counter
if self.ant_counter == len(self.ants):
self.avg_path_cost /= len(self.ants)
print("Best: %s, %s, %s, %s" % (
self.best_path_vec,
self.best_path_cost,
self.iter_counter,
self.avg_path_cost,
))
#outfile.write("\n%s\t%s\t%s" % (self.iter_counter, self.avg_path_cost, self.best_path_cost,))
self.cv.acquire()
self.cv.notify()
self.cv.release()
#outfile.close()
lock.release()
def done(self):
return self.iter_counter == self.num_iterations
# assign each ant a random start-node
def create_ants(self):
self.reset()
ants = []
for i in range(0, self.num_ants):
ant = Ant(i, random.randint(0, self.graph.num_nodes - 1), self)
ants.append(ant)
return ants
# changes the tau matrix based on evaporation/deposition
def global_updating_rule(self):
evaporation = 0
deposition = 0
for r in range(0, self.graph.num_nodes):
for s in range(0, self.graph.num_nodes):
if r != s:
delt_tau = self.best_path_mat[r][s] / self.best_path_cost
evaporation = (1 - self.Alpha) * self.graph.tau(r, s)
deposition = self.Alpha * delt_tau
self.graph.update_tau(r, s, evaporation + deposition)
class Sampler(object):
""" Sampler used to play, stop and mix multiple sounds.
.. warning:: A single sampler instance should be used at a time.
"""
def __init__(self, sr=22050, backend='sounddevice', timeout=1):
"""
:param int sr: samplerate used - all sounds added to the sampler will automatically be resampled if needed (- his can be a CPU consumming task, try to use sound with all identical sampling rate if possible.
:param str backend: backend used for playing sound. Can be either 'sounddevice' or 'dummy'.
"""
self.sr = sr
self.sounds = []
self.chunks = Queue(1)
self.chunk_available = Condition()
self.is_done = Event() # new event to prevent play to be called again before the sound is actually played
self.timeout = timeout # timeout value for graceful exit of the BackendStream
if backend == 'dummy':
from .dummy_stream import DummyStream
self.BackendStream = DummyStream
elif backend == 'sounddevice':
from sounddevice import OutputStream
self.BackendStream = OutputStream
else:
raise ValueError("Backend can either be 'sounddevice' or 'dummy'")
# TODO: use a process instead?
self.play_thread = Thread(target=self.run)
self.play_thread.daemon = True
self.play_thread.start()
def __enter__(self):
return self
def __exit__(self, *args):
self.play_thread.join()
def play(self, sound):
""" Adds and plays a new Sound to the Sampler.
:param sound: sound to play
.. note:: If the sound is already playing, it will restart from the beginning.
"""
self.is_done.clear() # hold is_done until the sound is played
if self.sr != sound.sr:
raise ValueError('You can only play sound with a samplerate of {} (here {}). Use the Sound.resample method for instance.', self.sr, sound.sr)
if sound in self.sounds:
self.remove(sound)
with self.chunk_available:
self.sounds.append(sound)
sound.playing = True
self.chunk_available.notify()
self.is_done.wait() # wait for the sound to be entirely played
def remove(self, sound):
""" Remove a currently played sound. """
with self.chunk_available:
sound.playing = False
self.sounds.remove(sound)
# Play loop
def next_chunks(self):
""" Gets a new chunk from all played sound and mix them together. """
with self.chunk_available:
while True:
playing_sounds = [s for s in self.sounds if s.playing]
chunks = []
for s in playing_sounds:
try:
chunks.append(next(s.chunks))
except StopIteration:
s.playing = False
self.sounds.remove(s)
self.is_done.set() # sound was played, release is_done to end the wait in play
if chunks:
break
self.chunk_available.wait()
return numpy.mean(chunks, axis=0)
def run(self):
""" Play loop, i.e. send all sound chunk by chunk to the soundcard. """
self.running = True
def chunks_producer():
while self.running:
self.chunks.put(self.next_chunks())
t = Thread(target=chunks_producer)
t.start()
with self.BackendStream(samplerate=self.sr, channels=1) as stream:
while self.running:
try:
stream.write(self.chunks.get(timeout=self.timeout)) # timeout so stream.write() thread can exit
except Empty:
self.running = False # let play_thread exit
class NtTestBase(NetworkTablesInstance):
"""
Object for managing a live pair of NT server/client
"""
_wait_lock = None
_testing_verbose_logging = True
def shutdown(self):
logger.info("shutting down %s", self.__class__.__name__)
NetworkTablesInstance.shutdown(self)
if self._wait_lock is not None:
self._wait_init_listener()
def disconnect(self):
self._api.dispatcher.stop()
def _init_common(self, proto_rev):
# This resets the instance to be independent
self.shutdown()
self._api.dispatcher.setDefaultProtoRev(proto_rev)
self.proto_rev = proto_rev
if self._testing_verbose_logging:
self.enableVerboseLogging()
# self._wait_init()
def _init_server(self, proto_rev, server_port=0):
self._init_common(proto_rev)
self.port = server_port
def _init_client(self, proto_rev):
self._init_common(proto_rev)
def _wait_init(self):
self._wait_lock = Condition()
self._wait = 0
self._wait_init_listener()
def _wait_init_listener(self):
self._api.addEntryListener(
"",
self._wait_cb,
NetworkTablesInstance.NotifyFlags.NEW
| NetworkTablesInstance.NotifyFlags.UPDATE
| NetworkTablesInstance.NotifyFlags.DELETE
| NetworkTablesInstance.NotifyFlags.FLAGS,
)
def _wait_cb(self, *args):
with self._wait_lock:
self._wait += 1
# logger.info('Wait callback, got: %s', args)
self._wait_lock.notify()
@contextmanager
def expect_changes(self, count):
"""Use this on the *other* instance that you're making
changes on, to wait for the changes to propagate to the
other instance"""
if self._wait_lock is None:
self._wait_init()
with self._wait_lock:
self._wait = 0
logger.info("Begin actions")
yield
logger.info("Waiting for %s changes", count)
with self._wait_lock:
result, msg = (
self._wait_lock.wait_for(lambda: self._wait == count, 4),
"Timeout waiting for %s changes (got %s)" %
(count, self._wait),
)
logger.info("expect_changes: %s %s", result, msg)
assert result, msg
class IbApi(EWrapper):
""""""
data_filename = "ib_contract_data.db"
data_filepath = str(get_file_path(data_filename))
local_tz = get_localzone()
def __init__(self, gateway: BaseGateway):
""""""
super().__init__()
self.gateway = gateway
self.gateway_name = gateway.gateway_name
self.status = False
self.reqid = 0
self.orderid = 0
self.clientid = 0
self.account = ""
self.ticks = {}
self.orders = {}
self.accounts = {}
self.contracts = {}
self.tick_exchange = {}
self.history_req = None
self.history_condition = Condition()
self.history_buf = []
self.client = IbClient(self)
self.thread = Thread(target=self.client.run)
def connectAck(self): # pylint: disable=invalid-name
"""
Callback when connection is established.
"""
self.status = True
self.gateway.write_log("IB TWS连接成功")
self.load_contract_data()
def connectionClosed(self): # pylint: disable=invalid-name
"""
Callback when connection is closed.
"""
self.status = False
self.gateway.write_log("IB TWS连接断开")
def nextValidId(self, orderId: int): # pylint: disable=invalid-name
"""
Callback of next valid orderid.
"""
super().nextValidId(orderId)
if not self.orderid:
self.orderid = orderId
def currentTime(self, time: int): # pylint: disable=invalid-name
"""
Callback of current server time of IB.
"""
super().currentTime(time)
dt = datetime.fromtimestamp(time)
time_string = dt.strftime("%Y-%m-%d %H:%M:%S.%f")
msg = f"服务器时间: {time_string}"
self.gateway.write_log(msg)
def error(self, reqId: TickerId, errorCode: int, errorString: str): # pylint: disable=invalid-name
"""
Callback of error caused by specific request.
"""
super().error(reqId, errorCode, errorString)
msg = f"信息通知，代码：{errorCode}，内容: {errorString}"
self.gateway.write_log(msg)
def tickPrice( # pylint: disable=invalid-name
self, reqId: TickerId, tickType: TickType, price: float,
attrib: TickAttrib):
"""
Callback of tick price update.
"""
super().tickPrice(reqId, tickType, price, attrib)
if tickType not in TICKFIELD_IB2VT:
return
tick = self.ticks[reqId]
name = TICKFIELD_IB2VT[tickType]
setattr(tick, name, price)
# Update name into tick data.
contract = self.contracts.get(tick.vt_symbol, None)
if contract:
tick.name = contract.name
# Forex and spot product of IDEALPRO has no tick time and last price.
# We need to calculate locally.
exchange = self.tick_exchange[reqId]
if exchange is Exchange.IDEALPRO:
tick.last_price = (tick.bid_price_1 + tick.ask_price_1) / 2
tick.datetime = datetime.now(self.local_tz)
self.gateway.on_tick(copy(tick))
def tickSize(self, reqId: TickerId, tickType: TickType, size: int): # pylint: disable=invalid-name
"""
Callback of tick volume update.
"""
super().tickSize(reqId, tickType, size)
if tickType not in TICKFIELD_IB2VT:
return
tick = self.ticks[reqId]
name = TICKFIELD_IB2VT[tickType]
setattr(tick, name, size)
self.gateway.on_tick(copy(tick))
def tickString(self, reqId: TickerId, tickType: TickType, value: str): # pylint: disable=invalid-name
"""
Callback of tick string update.
"""
super().tickString(reqId, tickType, value)
if tickType != TickTypeEnum.LAST_TIMESTAMP:
return
tick = self.ticks[reqId]
dt = datetime.fromtimestamp(int(value))
tick.datetime = dt.replace(tzinfo=self.local_tz)
self.gateway.on_tick(copy(tick))
def orderStatus( # pylint: disable=invalid-name
self,
orderId: OrderId,
status: str,
filled: float,
remaining: float,
avgFillPrice: float,
permId: int,
parentId: int,
lastFillPrice: float,
clientId: int,
whyHeld: str,
mktCapPrice: float,
):
"""
Callback of order status update.
"""
super().orderStatus(
orderId,
status,
filled,
remaining,
avgFillPrice,
permId,
parentId,
lastFillPrice,
clientId,
whyHeld,
mktCapPrice,
)
orderid = str(orderId)
order = self.orders.get(orderid, None)
order.traded = filled
# To filter PendingCancel status
order_status = STATUS_IB2VT.get(status, None)
if order_status:
order.status = order_status
self.gateway.on_order(copy(order))
def openOrder( # pylint: disable=invalid-name
self,
orderId: OrderId,
ib_contract: Contract,
ib_order: Order,
orderState: OrderState,
):
"""
Callback when opening new order.
"""
super().openOrder(orderId, ib_contract, ib_order, orderState)
orderid = str(orderId)
order = OrderData(
symbol=ib_contract.conId,
exchange=EXCHANGE_IB2VT.get(ib_contract.exchange,
ib_contract.exchange),
type=ORDERTYPE_IB2VT[ib_order.orderType],
orderid=orderid,
direction=DIRECTION_IB2VT[ib_order.action],
volume=ib_order.totalQuantity,
gateway_name=self.gateway_name,
)
if order.type == OrderType.LIMIT:
order.price = ib_order.lmtPrice
elif order.type == OrderType.STOP:
order.price = ib_order.auxPrice
self.orders[orderid] = order
self.gateway.on_order(copy(order))
def updateAccountValue( # pylint: disable=invalid-name
self, key: str, val: str, currency: str, accountName: str):
"""
Callback of account update.
"""
super().updateAccountValue(key, val, currency, accountName)
if not currency or key not in ACCOUNTFIELD_IB2VT:
return
accountid = f"{accountName}.{currency}"
account = self.accounts.get(accountid, None)
if not account:
account = AccountData(accountid=accountid,
gateway_name=self.gateway_name)
self.accounts[accountid] = account
name = ACCOUNTFIELD_IB2VT[key]
setattr(account, name, float(val))
def updatePortfolio( # pylint: disable=invalid-name
self,
contract: Contract,
position: float,
marketPrice: float,
marketValue: float,
averageCost: float,
unrealizedPNL: float,
realizedPNL: float,
accountName: str,
):
"""
Callback of position update.
"""
super().updatePortfolio(
contract,
position,
marketPrice,
marketValue,
averageCost,
unrealizedPNL,
realizedPNL,
accountName,
)
if contract.exchange:
exchange = EXCHANGE_IB2VT.get(contract.exchange, None)
elif contract.primaryExchange:
exchange = EXCHANGE_IB2VT.get(contract.primaryExchange, None)
else:
exchange = Exchange.SMART # Use smart routing for default
if not exchange:
msg = f"存在不支持的交易所持仓{contract.conId} {contract.exchange} {contract.primaryExchange}"
self.gateway.write_log(msg)
return
try:
ib_size = int(contract.multiplier)
except ValueError:
ib_size = 1
price = averageCost / ib_size
pos = PositionData(
symbol=generate_symbol(contract),
exchange=exchange,
direction=Direction.NET,
volume=position,
price=price,
pnl=unrealizedPNL,
gateway_name=self.gateway_name,
)
self.gateway.on_position(pos)
def updateAccountTime(self, timeStamp: str): # pylint: disable=invalid-name
"""
Callback of account update time.
"""
super().updateAccountTime(timeStamp)
for account in self.accounts.values():
self.gateway.on_account(copy(account))
def contractDetails(self, reqId: int, contractDetails: ContractDetails): # pylint: disable=invalid-name
"""
Callback of contract data update.
"""
super().contractDetails(reqId, contractDetails)
# Generate symbol from ib contract details
ib_contract = contractDetails.contract
if not ib_contract.multiplier:
ib_contract.multiplier = 1
symbol = generate_symbol(ib_contract)
# Generate contract
contract = ContractData(
symbol=symbol,
exchange=EXCHANGE_IB2VT[ib_contract.exchange],
name=contractDetails.longName,
product=PRODUCT_IB2VT[ib_contract.secType],
size=ib_contract.multiplier,
pricetick=contractDetails.minTick,
net_position=True,
history_data=True,
stop_supported=True,
gateway_name=self.gateway_name,
)
if contract.vt_symbol not in self.contracts:
self.gateway.on_contract(contract)
self.contracts[contract.vt_symbol] = contract
self.save_contract_data()
def execDetails(self, reqId: int, contract: Contract,
execution: Execution): # pylint: disable=invalid-name
"""
Callback of trade data update.
"""
super().execDetails(reqId, contract, execution)
dt = datetime.strptime(execution.time, "%Y%m%d %H:%M:%S")
dt = dt.replace(tzinfo=self.local_tz)
trade = TradeData(
symbol=contract.conId,
exchange=EXCHANGE_IB2VT.get(contract.exchange, contract.exchange),
orderid=str(execution.orderId),
tradeid=str(execution.execId),
direction=DIRECTION_IB2VT[execution.side],
price=execution.price,
volume=execution.shares,
datetime=dt,
gateway_name=self.gateway_name,
)
self.gateway.on_trade(trade)
def managedAccounts(self, accountsList: str):
"""
Callback of all sub accountid.
"""
super().managedAccounts(accountsList)
if not self.account:
for account_code in accountsList.split(","):
self.account = account_code
self.gateway.write_log(f"当前使用的交易账号为{self.account}")
self.client.reqAccountUpdates(True, self.account)
def historicalData(self, reqId: int, ib_bar: IbBarData):
"""
Callback of history data update.
"""
dt = datetime.strptime(ib_bar.date, "%Y%m%d %H:%M:%S")
dt = dt.replace(tzinfo=self.local_tz)
bar = BarData(symbol=self.history_req.symbol,
exchange=self.history_req.exchange,
datetime=dt,
interval=self.history_req.interval,
volume=ib_bar.volume,
open_price=ib_bar.open,
high_price=ib_bar.high,
low_price=ib_bar.low,
close_price=ib_bar.close,
gateway_name=self.gateway_name)
self.history_buf.append(bar)
def historicalDataEnd(self, reqId: int, start: str, end: str):
"""
Callback of history data finished.
"""
self.history_condition.acquire()
self.history_condition.notify()
self.history_condition.release()
def connect(self, host: str, port: int, clientid: int, account: str):
"""
Connect to TWS.
"""
if self.status:
return
self.clientid = clientid
self.account = account
self.client.connect(host, port, clientid)
self.thread.start()
self.client.reqCurrentTime()
def close(self):
"""
Disconnect to TWS.
"""
if not self.status:
return
self.status = False
self.client.disconnect()
def subscribe(self, req: SubscribeRequest):
"""
Subscribe tick data update.
"""
if not self.status:
return
if req.exchange not in EXCHANGE_VT2IB:
self.gateway.write_log(f"不支持的交易所{req.exchange}")
return
# Extract ib contract detail
ib_contract = generate_ib_contract(req.symbol, req.exchange)
if not ib_contract:
self.gateway.write_log("代码解析失败，请检查格式是否正确")
return
# Get contract data from TWS.
self.reqid += 1
self.client.reqContractDetails(self.reqid, ib_contract)
# Subscribe tick data and create tick object buffer.
self.reqid += 1
self.client.reqMktData(self.reqid, ib_contract, "", False, False, [])
tick = TickData(
symbol=req.symbol,
exchange=req.exchange,
datetime=datetime.now(self.local_tz),
gateway_name=self.gateway_name,
)
self.ticks[self.reqid] = tick
self.tick_exchange[self.reqid] = req.exchange
def send_order(self, req: OrderRequest):
"""
Send a new order.
"""
if not self.status:
return ""
if req.exchange not in EXCHANGE_VT2IB:
self.gateway.write_log(f"不支持的交易所：{req.exchange}")
return ""
if req.type not in ORDERTYPE_VT2IB:
self.gateway.write_log(f"不支持的价格类型：{req.type}")
return ""
self.orderid += 1
ib_contract = generate_ib_contract(req.symbol, req.exchange)
if not ib_contract:
return ""
ib_order = Order()
ib_order.orderId = self.orderid
ib_order.clientId = self.clientid
ib_order.action = DIRECTION_VT2IB[req.direction]
ib_order.orderType = ORDERTYPE_VT2IB[req.type]
ib_order.totalQuantity = req.volume
ib_order.account = self.account
if req.type == OrderType.LIMIT:
ib_order.lmtPrice = req.price
elif req.type == OrderType.STOP:
ib_order.auxPrice = req.price
self.client.placeOrder(self.orderid, ib_contract, ib_order)
self.client.reqIds(1)
order = req.create_order_data(str(self.orderid), self.gateway_name)
self.gateway.on_order(order)
return order.vt_orderid
def cancel_order(self, req: CancelRequest):
"""
Cancel an existing order.
"""
if not self.status:
return
self.client.cancelOrder(int(req.orderid))
def query_history(self, req: HistoryRequest):
""""""
self.history_req = req
self.reqid += 1
ib_contract = generate_ib_contract(req.symbol, req.exchange)
if req.end:
end = req.end
end_str = end.strftime("%Y%m%d %H:%M:%S")
else:
end = datetime.now()
end_str = ""
delta = end - req.start
days = min(delta.days, 180) # IB only provides 6-month data
duration = f"{days} D"
bar_size = INTERVAL_VT2IB[req.interval]
if req.exchange == Exchange.IDEALPRO:
bar_type = "MIDPOINT"
else:
bar_type = "TRADES"
self.client.reqHistoricalData(self.reqid, ib_contract, end_str,
duration, bar_size, bar_type, 1, 1,
False, [])
self.history_condition.acquire() # Wait for async data return
self.history_condition.wait()
self.history_condition.release()
history = self.history_buf
self.history_buf = [] # Create new buffer list
self.history_req = None
return history
def load_contract_data(self):
""""""
f = shelve.open(self.data_filepath)
self.contracts = f.get("contracts", {})
f.close()
for contract in self.contracts.values():
self.gateway.on_contract(contract)
self.gateway.write_log("本地缓存合约信息加载成功")
def save_contract_data(self):
""""""
f = shelve.open(self.data_filepath)
f["contracts"] = self.contracts
f.close()
class InstanceRegistry:
"""Keeps track of running instances.
The InstanceRegistry is a simple in-memory database that stores
information about running instances of compute elements.
"""
def __init__(self, expected_instances: List[str]) -> None:
"""Construct an empty InstanceRegistry.
Args:
expected_instances: List of instance names expected to
register.
"""
self.__status = dict() # type: Dict[Reference, _InstanceStatus]
self.__deregistered_one = Condition() # doubles as lock for __status
self.__locations = dict() # type: Dict[Reference, List[str]]
self.__ports = dict() # type: Dict[Reference, List[Port]]
for instance_name in expected_instances:
self.__status[Reference(instance_name)] = _InstanceStatus.EXPECTED
def add(self, name: Reference, locations: List[str],
ports: List[Port]) -> None:
"""Add an instance to the registry.
Args:
name: Name of the instance.
locations: Network locations where it can be reached.
ports: List of ports of this instance.
Raises:
ValueError: If an instance with this name has already been
registered.
"""
if name in self.__locations or name in self.__ports:
raise ValueError('Instance already registered')
with self.__deregistered_one:
self.__status[name] = _InstanceStatus.REGISTERED
self.__locations[name] = locations
self.__ports[name] = ports
def get_locations(self, name: Reference) -> List[str]:
"""Retrieves the locations of a registered instance.
Args:
name: The name of the instance to get the location of.
Raises:
KeyError: If no instance with this name was registered.
"""
return self.__locations[name]
def get_ports(self, name: Reference) -> List[Port]:
"""Retrieves the ports of a registered instance.
Args:
name: The name of the instance whose ports to retrieve.
Raises:
KeyError: If no instance with this name was registered.
"""
return self.__ports[name]
def remove(self, name: Reference) -> None:
"""Remove an instance from the registry.
Args:
name: Name of the instance to remove.
Raises:
KeyError: If the instance does not exist.
"""
del (self.__locations[name])
del (self.__ports[name])
with self.__deregistered_one:
self.__status[name] = _InstanceStatus.DEREGISTERED
self.__deregistered_one.notify()
def wait(self) -> None:
"""Waits until all instance are deregistered.
This function blocks, and returns after each expected instance
has been registered and deregistered again, signalling the end
of the simulation run.
"""
# this is called from a different thread than add/remove
def all_deregistered() -> bool:
return all(
map(lambda x: x == _InstanceStatus.DEREGISTERED,
self.__status.values()))
with self.__deregistered_one:
while not all_deregistered():
self.__deregistered_one.wait()
class RLUQueue:
def __init__(self, size):
self.size = size
self.prev = [None] * size
self.next = [None] * size
self.prev.append(size)
self.next.append(size)
self.cv = Condition()
## interface
def put(self, idx):
assert 0 <= idx < self.size
self.cv.acquire()
if self.in_list(idx):
self.remove(idx)
self.append(idx)
self.cv.notify()
self.cv.release()
def get(self, block=True):
self.cv.acquire()
if not block and self._empty():
self.cv.release()
return None
while self._empty():
self.cv.wait()
ret = self.remove(self.next[-1])
self.cv.release()
return ret
def unget(self, idx):
assert 0 <= idx < self.size
with self.cv:
assert not self.in_list(idx)
self.prepend(idx)
def pop(self, idx):
assert 0 <= idx < self.size
with self.cv:
return self.remove(idx) if self.in_list(idx) else None
def empty(self):
with self.cv:
return self._empty()
## helper
def _empty(self):
return self.prev[-1] == self.size
def in_list(self, idx):
return self.prev[idx] is not None
def remove(self, idx):
self.next[self.prev[idx]] = self.next[idx]
self.prev[self.next[idx]] = self.prev[idx]
self.prev[idx] = None
self.next[idx] = None
return idx
def append(self, idx):
self.next[idx] = self.size
self.prev[idx] = self.prev[-1]
self.prev[self.next[idx]] = idx
self.next[self.prev[idx]] = idx
def prepend(self, idx):
self.next[idx] = self.next[-1]
self.prev[idx] = self.size
self.prev[self.next[idx]] = idx
self.next[self.prev[idx]] = idx
class OSXCentralManager(NSObject, Central):
"""
CentralManager is the host handle for performing scan, connect, disconnect to peripheral(s).
After a peripheral is connected, a peripheral handle would be returned.
CBCentralManager Class Reference:
https://developer.apple.com/librarY/mac/documentation/CoreBluetooth/Reference/CBCentralManager_Class/translated_content/CBCentralManager.html
CBCentralManagerDelegate Protocol Reference:
https://developer.apple.com/librarY/mac/documentation/CoreBluetooth/Reference/CBCentralManagerDelegate_Protocol/translated_content/CBCentralManagerDelegate.html
"""
def init(self):
try:
super().__init__()
except:
super(OSXCentralManager, self).__init__()
# enable trace
self.trace.traceInstance(self)
# initialize manager with delegate
self.logger.info("Initialize CBCentralManager")
self.manager = CBCentralManager.alloc().initWithDelegate_queue_(
self, nil)
self.scanedList = []
self.connectedList = []
self.BLEReady_callback = None
self.BLEAvailableList_callback = None
self.BLEConnectedList_callback = None
self.cv = Condition()
self.ready = False
self.wait4Startup()
self._stop = Event()
return self
# decorators for condition variables
def _waitResp(func):
@wraps(func)
def wrapper(self, *args, **kwargs):
with self.cv:
self.ready = False
ret = func(self, *args, **kwargs)
while True:
self.cv.wait(0)
NSRunLoop.currentRunLoop().runMode_beforeDate_(
NSDefaultRunLoopMode, NSDate.distantPast())
if self.ready:
break
return ret
return wrapper
def _notifyResp(func):
@wraps(func)
def wrapper(self, *args, **kwargs):
ret = func(self, *args, **kwargs)
self.ready = True
with self.cv:
try:
self.cv.notify()
except Exception as e:
print e
return ret
return wrapper
@_waitResp
def wait4Startup(self):
self.logger.debug("Waiting CentralManager to be ready ...")
@python_method
def setBLEReadyCallback(self, func):
self.BLEReady_callback = func
@python_method
def setBLEAvailableListCallback(self, func):
self.BLEAvailableList_callback = func
@python_method
def setBLEConnectedListCallback(self, func):
self.BLEConnectedList_callback = func
@python_method
def updateAvailableList(self):
if self.BLEAvailableList_callback:
try:
self.BLEAvailableList_callback(self.scanedList)
except:
pass
@python_method
def updateConnectedList(self):
if self.BLEConnectedList_callback:
try:
self.BLEConnectedList_callback(self.connectedList)
except:
pass
@python_method
def loop(self, duration=0):
if duration == 0:
self.logger.info("Stop the loop by Ctrl+C ...")
startTime = datetime.now()
while True:
try:
NSRunLoop.currentRunLoop().runMode_beforeDate_(
NSDefaultRunLoopMode, NSDate.distantPast())
if duration > 0 and datetime.now() - startTime > timedelta(
seconds=duration):
break
if self._stop.isSet():
break
except KeyboardInterrupt:
print "\nEnd loop ..."
break
@python_method
def stop(self):
self._stop.set()
@python_method
def startScan(self,
withServices=[],
timeout=5,
numOfPeripherals=1,
allowDuplicates=False):
self.logger.debug("Start Scan")
if len(self.scanedList):
del self.scanedList
self.scanedList = []
if allowDuplicates:
value = NSNumber.numberWithBool_(YES)
else:
value = NSNumber.numberWithBool_(NO)
options = NSDictionary.dictionaryWithObject_forKey_(
value, CBCentralManagerScanOptionAllowDuplicatesKey)
self.manager.scanForPeripheralsWithServices_options_(nil, options)
startTime = datetime.now()
with self.cv:
while True:
self.cv.wait(0.1)
NSRunLoop.currentRunLoop().runMode_beforeDate_(
NSDefaultRunLoopMode, NSDate.distantPast())
if timeout > 0 and datetime.now() - startTime > timedelta(
seconds=timeout):
self.stopScan()
raise BLETimeoutError("Scan timeout after %s seconds!!" %
timeout)
if len(withServices):
if len(self.scanedList):
tmpList = self.scanedList[:]
self.scanedList = []
for service in withServices:
for p in tmpList:
if service in p.advServiceUUIDs:
self.scanedList.append(p)
if numOfPeripherals > 0 and len(
self.scanedList) >= numOfPeripherals:
self.logger.info("Found %s peripherals." %
len(self.scanedList))
break
self.stopScan()
# return the first found peripheral
if len(self.scanedList):
return self.scanedList[0]
return None
@python_method
def stopScan(self):
self.logger.debug("Stop Scan")
self.manager.stopScan()
@python_method
def getScanedList(self):
return self.scanedList
@python_method
def getConnectedList(self):
return self.connectedList
@python_method
@_waitResp
def retrieveConnectedPeripherals(self):
self.logger.info("Deprecated in OSX v10.9.")
return
# self.manager.retrieveConnectedPeripherals()
@python_method
def retrieveConnectedPeripheralsWithServices(self, services):
if not isinstance(services, list) and isinstance(services, str):
services = list(services.split())
known_services = NSMutableArray.alloc().init()
for service in services:
UUID = CBUUID.UUIDWithString_(service)
if UUID is not nil:
known_services.addObject_(UUID)
peripherals = self.manager.retrieveConnectedPeripheralsWithServices_(
known_services)
for p in peripherals:
if not self.findPeripheralFromList(p, self.scanedList):
temp = OSXPeripheral.alloc().init()
temp.instance = p
temp.UUID = uuid.UUID(p._.identifier.UUIDString())
temp.name = p._.name
self.scanedList.append(temp)
# update lists
self.updateAvailableList()
@python_method
def retrievePeriphersWithIdentifiers(self, identifiers):
if not isinstance(identifiers, list) and isinstance(identifiers, str):
identifiers = list(identifiers.split())
known_identifiers = NSMutableArray.alloc().init()
for identifier in identifiers:
UUID = NSUUID.alloc().UUIDWithString_(identifer)
if UUID is not nil:
known_identifiers.addObject_(UUID)
peripherals = self.manager.retrievePeripheralsWithIdentifiers_(
known_identifiers)
for p in peripherals:
if not self.findPeripheralFromList(p, self.scanedList):
temp = OSXPeripheral.alloc().init()
temp.instance = p
temp.UUID = uuid.UUID(p._.identifier.UUIDString())
temp.name = p._.name
self.scanedList.append(temp)
# update lists
self.updateAvailableList()
@python_method
def connectPeripheral(self, peripheral):
with self.cv:
self.ready = False
self.logger.debug("Connecting to Peripheral: " + str(peripheral))
options = NSDictionary.dictionaryWithObject_forKey_(
NSNumber.numberWithBool_(YES),
CBConnectPeripheralOptionNotifyOnDisconnectionKey)
peripheral.state = Peripheral.CONNECTING
self.manager.connectPeripheral_options_(peripheral.instance,
options)
while True:
self.cv.wait(0)
NSRunLoop.currentRunLoop().runMode_beforeDate_(
NSDefaultRunLoopMode, NSDate.distantPast())
if self.ready == True:
break
# return the newly added peripheral
if peripheral in self.connectedList:
return peripheral
return None
@python_method
def disconnectAllPeripherals(self):
self.logger.debug("Disconnecting all Peripherals")
for p in self.connectedList:
self.disconnectPeripheral(p)
@python_method
@_waitResp
def disconnectPeripheral(self, peripheral):
self.logger.debug("Disconnecting Peripheral: " + str(peripheral))
self.manager.cancelPeripheralConnection_(peripheral.instance)
@python_method
@staticmethod
def findPeripheralFromList(peripheral, peripherals):
name = peripheral._.name
UUID = uuid.UUID(peripheral._.identifier.UUIDString())
for p in peripherals:
if not isinstance(p, Peripheral):
return None
if name == p.name and UUID == p.UUID:
return p
return None
# CBCentralManager delegate methods
# Monitoring Connections with Peripherals
def centralManager_didConnectPeripheral_(self, central, peripheral):
self.didConnectPeripheral(central, peripheral)
@python_method
@_notifyResp
def didConnectPeripheral(self, central, peripheral):
self.logger.info(
"Peripheral %s (%s) is connected" %
(peripheral._.name, peripheral._.identifier.UUIDString()))
p = self.findPeripheralFromList(peripheral, self.scanedList)
if p:
if p not in self.connectedList:
self.connectedList.append(p)
self.scanedList.remove(p)
# update peripheral state
p.state = Peripheral.CONNECTED
# update lists
self.updateAvailableList()
self.updateConnectedList()
def centralManager_didDisconnectPeripheral_error_(self, central,
peripheral, error):
self.didDisconnectPeripheral(central, peripheral, error)
@python_method
@_notifyResp
def didDisconnectPeripheral(self, central, peripheral, error):
self.logger.info("Peripheral %s is disconnected" % peripheral._.name)
p = self.findPeripheralFromList(peripheral, self.connectedList)
if p:
p.state = Peripheral.DISCONNECTED
self.connectedList.remove(p)
if p not in self.scanedList:
self.scanedList.append(p)
# update lists
self.updateConnectedList()
def centralManager_didFailToConnectPeripheral_error_(
self, central, peripheral, error):
self.didFailtoConnectPeripheral(central, peripheral, error)
@python_method
@_notifyResp
def didFailtoConnectPeripheral(self, central, peripheral, error):
self.logger.debug("Fail to connect Peripheral %s" % peripheral._.name)
p = self.findPeripheralFromList(peripheral, self.scanedList)
if p:
p.state = Peripheral.DISCONNECTED
# update lists
self.updateAvailableList()
# Discovering and Retrieving Peripherals
def centralManager_didDiscoverPeripheral_advertisementData_RSSI_(
self, central, peripheral, advertisementData, rssi):
self.didDiscoverPeripheral(central, peripheral, advertisementData,
rssi)
@python_method
@_notifyResp
def didDiscoverPeripheral(self, central, peripheral, advertisementData,
rssi):
temp = OSXPeripheral.alloc().init()
idx = -1
p = None
try:
idx = self.scanedList.index(temp)
except ValueError:
idx = -1
except Exception as e:
self.logger.error(str(e))
if idx >= 0:
p = self.scanedList[idx]
else:
p = temp
p.instance = peripheral
p.UUID = uuid.UUID(peripheral._.identifier.UUIDString())
p.name = peripheral._.name
# handle advertisement data
# local name
if CBAdvertisementDataLocalNameKey in advertisementData:
p.advLocalName = advertisementData[CBAdvertisementDataLocalNameKey]
# manufacturer data
if CBAdvertisementDataManufacturerDataKey in advertisementData:
p.advManufacturerData = advertisementData[
CBAdvertisementDataManufacturerDataKey]
# provided services UUIDs
if CBAdvertisementDataServiceUUIDsKey in advertisementData:
p.advServiceUUIDS = []
UUIDs = advertisementData[CBAdvertisementDataServiceUUIDsKey]
for UUID in UUIDs:
p.advServiceUUIDs.append(CBUUID2String(UUID._.data))
# Tx Power Level
if CBAdvertisementDataTxPowerLevelKey in advertisementData:
p.advTxPowerLevel = int(
advertisementData[CBAdvertisementDataTxPowerLevelKey])
# ServiceData
if CBAdvertisementDataServiceDataKey in advertisementData:
p.advServiceData = advertisementData[
CBAdvertisementDataServiceDataKey]
# OverflowServiceUUIDs
if CBAdvertisementDataOverflowServiceUUIDsKey in advertisementData:
p.advOverflowServiceUUIDs = []
UUIDs = advertisementData[
CBAdvertisementDataOverflowServiceUUIDsKey]
for UUID in UUIDs:
p.advOverflowServiceUUIDs.append(CBUUID2String(UUID._.data))
# IsConnectable
if CBAdvertisementDataIsConnectable in advertisementData:
p.advIsConnectable = advertisementData[
CBAdvertisementDataIsConnectable]
# SolicitedServiceUUIDs
if CBAdvertisementDataSolicitedServiceUUIDsKey in advertisementData:
p.advSolicitedServiceUUIDs = []
UUIDs = advertisementData[
CBAdvertisementDataSolicitedServiceUUIDsKey]
for UUID in UUIDs:
p.advSolicitedServiceUUIDs.append(CBUUID2String(UUID._.data))
# RSSI
p.rssi = rssi
if p not in self.scanedList:
self.scanedList.append(p)
self.logger.info("Found Peripheral %s", peripheral._.name)
self.logger.info("RSSI: %d", rssi)
self.logger.info("UUID: %s", peripheral._.identifier.UUIDString())
self.logger.info("State: %s", peripheral._.state)
# update lists
self.updateAvailableList()
def centralManager_didRetrieveConnectedPeripherals_(
self, central, peripherals):
self.didRetrieveConnectedPeripherals(central, peripherals)
@python_method
@_notifyResp
def didRetrieveConnectedPeripherals(self, central, peripherals):
self.logger.info("Deprecated in OSX v10.9.")
return
self.logger.info("didRetrieveConnectedPeripherals")
for p in peripherals:
print p
def centralManager_didRetrievePeripherals_(self, central, peripherals):
self.logger.info("didRetrievePeripherals")
# Monitoring Changes to the Central Manager's State
def centralManagerDidUpdateState_(self, central):
self.didUpdateState(central)
@python_method
@_notifyResp
def didUpdateState(self, central):
ble_state = central._.state
if ble_state == CBCentralManagerStateUnkown:
self.logger.debug("CentralManager State: Unkown")
elif ble_state == CBCentralManagerStateResetting:
self.logger.debug("CentralManager State: Resetting")
elif ble_state == CBCentralManagerStateUnsupported:
self.logger.debug("CentralManager State: Unsupported")
elif ble_state == CBCentralManagerStateUnauthorized:
self.logger.debug("CentralManager State: Unauthorized")
elif ble_state == CBCentralManagerStatePoweredOff:
self.logger.debug("CentralManager State: PoweredOff")
elif ble_state == CBCentralManagerStatePoweredOn:
self.logger.debug("CentralManager State: PoweredOn")
self.logger.info("BLE is ready!!")
if self.BLEReady_callback:
try:
self.BLEReady_callback()
except Exception as e:
self.logger.error("BLEReady_callback error")
self.logger.error(str(e))
else:
self.logger.info("Cannot get CBCentralManager's state!!")
raise BLECentralManagerStateError
def centralManager_willRestoreState_(self, central, dict):
pass
class EarsGPIO(Ears):
ENCODERS_CHANNELS = [24, 23]
MOTOR_CHANNELS = [[12, 11], [10, 9]]
ENABLE_CHANNELS = [5, 6]
HOLES = Ears.STEPS
FORWARD_INCREMENT = 1
BACKWARD_INCREMENT = -1
def __init__(self):
self.running = [False, False]
self.targets = [0, 0]
self.encoder_cv = Condition()
self.positions = [0, 0]
self.directions = [1, 1]
self.executor = ThreadPoolExecutor(max_workers=1)
self.lock = asyncio.Lock()
GPIO.setwarnings(True)
GPIO.setmode(GPIO.BCM)
for channel in EarsGPIO.ENCODERS_CHANNELS:
GPIO.setup(channel, GPIO.IN)
try:
GPIO.add_event_detect(channel,
GPIO.RISING,
callback=self._encoder_cb)
except RuntimeError:
print('Could not set edge detection (please reboot ?)')
sys.exit(1)
for pairs in EarsGPIO.MOTOR_CHANNELS:
for channel in pairs:
GPIO.setup(channel, GPIO.OUT)
GPIO.output(channel, GPIO.LOW)
for channel in EarsGPIO.ENABLE_CHANNELS:
GPIO.setup(channel, GPIO.OUT)
GPIO.output(channel, GPIO.HIGH)
def _encoder_cb(self, channel):
"""
Callback from GPIO.
Thread: Rpi.GPIO event thread
"""
if channel == EarsGPIO.ENCODERS_CHANNELS[0]:
ear = 0
elif channel == EarsGPIO.ENCODERS_CHANNELS[1]:
ear = 1
with self.encoder_cv:
direction = self.directions[ear]
if direction == 0:
self.positions[ear] = None
(loop, callback) = self.callback
loop.call_soon_threadsafe(lambda ear=ear: callback(ear))
else:
self.positions[ear] = (self.positions[ear] +
direction) % EarsGPIO.HOLES
if self.targets[ear] == None: # reset mode
self.encoder_cv.notify()
else:
if self.positions[ear] == self.targets[ear]:
self._stop_motor(ear)
self.encoder_cv.notify()
elif self.positions[ear] == (self.targets[ear] %
EarsGPIO.HOLES):
if self.targets[ear] >= EarsGPIO.HOLES:
self.targets[
ear] = self.targets[ear] - EarsGPIO.HOLES
elif self.targets[ear] < 0:
self.targets[
ear] = self.targets[ear] + EarsGPIO.HOLES
def _stop_motor(self, ear):
"""
Stop motor by changing the channels GPIOs.
Thread: RPi.GPIO event
"""
for channel in EarsGPIO.MOTOR_CHANNELS[ear]:
GPIO.output(channel, GPIO.LOW)
self.running[ear] = False
self.directions[ear] = 0
def _start_motor(self, ear, direction):
"""
Start motor for given ear.
ear = 0 or 1
direction = 1 or -1
Threads: main loop or executor
"""
dir_ix = int((1 - direction) / 2)
GPIO.output(EarsGPIO.MOTOR_CHANNELS[ear][1 - dir_ix], GPIO.LOW)
GPIO.output(EarsGPIO.MOTOR_CHANNELS[ear][dir_ix], GPIO.HIGH)
self.running[ear] = True
self.directions[ear] = direction
def on_move(self, loop, callback):
self.callback = (loop, callback)
async def reset_ears(self, target_left, target_right):
async with self.lock:
await asyncio.get_event_loop().run_in_executor(
self.executor, self._do_reset_ears, target_left, target_right)
def _do_reset_ears(self, target_left, target_right):
"""
Reset ears by running a detection and ignoring the result.
Thread: executor
"""
self.positions = [None, None]
self._run_detection(target_left, target_right)
def _run_detection(self, target_left, target_right):
"""
Run detection of any ear in unknown position.
Thread: executor
"""
for ear in [0, 1]:
if self.positions[ear] == None:
self.positions[ear] = 0
self.targets[ear] = None
self._start_motor(ear, EarsGPIO.FORWARD_INCREMENT)
start = time.time()
previous_risings = [start, start]
with self.encoder_cv:
current_positions = self.positions.copy()
while self.running[0] or self.running[1]:
if self.encoder_cv.wait(0.3):
# Got a signal
now = time.time()
for ear in range(2):
if self.targets[ear] == None and self.positions[
ear] != current_positions[ear]:
delta = now - previous_risings[ear]
if delta > 0.4:
# passed the missing hole
if target_left != None and ear == Ears.LEFT_EAR:
self.targets[ear] = target_left
elif target_right != None and ear == Ears.RIGHT_EAR:
self.targets[ear] = target_right
else:
self.targets[ear] = (
self.directions[ear] -
self.positions[ear]) % EarsGPIO.HOLES
self.positions[ear] = self.directions[ear]
current_positions[ear] = self.positions[ear]
previous_risings[ear] = now
else:
# Got no signal.
now = time.time()
for ear in range(2):
if self.targets[ear] == None:
delta = now - previous_risings[ear]
if delta > 0.4:
# At missing hole
if target_left != None and ear == Ears.LEFT_EAR:
self.targets[ear] = target_left
elif target_right != None and ear == Ears.RIGHT_EAR:
self.targets[ear] = target_right
else:
self.targets[ear] = (
-self.positions[ear]) % EarsGPIO.HOLES
self.positions[ear] = 0
return self.positions.copy()
async def move(self, motor, delta, direction):
await self.go(motor, self.targets[motor] + delta, direction)
async def wait_while_running(self):
await asyncio.get_event_loop().run_in_executor(
self.executor, self._do_wait_while_running)
def _do_wait_while_running(self):
"""
Wait until motors are no longer running, using a condition variable.
Thread: executor
"""
with self.encoder_cv:
while self.running[0] or self.running[1]:
self.encoder_cv.wait()
async def detect_positions(self):
"""
Get the position of the ears, running a detection if required.
"""
async with self.lock:
if self.positions[0] == None or self.positions[1] == None:
return await asyncio.get_event_loop().run_in_executor(
self.executor, self._run_detection, None, None)
return (self.positions[0], self.positions[1])
async def go(self, ear, position, direction):
"""
Go to a specific position.
If direction is 0, turn forward, otherwise, turn backward
If position is not within 0-16, it represents additional turns.
For example, 17 means to position the ear at 0 after at least a complete turn.
"""
async with self.lock:
# Return ears to a known state
if self.positions[0] == None or self.positions[1] == None:
await asyncio.get_event_loop().run_in_executor(
self.executor, self._run_detection, 0, 0)
self.targets[ear] = position
if direction:
dir = EarsGPIO.BACKWARD_INCREMENT
else:
dir = EarsGPIO.FORWARD_INCREMENT
if self.positions[ear] == self.targets[ear] % EarsGPIO.HOLES:
if self.targets[ear] >= EarsGPIO.HOLES:
self.targets[ear] = self.targets[ear] - EarsGPIO.HOLES
elif self.targets[ear] < 0:
self.targets[ear] = self.targets[ear] + EarsGPIO.HOLES
else:
return # we already are at requested position
self._start_motor(ear, dir)
class TileServer(object):
'''Base implementation for a tile provider.
Check GoogleTileServer and YahooTileServer if you intend to use more
'''
provider_name = 'unknown'
providers = dict()
@staticmethod
def register(cls):
TileServer.providers[cls.provider_name] = cls
def __init__(self, poolsize=TILESERVER_POOLSIZE):
self.cache_path = join(dirname(__file__), 'cache', self.provider_name)
if not isdir(self.cache_path):
makedirs(self.cache_path)
black = Loader.image(join('documents', 'black.png'))
#Loader._loading_image = black
self.q_in = deque()
self.q_out = deque()
self.q_count = 0
self.c_in = Condition()
self.workers = []
self.poolsize = poolsize
self.uniqid = 1
self.want_close = False
self.available_maptype = dict(roadmap='Roadmap')
self.hcsvnt = Loader.image(join('documents', 'hcsvnt.png'))
def start(self):
'''Start all the workers
'''
for i in xrange(self.poolsize):
self.create_worker()
def create_worker(self):
'''Create a new worker, and append to the list of current workers
'''
thread = Thread(target=self._worker_run,
args=(self.c_in, self.q_in, self.q_out))
thread.daemon = True
thread.start()
self.workers.append(thread)
def stop(self, wait=False):
'''Stop all workers
'''
self.want_close = True
if wait:
for x in self.workers:
x.join()
def post_download(self, filename):
'''Callback called after the download append. You can use it for
doing some image processing, like cropping
.. warning::
This function is called inside a worker Thread.
'''
pass
def to_filename(self, nx, ny, zoom, maptype, format):
fid = self.to_id(nx, ny, zoom, maptype, format)
hash = fid[0:2]
return join(self.cache_path, hash, fid)
def to_id(self, nx, ny, zoom, maptype, format):
return '%d_%d_%d_%s.%s' % (nx, ny, zoom, maptype, format)
def exist(self, nx, ny, zoom, maptype, format='png'):
filename = self.to_filename(nx, ny, zoom, maptype, format)
img = Cache.get('tileserver.tiles', filename)
return bool(img)
def get(self, nx, ny, zoom, maptype, format='png'):
'''Get a tile
'''
filename = self.to_filename(nx, ny, zoom, maptype, format)
img = Cache.get('tileserver.tiles', filename)
# check if the tile is already being loaded
if img is False:
return None
# check if the tile exist in the cache
if img is not None:
return img
# no tile, ask to workers to download
Cache.append('tileserver.tiles', filename, False)
self.q_count += 1
self.q_in.append((nx, ny, zoom, maptype, format))
self.c_in.acquire()
self.c_in.notify()
self.c_in.release()
return None
def update(self):
'''Must be called to get pull image from the workers queue
'''
pop = self.q_out.pop
while True:
try:
filename, image = pop()
self.q_count -= 1
except:
return
Cache.append('tileserver.tiles', filename, image)
def _worker_run(self, c_in, q_in, q_out):
'''Internal. Main function for every worker
'''
do = self._worker_run_once
while not self.want_close:
try:
do(c_in, q_in, q_out)
except:
Logger.exception(
'TileServerWorker: Unknown exception, stop the worker')
return
def _worker_run_once(self, c_in, q_in, q_out):
'''Internal. Load one image, process, and push.
'''
# get one tile to process
try:
nx, ny, zoom, maptype, format = q_in.pop()
except:
c_in.acquire()
c_in.wait()
c_in.release()
return
# check if the tile already have been downloaded
filename = self.to_filename(nx, ny, zoom, maptype, format)
loaded = True
if not isfile(filename):
loaded = False
# calculate the good tile index
tz = pow(2, zoom)
lx, ly = unit_to_latlon(2.0 * (nx + 0.5) / tz - 1,
1 - 2.0 * (ny + 0.5) / tz)
lx, ly = map(fix180, (lx, ly))
# get url for this specific tile
url = self.geturl(nx=nx,
ny=ny,
lx=lx,
ly=ly,
tilew=256,
tileh=256,
zoom=zoom,
format=format,
maptype=maptype)
for i in xrange(1, 3):
try:
conn = urlopen("http://%s%s" % (self.provider_host, url))
except Exception, ex:
print "ERROR IN %s/%s \n%s" % (self.provider_host, url,
str(ex))
continue
try:
data = conn.read()
conn.close()
except Exception, e:
print 'Exception %s' % (url)
Logger.error('TileServer: "%s": %s' % (str(e), filename))
Logger.error('TileServer: "%s": URL=%s' % (str(e), url))
continue
# discard error messages
if data[:5] == "<?xml":
msg = ""
try:
msg = data[data.index("<ServiceException>") +
18:data.index("</ServiceException")]
except:
pass
Logger.error('Tileserver: Received error fetching %s: %s' %
(url, msg))
continue
# write data on disk
try:
directory = sep.join(filename.split(sep)[:-1])
if not isdir(directory):
try:
mkdir(directory)
except:
pass # that was probably just a concurrency error - if dir is missing, all threads report it
with open(filename, 'wb') as fd:
fd.write(data)
except:
Logger.exception('Tileserver: Unable to write %s' %
filename)
continue
# post processing
self.post_download(filename)
loaded = True
break
