class MWPixelClock(object):
def __init__(self, conduitName):
self.conduit = Conduit(conduitName)
self.conduit.initialize()
self.conduit.register_local_event_code(0,'#stimDisplayUpdate')
self.conduit.register_callback_for_name('#stimDisplayUpdate', self.receive_event)
self.codes = []
self.cond = Condition()
self.maxCodes = 100
def receive_event(self, event):
for s in event.data:
if s is None:
continue
if s.has_key('bit_code'):
self.cond.acquire()
self.codes.append((s['bit_code'],event.time/1000000.))
# if len(self.codes) > 2:
# #logging.debug('MW bit_code = %i' % s['bit_code'])
# #print s['bit_code']
# #logging.debug("MW Delta: %s" % delta_code(self.codes[-1][0], self.codes[-2][0]))
while len(self.codes) > self.maxCodes:
self.codes.pop(0)
self.cond.notifyAll()
self.cond.release()
class _CoreScheduleThread(Thread):
def __init__(self,threadpool):
self.scheduletasks = [];
self.tasklock = Lock();
self.condition = Condition(Lock())
self.threadpool = threadpool
Thread.__init__(self)
def run(self):
while True:
self.condition.acquire()
if len(self.scheduletasks) == 0:
self.condition.wait();
else:
task = self.scheduletasks.pop(0)
if dates.current_timestamps()>=task.nexttime:
self.threadpool.execute(task.function,*task.args,**task.kwargs)
task.nexttime = dates.current_timestamps()+task.period;
else:
self.condition.wait(task.nexttime-dates.current_timestamps())
self.addtask(task)
self.condition.release()
def addtask(self,task): # copy on write
self.tasklock.acquire()
tasks = [ t for t in self.scheduletasks ]
tasks.append(task)
tasks.sort(key=lambda task:task.nexttime)
self.scheduletasks = tasks
self.tasklock.release()
class Node(object):
STATE_INIT = 0
STATE_STARTING = 1
STATE_RUNNING = 2
STATE_STOPPED = 3
STATE_PARTITIONED = 4
STATE_FAILED = 5
state_str = {STATE_INIT: "Initial",
STATE_STARTING: "Starting",
STATE_RUNNING: "Running",
STATE_STOPPED: "Stopped",
STATE_FAILED: "Failed"}
def __init__(self, node_id):
self.node_id = node_id
self.state = Node.STATE_INIT
self.cv_lock = Lock()
self.cv = Condition(self.cv_lock)
def wait_for_state(self, state):
self.cv.acquire()
while self.state not in (state, Node.STATE_FAILED):
self.cv.wait()
self.cv.release()
if self.state != state:
raise ChistributedException("Node entered failed state while waiting for state %s" % Node.state_str[state])
def set_state(self, state):
self.cv.acquire()
self.state = state
self.cv.notify_all()
self.cv.release()
class SimpleRemoteCall(object):
returncode = None
returned = False
def signal(self):
self.cond.acquire()
self.returned = True
self.cond.notify()
self.cond.release()
def errorHandler(self, error):
self.returncode = error
self.signal()
def successHandler(self, object):
self.returncode = object
self.signal()
def __call__(*arg, **kwargs):
pass
def __init__(self, remoteReference, *arg, **kwargs):
self.cond = Condition()
deref = remoteReference.callRemote(*arg, **kwargs)
deref.addCallbacks(self.successHandler, self.errorHandler)
def wait(self, timeout = None):
self.cond.acquire()
if not self.returned:
self.cond.wait(timeout)
self.cond.release()
if not self.returned:
raise TimeExpired('timeout')
def process_sequence(self, sequence):
"""
@param sequence: the movement sequence to execute
@return: dictionary of sensor readings
"""
self.log.debug('process sequence')
# parse the json message
jseq = json.loads(sequence)
moveseq = jseq['sequence']
for moves in moveseq:
self.log.debug('processing move %s' % moves)
condition = Condition()
thread_count = 0
for move in moves:
thread_count += 1
servo_instruction = self.process_move(move)
move_servo_thread = ServoThread(servo_instruction, self.pwm_queue, condition, thread_count, self.update_current_pulse)
move_servo_thread.setDaemon(False)
move_servo_thread.setName('Servo %d' % servo_instruction['channel'])
move_servo_thread.start()
condition.acquire()
if thread_count > 0:
condition.wait()
# wait for all threads to finish before doing next loop
condition.release()
class DataQueue ( Queue ):
def __init__ ( self, count ):
Queue.__init__(self)
self._count = count
self._cond = Condition()
def get ( self, block = True, timeout = None ):
self._cond.acquire()
if self.empty():
if not self._count:
raise Empty()
elif block:
self._cond.wait(timeout)
if self.empty() and not self._count:
self._cond.release()
raise Empty()
self._cond.release()
return Queue.get(self, block, timeout)
def put ( self, data ):
self._cond.acquire()
self._count = self._count - 1
Queue.put(self, data)
if self._count:
self._cond.notify()
else:
self._cond.notifyAll()
self._cond.release()
def remain ( self ):
return self._count + self.unfinished_tasks
class MessageQueue(object):
"""The message queue used internally by Gossip."""
def __init__(self):
self._queue = deque()
self._condition = Condition()
def pop(self):
self._condition.acquire()
try:
while len(self._queue) < 1:
self._condition.wait()
return self._queue.pop()
finally:
self._condition.release()
def __len__(self):
return len(self._queue)
def __deepcopy__(self, memo):
newmq = MessageQueue()
newmq._queue = copy.deepcopy(self._queue, memo)
return newmq
def appendleft(self, msg):
self._condition.acquire()
try:
self._queue.appendleft(msg)
self._condition.notify()
finally:
self._condition.release()
class External_command(object):
work_available = None
work = None
def __init__(self, command, max_workers = _MAX_WORKERS):
self.work_available = Condition()
self.work = []
for i in range(0, max_workers):
_Worker(command, self)
def process_command(self, data, result_callback, *callback_parameters):
wi = Work_item(tuple(data), result_callback, callback_parameters)
self.work_available.acquire()
self.work.append(wi)
self.work_available.notify()
self.work_available.release()
return wi
def shutdown(self):
self.work_available.acquire()
self.work.append(None)
self.work_available.notify()
self.work_available.release()
def process_result(self, result_callback, result, *callback_parameters):
try:
result_callback(result, *callback_parameters)
except Exception as ex:
if isinstance(ex, SystemExit):
raise
dump_exception('External_command: unhandled exception in external command results callback')
class ZumyROS:
def __init__(self):
self.zumy = Zumy()
rospy.init_node("zumy_ros")
self.cmd = (0, 0)
rospy.Subscriber("cmd_vel", Twist, self.cmd_callback, queue_size=1)
self.lock = Condition()
self.rate = rospy.Rate(30.0)
self.name = socket.gethostname()
self.heartBeat = rospy.Publisher("/" + self.name + "/heartBeat", String, queue_size=5)
self.imu_pub = rospy.Publisher("/" + self.name + "/imu", Imu, queue_size=1)
self.imu_count = 0
self.timestamp = time.time()
self.publisher = rospy.Publisher("from_zumy", Float64, queue_size=1)
self.msg = None
def cmd_callback(self, msg):
self.msg = msg
self.timestamp = time.time()
lv = 0.6
la = 0.4
v = msg.linear.x
print v
a = msg.angular.z
print a
r = lv * v + la * a
l = lv * v - la * a
self.lock.acquire()
self.cmd = (l, r)
self.lock.release()
def run(self):
while not rospy.is_shutdown():
time_now = time.time()
if time_now - self.timestamp > 0.5:
self.cmd = (0, 0)
self.lock.acquire()
self.zumy.cmd(*self.cmd)
imu_data = self.zumy.read_imu()
self.lock.release()
imu_msg = Imu()
imu_msg.header = Header(self.imu_count, rospy.Time.now(), self.name)
imu_msg.linear_acceleration.x = 9.81 * imu_data[0]
imu_msg.linear_acceleration.y = 9.81 * imu_data[1]
imu_msg.linear_acceleration.z = 9.81 * imu_data[2]
imu_msg.angular_velocity.x = 3.14 / 180.0 * imu_data[3]
imu_msg.angular_velocity.y = 3.14 / 180.0 * imu_data[4]
imu_msg.angular_velocity.z = 3.14 / 180.0 * imu_data[5]
self.imu_pub.publish(imu_msg)
self.heartBeat.publish("I am alive from Glew")
if self.msg != None:
self.publisher.publish(self.msg.linear.y)
self.rate.sleep()
# If shutdown, turn off motors
self.zumy.cmd(0, 0)
class External_command(object):
work_available = None
work = None
def __init__(self, command, max_workers = _MAX_WORKERS):
self.work_available = Condition()
self.work = []
for i in range(0, max_workers):
_Worker(command, self)
def process_command(self, data, result_callback, *callback_parameters):
wi = Work_item(tuple(data), result_callback, callback_parameters)
self.work_available.acquire()
self.work.append(wi)
self.work_available.notify()
self.work_available.release()
return wi
def shutdown(self):
self.work_available.acquire()
self.work.append(None)
self.work_available.notify()
self.work_available.release()
def process_result(self, result_callback, result, *callback_parameters):
try:
result_callback(result, *callback_parameters)
except:
print datetime.now(), 'External_command: unhandled exception in external command results callback'
print '-' * 70
print_exc(file = stdout)
print '-' * 70
stdout.flush()
class ZEventLoopThread(object):
def __init__(self, cb=None):
self._event_loop = None
self._thr = Thread(target=self._thread_func)
self._guard = Condition(Lock())
self._thr_callback = cb
def start_loop(self):
self._thr.start()
self._guard.acquire()
while self._event_loop is None:
self._guard.wait()
return self._event_loop
def _thread_func(self):
# Note: create event loop in the new thread
# instead in the constructor
event_loop = ZEventLoop()
if self._thr_callback:
self._thr_callback(event_loop)
with self._guard:
self._event_loop = event_loop
self._guard.notify()
event_loop.loop()
class RepceJob(object):
"""class representing message status we can use
for waiting on reply"""
def __init__(self, cbk):
"""
- .rid: (process-wise) unique id
- .cbk: what we do upon receiving reply
"""
self.rid = (os.getpid(), thread.get_ident(), time.time())
self.cbk = cbk
self.lever = Condition()
self.done = False
def __repr__(self):
return ':'.join([str(x) for x in self.rid])
def wait(self):
self.lever.acquire()
if not self.done:
self.lever.wait()
self.lever.release()
return self.result
def wakeup(self, data):
self.result = data
self.lever.acquire()
self.done = True
self.lever.notify()
self.lever.release()
class SharedCell(object):
"""Shared data for the producer/consumer problem."""
def __init__(self):
self._data = -1
self._writeable = True
self._condition = Condition()
def setData(self, data):
"""Producer's method to write to shared data."""
self._condition.acquire()
while not self._writeable:
self._condition.wait()
print "%s setting data to %d" % \
(currentThread().getName(), data)
self._data = data
self._writeable = False
self._condition.notify()
self._condition.release()
def getData(self):
"""Consumer's method to read from shared data."""
self._condition.acquire()
while self._writeable:
self._condition.wait()
print "%s accessing data %d" % \
(currentThread().getName(), self._data)
self._writeable = True
self._condition.notify()
self._condition.release()
return self._data
class LockedDirectories(object):
"""
Class that keeps a list of locked directories
"""
def __init__(self):
self.dirs = set()
self.cv = Condition()
def run_in(self, dir_):
"""
Start running in the directory and lock it
"""
self.cv.acquire()
while dir_ in self.dirs:
self.cv.wait()
self.dirs.add(dir_)
self.cv.release()
def done(self, dir_):
"""
Finished with the directory, unlock it
"""
self.cv.acquire()
self.dirs.remove(dir_)
self.cv.notify_all()
self.cv.release()
class Queue(object):
def __init__(self):
self.lock = Lock()
self.cond = Condition(self.lock)
self.data = []
self.live = True
def put(self, values):
self.cond.acquire()
self.data.extend(values)
self.cond.notify()
self.cond.release()
def get(self):
if not self.cond.acquire(False):
return []
self.cond.wait()
result = self.data
self.data = []
self.cond.release()
return result
def close(self):
self.cond.acquire()
self.live = False
self.cond.notify()
self.cond.release()
class NotificationQueue(Queue):
""" A queue class that notifies waiting threads when the queue is available for
reading."""
def __init__(self):
""" Constructor."""
Queue.__init__(self)
self.lock = Condition()
def get(self):
""" Get an item from the queue. This method is thread-safe.
Method must be called on a non-empty queue.
Return : the first item from the queue
Raises: a EmptyQueue exeception
"""
return Queue.get(self, False)
def put(self, item):
""" Add a new item to the queue. This method is thread-safe.
Threads that are waiting for get an item are notified.
item: new item to add to the queue.
Return : None
"""
self.lock.acquire()
Queue.put(self, item)
self.lock.notify()
self.lock.release()
def acquire(self):
self.lock.acquire()
def release(self):
self.lock.release()
def wait(self):
self.lock.wait()
def wait_statement(self, wtime, N=5):
'''
'''
start_time = time.time()
if self.debug:
time.sleep(1)
else:
if isinstance(wtime, str):
wtime = float(self._get_interpolation_value(wtime))
if wtime > N:
c = Condition()
c.acquire()
wd = WaitDialog(wtime=wtime,
condition=c,
parent=self)
do_later(wd.edit_traits, kind='livemodal')
c.wait()
c.release()
do_later(wd.close)
else:
time.sleep(wtime)
msg = 'actual wait time %0.3f s' % (time.time() - start_time)
self.log_statement(msg)
class ConnectorPool:
""" A thread-safe connection pool.
TODO: Make this an actual queue, not a stack. Nomenclature is imporant
sometimes.
"""
def __init__(self, Connector, count=20, user=None, password=None, db=None):
self.pool = [ Connector(user, password, db) for _ in range(count) ]
self.C_pool = Condition()
def dequeue(self):
""" Get connector.
:rtype: seaice.SeaIceConnector.SeaIceConnector
"""
self.C_pool.acquire()
while len(self.pool) == 0:
self.C_pool.wait()
db_con = self.pool.pop()
self.C_pool.release()
return db_con
def enqueue(self, db_con):
""" Release connector.
:param db_con: The connector.
:type db_con: seaice.SeaIceConnector.SeaIceConnector
"""
self.C_pool.acquire()
self.pool.append(db_con)
self.C_pool.notify()
self.C_pool.release()
class ZumyROS:
def __init__(self):
rospy.init_node('zumy_ros')
self.cmd = (0,0)
rospy.Subscriber('cmd_vel', Twist, self.cmd_callback)
self.lock = Condition()
self.rate = rospy.Rate(30.0)
self.zumy = Zumy()
self.name = socket.gethostname()
self.heartBeat = rospy.Publisher('/' + self.name + '/heartBeat', String, queue_size=5)
def cmd_callback(self, msg):
lv = 0.6
la = 0.4
v = msg.linear.x
a = msg.angular.z
r = lv*v + la*a
l = lv*v - la*a
self.lock.acquire()
self.cmd = (l,r)
self.lock.release()
def run(self):
while not rospy.is_shutdown():
self.lock.acquire()
self.zumy.cmd(*self.cmd)
self.heartBeat.publish("I am alive")
self.lock.release()
self.rate.sleep()
self.zumy.cmd(0,0)
class TmThread(Thread):
def __init__(self, target, args): # name, event_for_wait, event_for_set
Thread.__init__(self) # , target=target, args=args)
self.__target = target
self.__args = args
self.__kwargs = None
self.priority = 0
self.time = 0
#flag to Finish thread
self.Finishd = False
# Explicitly using Lock over RLock since the use of self.Finishd
# break reentrancy anyway, and I believe using Lock could allow
# one thread to Finish the worker, while another resumes; haven't
# checked if Condition imposes additional limitations that would
# prevent that. In Python 2, use of Lock instead of RLock also
# boosts performance.
self.Finish_cond = Condition(Lock())
def Finish(self):
self.Finishd = True
# If in sleep, we acquire immediately, otherwise we wait for thread
# to release condition. In race, worker will still see self.Finishd
# and begin waiting until it's set back to False
self.Finish_cond.acquire()
#should just resume the thread
def resume(self):
self.Finishd = False
# Notify so thread will wake after lock released
self.Finish_cond.notify()
# Now release the lock
self.Finish_cond.release()
class PeriodicWorker(Thread):
def __init__(self, func, config, duration=600, logger=cherrypy.log, logDB=None):
# use default RLock from condition
# Lock wan't be shared between the instance used only for wait
# func : function or callable object pointer
self.wakeUp = Condition()
self.stopFlag = False
self.taskFunc = func
self.config = config
self.duration = duration
self.logger = logger
self.logDB = logDB
try:
name = func.__name__
print(name)
except AttributeError:
name = func.__class__.__name__
print(name)
Thread.__init__(self, name=name)
cherrypy.engine.subscribe('start', self.start, priority=100)
cherrypy.engine.subscribe('stop', self.stop, priority=100)
def stop(self):
self.wakeUp.acquire()
self.stopFlag = True
self.wakeUp.notifyAll()
self.wakeUp.release()
def run(self):
while not self.stopFlag:
self.wakeUp.acquire()
try:
self.taskFunc(self.config)
self.heartBeatInfoToLogDB()
except Exception as e:
self.logger.error("Periodic Thread ERROR %s.%s %s"
% (getattr(e, "__module__", "__builtins__"),
e.__class__.__name__, str(e)))
traceMsg = traceback.format_exc()
for line in traceMsg.rstrip().split("\n"):
self.logger.error(" " + line)
self.heartBeatErrorToLogDB(traceMsg)
self.wakeUp.wait(self.duration)
self.wakeUp.release()
def heartBeatInfoToLogDB(self):
if self.logDB:
self.logDB.delete(mtype="error", this_thread=True)
self.logDB.post(mtype="info")
return
def heartBeatErrorToLogDB(self, msg):
if self.logDB:
self.logDB.post(msg=msg, mtype="error")
return
class SSHProtocol(Protocol):
def __init__(self,proxy):
print "SShProtocol %s"%self.rules
Protocol.__init__(self,self.rules)
self.proxy=proxy
self.cv=Condition()
def init(self):
print "OK initialized"
self.state=2
def handle(self,msg):
print "handle message %s"%msg
if msg['type'] == "send":
thread.start_new_thread(self.send, (msg,))
def send(self,message):
print "Sending Message %s"%message
self.release()
def release(self):
self.cv.acquire()
self.cv.notify()
self.cv.release()
def wait(self):
self.cv.acquire()
self.cv.wait()
self.cv.release()
rules={1:init,2:handle}
class Rtp_proxy_client_local(object):
is_local = True
wi_available = None
wi = None
nworkers = None
workers = None
def __init__(self, global_config, address = '/var/run/rtpproxy.sock', \
bind_address = None, nworkers = 1):
self.address = address
self.is_local = True
self.wi_available = Condition()
self.wi = []
self.nworkers = nworkers
self.workers = []
for i in range(0, self.nworkers):
self.workers.append(_RTPPLWorker(self))
def send_command(self, command, result_callback = None, *callback_parameters):
self.wi_available.acquire()
self.wi.append((command, result_callback, callback_parameters))
self.wi_available.notify()
self.wi_available.release()
def reconnect(self, address, bind_address = None):
self.shutdown()
self.address = address
self.workers = []
for i in range(0, self.nworkers):
self.workers.append(_RTPPLWorker(self))
def shutdown(self):
for rworker in self.workers:
rworker.shutdown()
self.workers = None
def test_call_self(self):
m = Mock()
cond = Condition()
self._recv_call_id = None
def event_cb(_, evt):
if evt.type == EventType.call_invite:
m()
self._recv_call_id = evt.request.call_id
# condition of event callback returning
cond.acquire()
cond.notify()
cond.release()
self.ctx.event_callback = event_cb
# start call:
# build and send invitation message
cond.acquire()
with self.ctx.lock:
msg = call.InitInvite(
self.ctx,
to_url='sip:{0[0]}:{0[1]}'.format(self.listen_address),
from_url='sip:{0[0]}:{0[1]}'.format(self.listen_address),
)
send_call_id = msg.call_id
self.ctx.call_send_init_invite(msg)
# wait event result!
cond.wait(1)
cond.release()
# assertion
self.assertEqual(m.call_count, 1)
self.assertEqual(self._recv_call_id, send_call_id)
class ServerLock:
"""
Class to be used for locking entry and exit to the venue server.
Mostly just a wrapper around a normal lock, but adds logging support.
"""
verbose = 0
def __init__(self, name = ""):
if self.verbose:
log.debug("Create server lock %s", name)
self.lock = Condition(Lock())
self.name = name
def acquire(self):
if self.verbose:
c = (traceback.extract_stack())[-2]
fileName = c[0]
line = c[1]
log.debug("Try to acquire server lock %s... %s:%s", self.name, fileName, line)
self.lock.acquire()
if self.verbose:
log.debug("Try to acquire server lock %s...done %s:%s", self.name, fileName, line)
def release(self):
if self.verbose:
c = (traceback.extract_stack())[-2]
fileName = c[0]
line = c[1]
log.debug("Releasing server lock %s %s:%s", self.name, fileName, line)
self.lock.release()
class SafeLoopArray(object):
def __init__(self , data, lengh = 10):
self._lengh = lengh
self._data = []
self._index = 0;
self._lock = Condition(Lock())
for i in range(lengh):
self._data.append(data)
def add(self , data):
self._lock.acquire()
try:
i = self._index%self._lengh
#print "self._lengh = {0},self._index={1},i={2}".format(self._lengh,self._index, i)
self._data[i] = data
self._index = self._index+1
finally:
self._lock.release()
def get(self , pos):
if (pos >= self._lengh):
print("out of range")
return None
self._lock.acquire()
try:
return self._data[pos]
finally:
self._lock.release()
class GridGenerationUpdateThread(Thread):
disconnect = False
elecgen = None
sleeper = None
def __init__(self, parent):
Thread.__init__(self)
self.disconnect = False
self.elecgen = parent
self.ngdata = ElectricityGenerationDataSource()
self.sleeper = Condition()
def stopUpdates(self):
self.disconnect = True
self.sleeper.acquire()
self.sleeper.notify()
self.sleeper.release()
def run(self):
while self.disconnect == False:
emxml = self.ngdata.DownloadRealtimeXML()
try:
self.elecgen.energyMix = self.ngdata.ParseRealtimeXML(emxml)
except Exception, exc:
trc.Error("failed to parse realtime xml")
trc.Error(str(emxml))
trc.Error(repr(exc))
trc.Error(repr(exc.message))
# go to sleep for a while - the realtime data doesn't update very
# often, so no need to download it constantly!
self.sleeper.acquire()
self.sleeper.wait(180)
self.sleeper.release()
class MultiProcBuilder:
def __init__(self, makeFile, param):
makeFileList = []
index = 1
while os.path.isfile(os.getcwd() + '\\' + makeFile.replace('.mak', 'C' + str(index) + '.mak')):
makeFileList.append(makeFile.replace('.mak', 'C' + str(index) + '.mak'))
index = index + 1
#TODO externalizar a extensoes para parametros configuraveis
makeFileLinker = makeFile.replace('.mak', 'L.mak')
self.condition = Condition()
self.output = []
self.group = MakeGroup(makeFileList, makeFileLinker, self.output, self.condition, param)
def start(self):
"""Dispara compilacao multiprocessada"""
self.group.start()
def readMsg(self):
self.condition.acquire()
out = ""
if len(self.output) == 0 and self.group.isAlive():
self.condition.wait()
while len(self.output) > 0:
out += self.output.pop(0)
self.condition.release()
return out
def getResult(self):
return self.group.status
class Barrier:
def __init__(self, num_threads):
self.num_threads = num_threads
self.count = num_threads
self.iter = 0
self.cond = Condition()
def wait(self):
self.cond.acquire()
i = self.iter
self.count -= 1
if self.count == 0:
self.iter += 1
self.count = self.num_threads
self.cond.notify_all()
self.cond.release()
return True
# Release lock and block this thread until notified/woken.
# Allow for spurious wake-ups.
while 1:
self.cond.wait(None)
if i != self.iter:
break
self.cond.release()
return False
def callFunctionInQtThread(self, functionToCall, blocking, *arguments):
"""
Calls the given function in the Qt thread. If this is used from the Qt main thread
the given function is directly called. If C{blocking} is C{True} and the method is
not called from the Qt main thread it will be ensured that the python thread is blocked
until the function is executed.
@param functionToCall: Pointer to the function that has to be executed.
@param arguments: Arguments required for the function.
@param blocking: Blocks the calling thread.
@type blocking: C{boolean}
"""
if get_ident() == self.__mainThreadIdentifier: # call directly
self._performCall(functionToCall, None, arguments)
else: # post event in the main Qt thread
event = _MyCustomEvent(functionToCall, arguments)
if blocking:
conditionVar = Condition(Lock())
conditionVar.acquire()
def _callback():
conditionVar.acquire()
conditionVar.notify()
conditionVar.release()
event.callback = _callback
qApp.postEvent(self, event)
conditionVar.wait()
conditionVar.release()
else:
qApp.postEvent(self, event)
class _SubprocessThread(Thread):
def __init__(self, executable_name, args, shell, cwd, verbose, stdout,
stderr, stdin_input):
Thread.__init__(self, name="Subprocess %s" % executable_name)
self._args = args
self._shell = shell
self._cwd = cwd
self._verbose = verbose
self._stdout = stdout
self._stderr = stderr
self._stdin_input = stdin_input
self._pid = None
self._started_cv = Condition()
self.stdout_result = None
self.stderr_result = None
self.returncode = None
self._exception = None
@property
def exception(self):
return self._exception
def run(self):
try:
self._started_cv.acquire()
stdin = PIPE if self._stdin_input else None
proc = Popen(self._args,
shell=self._shell,
cwd=self._cwd,
stdin=stdin,
stdout=self._stdout,
stderr=self._stderr)
self._pid = proc.pid
self._started_cv.notify()
self._started_cv.release()
if self._stdin_input:
proc.stdin.write(self._stdin_input)
proc.stdin.flush()
self.process_output(proc)
self.returncode = proc.returncode
except Exception as err: # pylint: disable=broad-except
self._exception = err
def get_pid(self):
self._started_cv.acquire()
while self._pid is None:
self._started_cv.wait()
self._started_cv.release()
return self._pid
def process_output(self, proc):
if self._verbose and self._stdout == PIPE and (self._stderr == PIPE or
self._stderr == STDOUT):
self.stdout_result = ""
self.stderr_result = ""
while True:
reads = [proc.stdout.fileno()]
if self._stderr == PIPE:
reads.append(proc.stderr.fileno())
ret = select(reads, [], [])
for file_no in ret[0]:
if file_no == proc.stdout.fileno():
read = output_as_str(proc.stdout.readline())
sys.stdout.write(read)
self.stdout_result += read
if self._stderr == PIPE and file_no == proc.stderr.fileno(
):
read = output_as_str(proc.stderr.readline())
sys.stderr.write(read)
self.stderr_result += read
if proc.poll() is not None:
break
else:
stdout_r, stderr_r = proc.communicate()
self.stdout_result = output_as_str(stdout_r)
self.stderr_result = output_as_str(stderr_r)
class SharedCell(object):
"""Synchronizes readers and writers around shared data,
to support concurrent reading and safe writing."""
def __init__(self, data):
"""Sets up the conditions and count of active readers."""
self.data = data
self.writing = False
self.readerCount = 0
self.okToRead = Condition()
self.okToWrite = Condition()
def beginRead(self):
"""Waits until a writer is not writing or the writers
condition queue is empty. Then increments the reader
count and notifies the next waiting reader."""
self.okToRead.acquire()
self.okToWrite.acquire()
while self.writing or len(self.okToWrite._waiters) > 0:
self.okToRead.wait()
self.readerCount += 1
self.okToRead.notify()
def endRead(self):
"""Notifies a waiting writer if there are
no active readers."""
self.readerCount -= 1
if self.readerCount == 0:
self.okToWrite.notify()
self.okToWrite.release()
self.okToRead.release()
def beginWrite(self):
"""Can write only when someone else is not
writing and there are no readers are ready."""
self.okToWrite.acquire()
self.okToRead.acquire()
while self.writing or self.readerCount != 0:
self.okToWrite.wait()
self.writing = True
def endWrite(self):
"""Notify the next waiting writer if the readers
condition queue is empty. Otherwise, notify the
next waiting reader."""
self.writing = False
if len(self.okToRead._waiters) > 0:
self.okToRead.notify()
else:
self.okToWrite.notify()
self.okToRead.release()
self.okToWrite.release()
def read(self, readerFunction):
"""Observe the data in the shared cell."""
self.beginRead()
# Enter reader's critical section
result = readerFunction(self.data)
# Exit reader's critical section
self.endRead()
return result
def write(self, writerFunction):
"""Modify the data in the shared cell."""
self.beginWrite()
# Enter writer's critical section
result = writerFunction(self.data)
# Exit writer's critical section
self.endWrite()
return result
class EventsPriorityQueue(PriorityQueue):
def __init__(self):
PriorityQueue.__init__(self)
self.first_element_changed = Condition(self.mutex)
def put_and_notify(self, item, block=True, timeout=None):
"""
Add event into events priority queue and notify thread waiting to get.
:param item: the event to be added
:param block: whether to block until the event is added
:param timeout: if not block, how long wait at most to add event
:return:
"""
log.debug("Adding Event:" + str(item))
self.not_full.acquire()
try:
first_element_before_insertion = None
if self._qsize() > 0:
first_element_before_insertion = self.queue[0]
if self.maxsize > 0:
if not block:
if self._qsize() == self.maxsize:
raise Full
elif timeout is None:
while self._qsize() == self.maxsize:
self.not_full.wait()
elif timeout < 0:
raise ValueError("'timeout' must be a non-negative number")
else:
endtime = _time() + timeout
while self._qsize() == self.maxsize:
remaining = endtime - _time()
if remaining <= 0.0:
raise Full
self.not_full.wait(remaining)
self._put(item)
self.unfinished_tasks += 1
self.not_empty.notify()
first_element_after_insertion = self.queue[0]
if first_element_before_insertion != first_element_after_insertion:
self.first_element_changed.notify()
finally:
self.not_full.release()
def get_next_element_when_ready(self):
"""
Get next event from events priority queue when it is ready:
for SystemExit event and dead event, pop from queue immediately;
for other active event, wait until next run time, then pop from queue.
:return:
"""
self.first_element_changed.acquire()
try:
isNotReady = True
while isNotReady:
if self._qsize() > 0:
first_element = self.queue[0]
if isinstance(first_element, SystemExit):
first_element = self._get()
break
if not first_element.flag_alive:
log.debug("Early termination of dead metric")
first_element = self._get()
break
timeout = (first_element.get_next_run_time() -
getUTCmillis()) / 1000.0
log.debug(
"Waiting on acquired first_element_changed LOCK " +
"for: %.2f" % timeout)
self.first_element_changed.wait(timeout)
else:
self.first_element_changed.wait()
first_element = self.queue[0]
if isinstance(first_element, SystemExit):
first_element = self._get()
break
if (first_element.get_next_run_time() - getUTCmillis()) <= 0 \
or not first_element.flag_alive:
isNotReady = False
first_element = self._get()
return first_element
finally:
self.first_element_changed.release()
class ArduinoConnectionDialog(QDialog, QObject):
signal_update = pyqtSignal()
def __init__(self, parent=None):
super(ArduinoConnectionDialog, self).__init__(parent)
self.ui = Ui_ArduinoConnectionDialog()
self.ui.setupUi(self)
self._dialog_started = Condition()
self._started = False
# Init status images
self.__ok_tick = QPixmap()
self.__ok_tick.load(create_local_full_path("images", "ok_b.png"))
self.__ok_tick = self.__ok_tick.scaled(100, 100)
self.__error_tick = QPixmap()
self.__error_tick.load(create_local_full_path("images", "error.png"))
self.__error_tick = self.__error_tick.scaled(100, 100)
# Init loading spinner
self.__spinner = QtWaitingSpinner(self.ui.widget)
self.ui.verticalLayout_2.addWidget(self.__spinner)
self.__label = QLabel()
self.__spinner.start()
# self.__spinner.setFileName(create_local_full_path("images", "connection.gif"))
# self.__spinner = QtWaitingSpinner(self)
# self.ui.loadingPanel.setMovie(self.movie)
self.resize(150, 150)
self.setFixedSize(self.size()) # Avoid Dialog resize
self.__update_callback = self.__no_change
self.signal_update.connect(self.handle_update)
# self.set_ok()
def __remove_spinner(self):
self.ui.widget.layout().removeWidget(self.__spinner)
# self.ui.verticalLayout_2.removeWidget(self.__spinner)
if self.__spinner is not None:
self.__spinner.deleteLater()
self.__spinner = None
self.ui.widget.layout().removeWidget(self.__label)
def exec_(self):
self._dialog_started.acquire()
self._started = True
self._dialog_started.notify_all()
self._dialog_started.release()
# Race condition here :/
QDialog.exec_(self)
def __no_change(self):
return
def __config_ok_tick(self):
self.__remove_spinner()
self.__label = QLabel(self.ui.widget)
self.ui.verticalLayout_2.addWidget(self.__label)
self.__label.setPixmap(self.__ok_tick)
self.ui.status_label.setText("Succesfully connected!")
self.ui.buttonBox.setStandardButtons(QDialogButtonBox.Ok)
def __config_error_tick(self):
self.__remove_spinner()
self.__label = QLabel(self.ui.widget)
self.ui.verticalLayout_2.addWidget(self.__label)
self.__label.setPixmap(self.__error_tick)
self.ui.status_label.setText(self.__error)
self.ui.buttonBox.setStandardButtons(QDialogButtonBox.Ok)
def set_ok(self):
self._dialog_started.acquire()
if not self._started:
self._dialog_started.wait()
self.__update_callback = self.__config_ok_tick
self.signal_update.emit()
def set_error(self, msg):
self._dialog_started.acquire()
if not self._started:
self._dialog_started.wait()
self.__update_callback = self.__config_error_tick
self.__error = msg
self.signal_update.emit()
def handle_update(self):
self.__update_callback()
def read(self, offset, length, callback=None):
assert 0 <= offset < offset + length <= self.total_size
idxl = offset // self.block_size
idxr = (offset + length - 1) // self.block_size
cv = Condition()
state = [idxr + 1 - idxl, None]
data_list = [None] * state[0]
for idx in xrange(idxl, idxr + 1):
rngl = max(offset, idx * self.block_size)
rngr = min(offset + length, (idx + 1) * self.block_size)
shift = rngl % self.block_size
to_read = rngr - rngl
def gcb(idx, shift, to_read):
def cb(err, obj, data):
with cv:
if state[1] is not None:
return False
if err:
state[1] = err
if callback:
callback(err, None)
else:
cv.notify()
return False
if to_read == self.block_size:
data_list[idx - idxl] = data
else:
with self.cache_lock:
self.cache.seek(
self.calc_offset(obj) + shift, os.SEEK_SET)
data_list[idx -
idxl] = self.cache.read(to_read)
state[0] = state[0] - 1
if state[0] == 0:
if callback is None:
cv.notify()
else:
callback(None, ''.join(data_list))
return False
return cb
self.pull(idx,
read_data=(to_read == self.block_size),
callback=gcb(idx, shift, to_read))
if callback is None:
cv.acquire()
while state[0] != 0:
cv.wait()
cv.release()
if state[1]:
raise state[1]
else:
assert all(x is not None for x in data_list)
return ''.join(data_list)
class Sender(object):
"""
qpid sender
"""
def __init__(self, broker, address):
self.broker = broker
self.address = address
self.address_self = str(uuid.uuid1()) + ''';{
create: always ,
delete: always ,
node: {type:queue , x-declare : {auto-delete: true}}
}'''
self.ret_msg_queue = Queue()
self.cond = Condition()
self.send_callback = {}
self.send_response = {}
self.connection = None
self.session = None
self.sender = None
def init(self):
self.connection = messaging.Connection(self.broker)
try:
self.connection.open()
self.session = self.connection.session()
self.sender = self.session.sender(self.address)
except messaging.MessagingError:
print traceback.format_exc()
return False
thread = Thread(target=self.ret_msg_processor)
thread.daemon = True
thread.start()
message_receiver = _MessageReceiver(self.broker, self.address_self)
message_receiver.start(self.ret_msg_queue, daemon=True)
return True
def destroy(self):
self.connection.close()
def send(self, message, callback=None):
if self.sender is None:
return False
message_id = uuid.uuid1().hex
message.reply_to = self.address_self
message.properties[message_id_tag] = message_id
if callback:
self.send_callback[message_id] = callback
try:
self.sender.send(message)
self.session.acknowledge()
except messaging.MessagingError:
print traceback.format_exc()
return False
return True
def sync_send(self, message, timeout=1):
if self.sender is None:
return None
message_id = uuid.uuid1().hex
message.reply_to = self.address_self
message.properties[message_id_tag] = message_id
try:
self.send_response[message_id] = None
self.sender.send(message)
self.session.acknowledge()
except messaging.MessagingError:
print traceback.format_exc()
return None
self.cond.acquire()
request_time = time.time()
while self.send_response[
message_id] is None and request_time + timeout > time.time():
self.cond.wait(0.1)
self.cond.release()
response = self.send_response.pop(message_id, None)
return response
def ret_msg_processor(self):
while 1:
message = self.ret_msg_queue.get()
if not isinstance(message, Message):
continue
message_id = message.properties.get(message_id_tag, None)
if message_id:
if message_id in self.send_response.keys():
self.cond.acquire()
self.send_response[message_id] = message
self.cond.notifyAll()
self.cond.release()
if message_id in self.send_callback.keys():
callback = self.send_callback.get(message_id, None)
if callback and hasattr(callback, '__call__'):
ret = callback(message)
if ret is None or ret:
self.send_callback.pop(message_id, None)
else:
self.send_callback.pop(message_id, None)
class DataPlane:
"""
Class defining access primitives to the data plane
Controls a list of DataPlanePort objects
"""
def __init__(self):
self.port_list = {}
# pkt_sync serves double duty as a regular top level lock and
# as a condition variable
self.pkt_sync = Condition()
# These are used to signal async pkt arrival for polling
self.want_pkt = False
self.want_pkt_port = None # What port required (or None)
self.got_pkt_port = None # On what port received?
self.packets_pending = 0 # Total pkts in all port queues
self.logger = logging.getLogger("dataplane")
self.pkt_handler = None
def port_add(self, interface_name, port_number):
"""
Add a port to the dataplane
TBD: Max packets for queue?
@param interface_name The name of the physical interface like eth1
@param port_number The port number used to refer to the port
"""
self.port_list[port_number] = DataPlanePort(interface_name,
port_number, self)
self.port_list[port_number].start()
if self.pkt_handler is not None:
self.port_list[port_number].register(self.pkt_handler)
def register(self, pkt_handler):
"""
Register pkt_handler for all ports
"""
self.pkt_handler = pkt_handler
for port_number in self.port_list.keys():
self.port_list[port_number].register(pkt_handler)
def send(self, port_number, packet, queue_id=0):
"""
Send a packet to the given port
@param port_number The port to send the data to
@param packet Raw packet data to send to port
@param queue_id The queue to send to (to be implemented)
"""
#@todo Verify port_number is in keys of port_list
self.logger.debug("Sending %d bytes to port %d" %
(len(packet), port_number))
bytes = self.port_list[port_number].send(packet, queue_id=queue_id)
if bytes != len(packet):
self.logger.error(
"Unhandled send error, length mismatch %d != %d" %
(bytes, len(packet)))
return bytes
def flood(self, packet):
"""
Send a packet to all ports
@param packet Raw packet data to send to port
"""
for port_number in self.port_list.keys():
bytes = self.port_list[port_number].send(packet)
if bytes != len(packet):
self.logger.error("Unhandled send error" +
", port %d, length mismatch %d != %d" %
(port_number, bytes, len(packet)))
def _oldest_packet_find(self):
# Find port with oldest packet
min_time = 0
min_port = -1
for port_number in self.port_list.keys():
ptime = self.port_list[port_number].timestamp_head()
if ptime:
if (min_port == -1) or (ptime < min_time):
min_time = ptime
min_port = port_number
oft_assert(min_port != -1, "Could not find port when pkts pending")
return min_port
def poll(self, port_number=None, timeout=None):
"""
Poll one or all dataplane ports for a packet
If port_number is given, get the oldest packet from that port.
Otherwise, find the port with the oldest packet and return
that packet.
@param port_number If set, get packet from this port
@param timeout If positive and no packet is available, block
until a packet is received or for this many seconds
@return The triple port_number, packet, pkt_time where packet
is received from port_number at time pkt_time. If a timeout
occurs, return None, None, None
"""
self.pkt_sync.acquire()
# Check if requested specific port and it has a packet
if port_number and len(self.port_list[port_number].packets) != 0:
pkt, time = self.port_list[port_number].dequeue(use_lock=False)
self.pkt_sync.release()
oft_assert(pkt,
"Poll: packet not found on port " + str(port_number))
return port_number, pkt, time
# Check if requested any port and some packet pending
if not port_number and self.packets_pending != 0:
port = self._oldest_packet_find()
pkt, time = self.port_list[port].dequeue(use_lock=False)
self.pkt_sync.release()
oft_assert(pkt, "Poll: oldest packet not found")
return port, pkt, time
# No packet pending; blocking call requested?
if not timeout:
self.pkt_sync.release()
return None, None, None
# Desired packet isn't available and timeout is specified
# Already holding pkt_sync; wait on pkt_sync variable
self.want_pkt = True
self.want_pkt_port = port_number
self.got_pkt_port = None
self.pkt_sync.wait(timeout)
self.want_pkt = False
if self.got_pkt_port:
pkt, time = \
self.port_list[self.got_pkt_port].dequeue(use_lock=False)
self.pkt_sync.release()
oft_assert(
pkt, "Poll: pkt reported, but not found at " +
str(self.got_pkt_port))
return self.got_pkt_port, pkt, time
self.pkt_sync.release()
self.logger.debug("Poll time out, no packet from " + str(port_number))
return None, None, None
def kill(self, join_threads=True):
"""
Close all sockets for dataplane
@param join_threads If True call join on each thread
"""
for port_number in self.port_list.keys():
self.port_list[port_number].kill()
if join_threads:
self.logger.debug("Joining " + str(port_number))
self.port_list[port_number].join()
self.logger.info("DataPlane shutdown")
def show(self, prefix=''):
print prefix + "Dataplane Controller"
print prefix + "Packets pending" + str(self.packets_pending)
for pnum, port in self.port_list.items():
print prefix + "OpenFlow Port Number " + str(pnum)
port.show(prefix + ' ')
class Obstacle_Avoidance:
"""
A class for performing obstacle avoidance by running a sonar thread and a motor control thread
in parallel.
Attributes:
max_speed (int): The maximum speed value for the motor driver.
decel_rate (float): How quickly (from 0 to 1) the rover should decelerate.
turn_time (float): How long (in seconds) the rover should turn left or right.
reverse_time (float): How long (in seconds) the rover should reverse.
turn_dist (int): How close an obstacle must be (in mm) before the rover turns.
reverse_dist (int): How close an obstacle must be (in mm) before the rover reverses.
offset (float): Since the wheels aren't straight, one wheel will turn slower.
"""
class Direction(Enum):
"""This enumerates the possible directions the rover can move/turn."""
FORWARD = 0
BACKWARD = 1
RIGHT = 2
LEFT = 3
def __init__(self,
threshold=0.1,
buffer_size=5,
max_speed=480,
decel_rate=0.05,
turn_time=1,
reverse_time=1,
turn_dist=1000,
reverse_dist=500,
offset=0.8,
debug=False):
"""
Constructs an Obstacle_Avoidance object using a series of overridable default parameters.
Keyword Arguments:
threshold (float): What threshold value to pass to the Sonar object.
buffer_size (int): What buffer_size value to pass to the Sonar object.
max_speed (int): Refer to the max_speed class attribute.
decel_rate (float): Refer to the decel_rate class attribute.
turn_time (float): Refer to the turn_time class attribute.
reverse_time (float): Refer to the reverse_time class attribute.
turn_dist (int): Refer to the turn_dist class attribute.
reverse_dist (int): Refer to the reverse_dist class attribute.
offset (float): Refer to the offset class attribute.
debug (bool): Flag that controls whether or not to allow remote debugging.
"""
# If debug is enabled, wait for the remote debugger to attach before continuing.
if debug:
ptvsd.enable_attach("rover_senpai")
print("Waiting for debugger to attach...", file=stderr)
ptvsd.wait_for_attach()
print("Debugger attached, continuing...", file=stderr)
dual_mc33926.io_init_motor_drive() # Initializes the motor driver.
self.max_speed = max_speed
self.decel_rate = decel_rate
self.turn_time = turn_time
self.reverse_time = reverse_time
self.turn_dist = turn_dist
self.reverse_dist = reverse_dist
self.offset = offset
self._sonar_data = (
0, b'') # Distance and the sonar indicator (b'L' or b'R').
self._stop_flag = False # Causes the threads to stop when set to true.
self._condition = Condition(
) # Condition object protects _sonar_data and _stop_flag.
self._sonar = Sonar(threshold=threshold, buffer_size=buffer_size)
self._motor = dual_mc33926.MotorDriver()
self._sonar_thread = Thread(target=self._run_sonar, daemon=True)
self._motor_thread = Thread(target=self._run_motor, daemon=True)
def _run_sonar(self):
"""Continously runs the sonars and notifies the motor thread whenever an object is close."""
while True:
# Gets a measurement from the sonars.
dist, dir = self._sonar.measure()
# If the measured sonar distance is less than our turning threshold...
if dist < self.turn_dist:
self._condition.acquire()
# If the stop flag has been set by stop(), release the lock and break out.
if self._stop_flag:
self._condition.release()
break
# Otherwise, publish the distance and which sonar detected it.
self._sonar_data = (dist, dir)
self._condition.notify_all()
self._condition.release()
def _run_motor(self):
"""Continuously runs the motors and turns away from any obstacles deteced by the sonars."""
self._motor.enable()
def move(direction=Direction.FORWARD, duration=None):
"""
Helper method which simply moves/turns the rover in a given direction.
Keyword Arguments:
direction (Direction): The direction the rover should turn/move.
duration (float): How long the rover should move in this direction.
"""
# First, gradually stop whatever the motors were doing before.
for i in range(1 - self.decel_rate, 0, -self.decel_rate):
self._motor.set_speeds(self._spd1 * i, self._dir1,
self._spd2 * i, self._dir2)
sleep(0.05)
# Determine speed and direction of motors based on the desired direction.
if direction == Direction.FORWARD:
self._spd1 = self.max_speed
self._spd2 = self.max_speed * self.offset
self._dir1 = 1
self._dir2 = 1
elif direction == Direction.BACKWARD:
self._spd1 = self.max_speed
self._spd2 = self.max_speed * self.offset
self._dir1 = 0
self._dir2 = 0
elif direction == Direction.RIGHT:
self._spd1 = 0
self._spd2 = self.max_speed * self.offset
self._dir1 = 1
self._dir2 = 1
else:
self._spd1 = self.max_speed
self._spd2 = 0
self._dir1 = 1
self._dir2 = 1
# Start moving.
self._motor.set_speeds(self._spd1, self._dir1, self._spd2,
self._dir2)
# Wait for a duration (if specified) before accepting another move command.
if (type(duration) in [int, float]) and (duration > 0):
sleep(duration)
# The rover continuously moves forward, reversing and turning away from detected obstacles.
while True:
# Move forward until sonar publishes data about a detected obstacle.
move()
self._condition.acquire()
self._condition.wait()
# If the stop flag has been set by stop(), release the lock and break out.
if self._stop_flag:
self._condition.release()
break
# Otherwise, save the distance from the obstacle (in mm) and which sonar detected it.
dist, dir = self._sonar_data.deepcopy()
self._condition.release()
# If the object is very close, first reverse from it.
if dist < self.reverse_dist:
move(direction=Direction.BACKWARD, duration=self.reverse_time)
# Turn right away from objects detected by the left sonar.
if dir == b'R':
move(direction=Direction.LEFT, duration=self.turn_time)
# Turn left away from objects detected by the right sonar.
elif dir == b'L':
move(direction=Direction.RIGHT, duration=self.turn_time)
# Raise an exception if we see an unrecognized sonar label.
else:
raise ValueError("Invalid sonar label")
# Disable the motors once we break out of the loop.
self._motor.disable()
def start(self):
"""This method starts the obstacle avoidance algorithm."""
if not (self._motor_thread.is_alive()
and self._sonar_thread.is_alive()):
self._stop_flag = False
self._sonar_thread.start()
self._motor_thread.start()
print("Obstacle avoidance is now running.", file=stderr)
else:
print("Obstacle avoidance is already running.", file=stderr)
def stop(self):
"""This method stops the obstacle avoidance algorithm."""
if self._motor_thread.is_alive() and self._sonar_thread.is_alive():
self._condition.acquire()
self._stop_flag = True
self._condition.notify_all()
self._condition.release()
self._sonar_thread.join()
self._motor_thread.join()
print("Obstacle avoidance has stopped.", file=stderr)
else:
print("Obstacle avoidance is not currently running.", file=stderr)
class RDMTestThread(Thread):
"""The RDMResponder tests are closely coupled to the Wrapper (yuck!). So we
need to run this all in a separate thread. This is all a bit of a hack and
you'll get into trouble if multiple things are running at once...
"""
RUNNING, COMPLETED, ERROR = range(3)
TESTS, COLLECTOR = range(2)
def __init__(self, pid_store, logs_directory):
super(RDMTestThread, self).__init__()
self._pid_store = pid_store
self._logs_directory = logs_directory
self._terminate = False
self._request = None
# guards _terminate and _request
self._cv = Condition()
self._wrapper = None
self._test_state_lock = Lock() # guards _test_state
self._test_state = {}
def Stop(self):
self._cv.acquire()
self._terminate = True
self._cv.notify()
self._cv.release()
def ScheduleTests(self, universe, uid, test_filter, broadcast_write_delay,
inter_test_delay, dmx_frame_rate, slot_count):
"""Schedule the tests to be run. Callable from any thread. Callbable by any
thread.
Returns:
An error message, or None if the tests were scheduled.
"""
if not self._CheckIfConnected():
return 'Lost connection to OLAD'
self._cv.acquire()
if self._request is not None:
self._cv.release()
return 'Existing request pending'
self._request = lambda: self._RunTests(
universe, uid, test_filter, broadcast_write_delay,
inter_test_delay, dmx_frame_rate, slot_count)
self._cv.notify()
self._cv.release()
return None
def ScheduleCollector(self, universe, skip_queued_messages):
"""Schedule the collector to run on a universe. Callable by any thread.
Returns:
An error message, or None if the collection was scheduled.
"""
if not self._CheckIfConnected():
return 'Lost connection to OLAD'
self._cv.acquire()
if self._request is not None:
self._cv.release()
return 'Existing request pending'
self._request = lambda: self._RunCollector(universe,
skip_queued_messages)
self._cv.notify()
self._cv.release()
return None
def Stat(self):
"""Check the state of the tests. Callable by any thread.
Returns:
The status of the tests.
"""
self._test_state_lock.acquire()
state = dict(self._test_state)
self._test_state_lock.release()
return state
def run(self):
self._wrapper = ClientWrapper()
self._collector = ModelCollector(self._wrapper, self._pid_store)
while True:
self._cv.acquire()
if self._terminate:
logging.info('quitting test thread')
self._cv.release()
return
if self._request is not None:
request = self._request
self._request = None
self._cv.release()
request()
continue
# nothing to do, go into the wait
self._cv.wait()
self._cv.release()
def _UpdateStats(self, tests_completed, total_tests):
self._test_state_lock.acquire()
self._test_state['tests_completed'] = tests_completed
self._test_state['total_tests'] = total_tests
self._test_state_lock.release()
def _RunTests(self, universe, uid, test_filter, broadcast_write_delay,
inter_test_delay, dmx_frame_rate, slot_count):
self._test_state_lock.acquire()
self._test_state = {
'action': self.TESTS,
'tests_completed': 0,
'total_tests': None,
'state': self.RUNNING,
'duration': 0,
}
start_time = datetime.now()
self._test_state_lock.release()
runner = TestRunner.TestRunner(universe, uid, broadcast_write_delay,
inter_test_delay, self._pid_store,
self._wrapper)
for test in TestRunner.GetTestClasses(TestDefinitions):
runner.RegisterTest(test)
dmx_sender = None
if dmx_frame_rate > 0 and slot_count > 0:
logging.info(
'Starting DMXSender with slot count %d and FPS of %d' %
(slot_count, dmx_frame_rate))
dmx_sender = DMXSender(self._wrapper, universe, dmx_frame_rate,
slot_count)
try:
tests, device = runner.RunTests(test_filter, False,
self._UpdateStats)
except Exception as e:
self._test_state_lock.acquire()
self._test_state['state'] = self.ERROR
self._test_state['exception'] = str(e)
self._test_state['traceback'] = traceback.format_exc()
self._test_state_lock.release()
return
finally:
if dmx_sender is not None:
dmx_sender.Stop()
timestamp = int(time())
end_time = datetime.now()
test_parameters = {
'broadcast_write_delay': broadcast_write_delay,
'inter_test_delay': inter_test_delay,
'dmx_frame_rate': dmx_frame_rate,
'dmx_slot_count': slot_count,
}
log_saver = TestLogger.TestLogger(self._logs_directory)
logs_saved = True
try:
log_saver.SaveLog(uid, timestamp, end_time, tests, device,
test_parameters)
except TestLogger.TestLoggerException:
logs_saved = False
self._test_state_lock.acquire()
# we can't use total_seconds() since it requires Python 2.7
time_delta = end_time - start_time
self._test_state['duration'] = (time_delta.seconds +
time_delta.days * 24 * 3600)
self._test_state['state'] = self.COMPLETED
self._test_state['tests'] = tests
self._test_state['logs_saved'] = logs_saved
self._test_state['timestamp'] = timestamp
self._test_state['uid'] = uid
self._test_state_lock.release()
def _RunCollector(self, universe, skip_queued_messages):
"""Run the device model collector for a universe."""
logging.info('Collecting for %d' % universe)
self._test_state_lock.acquire()
self._test_state = {
'action': self.COLLECTOR,
'state': self.RUNNING,
}
self._test_state_lock.release()
try:
output = self._collector.Run(universe, skip_queued_messages)
except Exception as e:
self._test_state_lock.acquire()
self._test_state['state'] = self.ERROR
self._test_state['exception'] = str(e)
self._test_state['traceback'] = traceback.format_exc()
self._test_state_lock.release()
return
self._test_state_lock.acquire()
self._test_state['state'] = self.COMPLETED
self._test_state['output'] = output
self._test_state_lock.release()
def _CheckIfConnected(self):
"""Check if the client is connected to olad.
Returns:
True if connected, False otherwise.
"""
# TODO(simon): add this check, remember it needs locking.
return True
class ZumyROS:
def __init__(self):
self.zumy = Zumy()
rospy.init_node('zumy_ros')
self.cmd = (0, 0)
rospy.Subscriber('cmd_vel', Twist, self.cmd_callback, queue_size=1)
self.lock = Condition()
self.rate = rospy.Rate(30.0)
self.name = socket.gethostname()
self.heartBeat = rospy.Publisher('heartBeat', String, queue_size=1)
self.imu_pub = rospy.Publisher('imu', Imu, queue_size=5)
self.r_enc_pub = rospy.Publisher('r_enc', Int32, queue_size=1)
self.l_enc_pub = rospy.Publisher('l_enc', Int32, queue_size=1)
self.volt_pub = rospy.Publisher('voltage', Float32, queue_size=1)
self.imu_count = 0
def cmd_callback(self, msg):
lv = 0.6
la = 0.4
v = msg.linear.x
a = msg.angular.z
r = lv * v + la * a
l = lv * v - la * a
self.lock.acquire()
self.cmd = (l, r)
self.lock.release()
def run(self):
while not rospy.is_shutdown():
self.lock.acquire()
self.zumy.cmd(*self.cmd)
imu_data = self.zumy.read_imu()
enc_data = self.zumy.read_enc()
volt_data = self.zumy.read_voltage()
self.lock.release()
imu_msg = Imu()
imu_msg.header = Header(self.imu_count, rospy.Time.now(),
self.name)
imu_msg.linear_acceleration.x = 9.81 * imu_data[0]
imu_msg.linear_acceleration.y = 9.81 * imu_data[1]
imu_msg.linear_acceleration.z = 9.81 * imu_data[2]
imu_msg.angular_velocity.x = 3.14 / 180.0 * imu_data[3]
imu_msg.angular_velocity.y = 3.14 / 180.0 * imu_data[4]
imu_msg.angular_velocity.z = 3.14 / 180.0 * imu_data[5]
self.imu_pub.publish(imu_msg)
enc_msg = Int32()
enc_msg.data = enc_data[0]
self.r_enc_pub.publish(enc_msg)
enc_msg = Int32()
enc_msg.data = enc_data[1]
self.l_enc_pub.publish(enc_msg)
volt_msg = Float32()
volt_msg.data = volt_data
self.volt_pub.publish(volt_msg)
#self.heartBeat.publish("I am alive")
self.rate.sleep()
# If shutdown, turn off motors
self.zumy.cmd(0, 0)
class TimedTaskQueue:
__single = None
def __init__(self,
nameprefix='TimedTaskQueue',
isDaemon=True,
debug=False):
self.debug = debug
self.cond = Condition()
self.queue = []
self.count = 0.0
self.thread = Thread(target=self.run)
self.thread.setDaemon(isDaemon)
self.thread.setName(nameprefix + self.thread.getName())
self.thread.start()
if DEBUG_STACK:
self.callstack = {}
def add_task(self, task, t=0, id=None, pos=None):
if task is None:
print_stack()
self.cond.acquire()
when = time() + t
if DEBUG_STACK:
self.callstack[self.count] = format_stack()
if id != None:
self.queue = filter(lambda item: item[3] != id, self.queue)
item = (when, self.count, task, id)
if pos is None:
self.queue.append(item)
else:
self.queue.insert(pos, item)
self.count += 1.0
self.cond.notify()
self.cond.release()
if DEBUG or self.debug:
log('ttqueue:add_task: t', t, 'task', task, 'id', id, 'len(queue)',
len(self.queue))
def run(self):
while True:
task = None
timeout = None
flag = False
self.cond.acquire()
while True:
while len(self.queue) == 0 or flag:
flag = False
if timeout is None:
self.cond.wait()
else:
self.cond.wait(timeout)
self.queue.sort()
when, count, task, id = self.queue[0]
now = time()
if now < when:
timeout = when - now
if DEBUG or self.debug:
log('ttqueue::run: event not due: timeout', timeout,
'task', task)
flag = True
else:
self.queue.pop(0)
if DEBUG or self.debug:
log('ttqueue::run: event due: task', task,
'len(queue)', len(self.queue))
if DEBUG_STACK:
stack = self.callstack.pop(count)
break
self.cond.release()
try:
if task == 'stop':
break
elif task == 'quit':
if len(self.queue) == 0:
break
else:
when, count, task, id = self.queue[-1]
t = when - time() + 0.001
self.add_task('quit', t)
else:
t = time()
task()
if DEBUG or self.debug:
log('ttqueue::run: task finished: time',
time() - t, 'task', task)
except:
log_exc()
if DEBUG_STACK:
print >> sys.stderr, '<<<<<<<<<<<<<<<<'
print >> sys.stderr, 'TASK QUEUED FROM'
print >> sys.stderr, ''.join(stack)
print >> sys.stderr, '>>>>>>>>>>>>>>>>'
if DEBUG:
log('ttqueue::run: exit loop')
class StreamChannel(Channel):
'''
The StreamChannel should provide access methods to remote streams (windows like). Beside pure data transfer (read/write),
the channel needs to be able to communicate control data to the peer (for example: close, flush etc.). There are multiple options
to achieve this:
1) Keep the channel bidirectional and mix pure data with control data, distinguished by some header bytes
+ additional logic is kept in channel
- complex implemetation (priority of control data over already enqueued channel data has to be assured, channel control
data has to have priority over other data from different channels), adjustments to multiple communication layers are needed
anyway
2) Keep the channels standard functionality (read/write) and use the global control channel to communicate data
+ simpler implementation
- the channel itself needs access to the Client object, which again is able to send and receive control messages
- a control message and its remote handler have to be implemented for every possible "channel control communication type"
+ the client is able to create control messages, to push them to the server
3) The channel could hold a second communictaion queue with outbound control messages, the outer client implementation sends
these control requests and receives the answers. Instead of using a own CTRL_MSG for every type of possible control communication,
a single remote method could be used (CTRL_RUN_METHOD). Complexity could be hidden in a "big" remote method with a "big" local
response handler.
+ remote methods are handled by control messages (RUN_METHOD + RUN_METHOD_RESPONSE) and thus are already given priority
(at least as long as the other layers are able to assure that global control channel data is priorized)
+ the handler for the "channel_control_method" call (ipmlemented on client level, as all the other handlers), could
pass in channel control method responses to the correct channel
- this doesn't allow "push notifications" from the other peer, as remote methodes have to be requested from server
to client
Test approach is method 1:
'''
BUFFER_SIZE = 3000
CHANNEL_CONTROL_REQUEST_STATE = 1
CHANNEL_CONTROL_REQUEST_READ = 2
CHANNEL_CONTROL_REQUEST_FLUSH = 3
CHANNEL_CONTROL_REQUEST_CLOSE = 4 # this means to close the stream, not the channel
CHANNEL_CONTROL_REQUEST_POSITION = 5
CHANNEL_CONTROL_REQUEST_LENGTH = 6
CHANNEL_CONTROL_REQUEST_READ_TIMEOUT = 7
CHANNEL_CONTROL_REQUEST_WRITE_TIMEOUT = 8
CHANNEL_CONTROL_REQUEST_SEEK = 9
CHANNEL_CONTROL_REQUEST_WRITE = 10
CHANNEL_CONTROL_INFORM_REMOTEBUFFER_LIMIT = 1001
CHANNEL_CONTROL_INFORM_REMOTEBUFFER_SIZE = 1002
CHANNEL_CONTROL_INFORM_WRITE_SUCCEEDED = 1003
CHANNEL_CONTROL_INFORM_READ_SUCCEEDED = 1004
CHANNEL_CONTROL_INFORM_WRITE_FAILED = 1005
CHANNEL_CONTROL_INFORM_READ_FAILED = 1006
CHANNEL_CONTROL_INFORM_FLUSH_SUCCEEDED = 1007
CHANNEL_CONTROL_INFORM_FLUSH_FAILED = 1008
def __init__(self, channel_id, stream_id, passthrough = True):
# stream attributes
self.__can_read = False
self.__can_seek = False
self.__can_timeout = False
self.__can_write = False
self.__length = 0
self.__position = 0
self.__read_timeout = 0
self.__write_timeout = 0
self.__passthrough = passthrough
if not passthrough:
self.__write_condition = Condition()
self.__read_condition = Condition()
self.__flush_condition = Condition()
self.__write_succeeded = False
self.__read_succeeded = False
self.__flush_succeeded = False
self.__write_size = 0
self.__read_size = 0
self.__read_data = ""
# internal attributes
self.__stream_id = stream_id # id (hasCode) of the stream bound to this channel, if any
self.__control_in_queue = Queue.Queue()
# we start with a id of '-1' to indicate that the channel isn't bound to a remote stream
# th channel is bidirectional, to allow control data to be sent (even it is a read only or write only stream)
# encoding is BYTE (raw)
super(StreamChannel, self).__init__(channel_id, Channel.TYPE_BIDIRECTIONAL, Channel.ENCODING_BYTEARRAY)
@property
def CanRead(self):
return self.__can_read
@property
def CanSeek(self):
return self.__can_seek
@property
def CanTimeout(self):
return self.__can_timeout
@property
def CanWrite(self):
return self.__can_write
@property
def Length(self):
return self.__length
@property
def Position(self):
return self.__position
@Position.setter
def Position(self, value):
self.__position = value
@property
def ReadTimeout():
return self.__read_timeout
@ReadTimeout.setter
def ReadTimeout(self, value):
self.__read_timeout = value
@property
def WriteTimeout():
return self.__write_timeout
@WriteTimeout.setter
def WriteTimeout(self, value):
self.__write_timeout = value
def Close(self):
control_msg = struct.pack("!I", StreamChannel.CHANNEL_CONTROL_REQUEST_CLOSE)
self.__sendControlMessage(control_msg)
def Dispose(self):
pass
def Flush(self):
control_msg = struct.pack("!I", StreamChannel.CHANNEL_CONTROL_REQUEST_FLUSH)
self.__sendControlMessage(control_msg)
if not self.__passthrough:
self.__flush_condition.acquire()
self.__flush_condition.wait(timeout=None)
succeeded = self.__flush_succeeded
self.__flush_condition.release()
return succeeded
def Read(self, count, timeout=0):
# on demand read:
#control_msg = struct.pack("!Iii", StreamChannel.CHANNEL_CONTROL_REQUEST_READ, count, timeout)
#self.__sendControlMessage(control_msg)
if self.__passthrough:
if not self.hasInput():
return ""
return self.readInput()
else:
# if passthrough is disable, data is written as control message and an answer is expected
control_msg = struct.pack("!Iii", StreamChannel.CHANNEL_CONTROL_REQUEST_READ, count, timeout)
self.__sendControlMessage(control_msg)
# we wait till an answer is received (based on a condition)
self.__read_condition.acquire()
self.__read_condition.wait(timeout=None)
#check if write succeeded
succeeded = self.__read_succeeded
read_size = self.__read_size
read_data = self.__read_data
self.__read_condition.release()
if succeeded:
return read_data
else:
raise ChannelException("Error reading from StreamChannel {0}".format(self.id))
def ReadByte(self):
pass
def Seek(self, offset, origin):
control_msg = struct.pack("!Iii", StreamChannel.CHANNEL_CONTROL_REQUEST_SEEK, offset, origin)
self.__sendControlMessage(control_msg)
def Write(self, data):
res = self.__writeData(data)
if not self.__passthrough:
return res
def WriteByte(self, byte):
res = self.__writeData(byte)
if not self.__passthrough:
return res
def __writeData(self, data):
if self.__passthrough:
# in passthrough mode data is written as dedicated data message
header = struct.pack("!B", 0) # header byte 0 indicates that data is written
super(StreamChannel, self).writeOutput(header + data)
else:
# if passthrough is disable, data is written as control message and an answer is expected
control_msg = struct.pack("!Ii", StreamChannel.CHANNEL_CONTROL_REQUEST_WRITE, len(data))
control_msg += data
self.__sendControlMessage(control_msg)
# we wait till an answer is received (based on a condition)
self.__write_condition.acquire()
self.__write_condition.wait(timeout=None)
#check if write succeeded
succeeded = self.__write_succeeded
write_size = self.__write_size
self.__write_condition.release()
if succeeded:
return write_size
else:
return -1 # indicate write error (alternatively a ChannelException could be raised)
def __sendControlMessage(self, control_data):
header = struct.pack("!B", 1) # header byte 1 indicates that control data
# naive, if the queue is full, that message is pending as it hasn't been priorized
super(StreamChannel, self).writeOutput(header + control_data)
def writeOutput(self, data):
res = self.__writeData(data)
if not self.__passthrough:
return res
#def readInput(self):
#print "StreamChannel error: writeOutput shouldn't be called, use Write instead"
def enqueueInput(self, data):
data_type = struct.unpack("!B", data[0])[0]
data = data[1:]
if data_type == 0:
self._Channel__in_queue.put(data) # normal data
else:
self.__dispatchControlMessage(data) # control data
def __dispatchControlMessage(self, control_data):
#print "Channel with id '{0}' received control data: {1}".format(self.id, repr(control_data))
# grab control_msg id
control_msg = struct.unpack("!I", control_data[:4])[0]
data = control_data[4:]
if control_msg == self.CHANNEL_CONTROL_INFORM_WRITE_FAILED:
# set event for write operation response
self.__write_condition.acquire()
self.__write_succeeded = False
self.__write_condition.notifyAll()
self.__write_condition.release()
elif control_msg == self.CHANNEL_CONTROL_INFORM_WRITE_SUCCEEDED:
# read how many bytes have been written
size_written = struct.unpack("!i", data[:4])[0]
# set event for write operation response
self.__write_condition.acquire()
self.__write_size = size_written
self.__write_succeeded = True
self.__write_condition.notifyAll()
self.__write_condition.release()
elif control_msg == self.CHANNEL_CONTROL_INFORM_READ_SUCCEEDED:
# read how many bytes have been read
size_read = struct.unpack("!i", data[:4])[0]
data_read = data[4:]
# set event for read operation response
self.__read_condition.acquire()
self.__read_size = size_read
self.__read_data = data_read
self.__read_succeeded = True
self.__read_condition.notifyAll()
self.__read_condition.release()
elif control_msg == self.CHANNEL_CONTROL_INFORM_READ_FAILED:
# set event for read operation response
self.__read_condition.acquire()
self.__read_succeeded = False
self.__read_condition.notifyAll()
self.__read_condition.release()
elif control_msg == self.CHANNEL_CONTROL_INFORM_FLUSH_SUCCEEDED:
self.__flush_condition.acquire()
self.__flush_succeeded = True
self.__flush_condition.notifyAll()
self.__flush_condition.release()
elif control_msg == self.CHANNEL_CONTROL_INFORM_FLUSH_FAILED:
self.__flush_condition.acquire()
self.__flush_succeeded = False
self.__flush_condition.notifyAll()
self.__flush_condition.release()
def onClose(self):
if not self.__passthrough:
# set error for pending writes
self.__write_condition.acquire()
self.__write_succeeded = False
self.__write_condition.notifyAll()
self.__write_condition.release()
# set error for pending reads
self.__read_condition.acquire()
self.__read_succeeded = False
self.__read_condition.notifyAll()
self.__read_condition.release()
super(StreamChannel, self).onClose()
class UcsCentralEventHandle(object):
"""This class provides api to add and remove event handler."""
def __init__(self, handle):
self.__handle = handle
self.__lock_object = None
self.__wbs = []
self.__wbs_lock = Lock()
self.__enqueue_thread = None
self.__condition = Condition()
self.__event_chan_resp = None
self.__dequeue_thread = None
self.__lowest_timeout = None
self.__wb_to_remove = []
def __get_mo_elem(self, xml_str):
"""
Internal method to extract mo elements from xml string
"""
root = xc.extract_root_elem(xml_str)
mo_elems = []
if root.tag == "methodVessel":
for in_stimuli in root:
for cmce in in_stimuli:
for in_config in cmce:
for mo_elem in in_config:
mo_elems.append(
(mo_elem, cmce.attrib.get('inEid')))
elif root.tag == "configMoChangeEvent":
for in_config in root:
for mo_elem in in_config:
mo_elems.append(mo_elem)
return mo_elems
def __enqueue_function(self):
"""
Internal method used by add_event_handler.
Provides functionality of enqueue/dequeue of the events and
triggering callbacks.
"""
try:
xml_query = '<eventSubscribe cookie="%s"/>' % self.__handle.cookie
self.__event_chan_resp = self.__handle.post_xml(
xml_str=xml_query.encode(), read=False)
except Exception:
raise
try:
while self.__event_chan_resp and len(self.__wbs):
if self.__handle.cookie is None or \
self.__event_chan_resp is None:
break
resp = self.__event_chan_resp.readline()
resp = self.__event_chan_resp.read(int(resp))
for mo_elem in self.__get_mo_elem(resp):
gmo = ucscentralmo.generic_mo_from_xml_elem(mo_elem[0])
mce = MoChangeEvent(event_id=mo_elem[1],
mo=gmo.to_mo(),
change_list=gmo.properties.keys())
for watch_block in self.__wbs:
if watch_block.fmce(mce):
watch_block.enqueue(mce)
with self.__condition:
self.__condition.notify()
if len(self.__wbs) == 0:
self.__condition.acquire()
self.__condition.notify()
self.__condition.release()
except:
raise
def __thread_enqueue_start(self):
"""
Internal method to start the enqueue thread which adds the events in
an internal queue.
"""
self.__enqueue_thread = Thread(name="enqueue_thread",
target=self.__enqueue_function)
self.__enqueue_thread.daemon = True
self.__enqueue_thread.start()
def __time_left(self, watch_block):
timeout_sec = watch_block.params["timeout_sec"]
start_time = watch_block.params["start_time"]
time_diff = datetime.datetime.now() - start_time
if time_diff.seconds < timeout_sec:
return timeout_sec - time_diff.seconds
else:
return 0
# return 2147483647
def __dequeue_mce(self, time_left, watch_block):
if time_left and time_left > 0:
if self.__lowest_timeout is None or \
self.__lowest_timeout > time_left:
self.__lowest_timeout = time_left
mce = watch_block.dequeue(time_left)
else:
mce = watch_block.dequeue(2147483647)
return mce
def __prop_val_exist(self,
mo,
prop,
success_value,
failure_value,
transient_value,
change_list=None):
if isinstance(mo, ucscentralmo.GenericMo):
n_prop = prop
n_prop_val = mo.properties[n_prop]
elif prop not in mo.prop_meta:
n_prop = prop
n_prop_val = getattr(mo, n_prop)
else:
n_prop = mo.prop_meta[prop].xml_attribute
n_prop_val = getattr(mo, n_prop)
if change_list and n_prop not in change_list:
return False
if (len(success_value) > 0 and n_prop_val in success_value) or \
(len(failure_value) > 0 and n_prop_val in failure_value) or \
(len(transient_value) > 0 and n_prop_val in transient_value):
return True
return False
def __dequeue_mo_prop_poll(self,
mo,
prop,
poll_sec,
watch_block,
timeout_sec=None,
time_left=None):
success_value = watch_block.params["success_value"]
failure_value = watch_block.params["failure_value"]
transient_value = watch_block.params["transient_value"]
if not success_value or len(success_value) < 1:
raise ValueError("success_value is missing.")
pmo = self.__handle.query_dn(mo.dn)
if pmo is None:
UcsCentralWarning('Mo ' + pmo.dn + ' not found.')
return
if timeout_sec is not None and time_left is not None and time_left > 0:
if time_left < poll_sec:
poll_sec = timeout_sec - time_left
if self.__lowest_timeout is None or self.__lowest_timeout > poll_sec:
self.__lowest_timeout = poll_sec
if self.__prop_val_exist(pmo, prop, success_value, failure_value,
transient_value):
log.info("Successful")
self.__wb_to_remove.append(watch_block)
def __dequeue_mo_prop_event(self, prop, watch_block, time_left=None):
success_value = watch_block.params["success_value"]
failure_value = watch_block.params["failure_value"]
transient_value = watch_block.params["transient_value"]
if not success_value or len(success_value) < 1:
raise ValueError("success_value is missing.")
# dequeue mce
mce = self.__dequeue_mce(time_left, watch_block)
if mce is None:
return
# checks if prop value exist in success or failure or transient values
attributes = mce.change_list
if self.__prop_val_exist(mce.mo, prop, success_value, failure_value,
transient_value, attributes):
if watch_block.callback:
ctxt = watch_block.params['context']
ctxt["done"] = True
watch_block.callback(mce)
self.__wb_to_remove.append(watch_block)
def __dequeue_mo_until_removed(self, watch_block, time_left=None):
# dequeue mce
mce = self.__dequeue_mce(time_left, watch_block)
if mce is None:
return
if watch_block.callback is not None:
watch_block.callback(mce)
# watch mo until gets deleted
if mce.mo.status == "deleted":
self.__wb_to_remove.append(watch_block)
def __dequeue_all_class_id(self, watch_block, time_left=None):
# dequeue mce
mce = self.__dequeue_mce(time_left, watch_block)
if mce is not None and watch_block.callback is not None:
watch_block.callback(mce)
def __dequeue_function(self):
"""
Internal method to dequeue to events.
"""
while len(self.__wbs):
self.__lowest_timeout = None
self.__wb_to_remove = []
try:
for watch_block in self.__wbs:
mo = watch_block.params["managed_object"]
prop = watch_block.params["prop"]
poll_sec = watch_block.params["poll_sec"]
timeout_sec = watch_block.params["timeout_sec"]
# checks if watch_block is not timed out, else remove
time_left = None
if timeout_sec is not None:
time_left = self.__time_left(watch_block)
if time_left <= 0:
self.__wb_to_remove.append(watch_block)
continue
# poll for mo. Not to monitor event.
if poll_sec is not None and mo is not None:
self.__dequeue_mo_prop_poll(mo, prop, poll_sec,
watch_block, timeout_sec,
time_left)
elif mo is not None:
# watch mo until prop_val changed to desired value
if prop is not None:
self.__dequeue_mo_prop_event(
prop, watch_block, time_left)
# watch mo until it is removed
else:
self.__dequeue_mo_until_removed(
watch_block, time_left)
elif mo is None:
# watch all event or specific to class_id
self.__dequeue_all_class_id(watch_block, time_left)
except Exception as e:
log.info(str(e))
self.__wb_to_remove.append(watch_block)
# removing watch_block
if len(self.__wb_to_remove):
self.__wbs_lock.acquire()
for wb in self.__wb_to_remove:
if "context" in wb.params:
ctxt = wb.params['context']
ctxt["done"] = True
self.watch_block_remove(wb)
self.__wb_to_remove = []
self.__wbs_lock.release()
# wait for more events only if watch_block exists
if len(self.__wbs):
with self.__condition:
self.__condition.wait(self.__lowest_timeout)
return
def __thread_dequeue_start(self):
"""
Internal method to start dequeue thread.
"""
self.__dequeue_thread = Thread(name="dequeue_thread",
target=self.__dequeue_function)
self.__dequeue_thread.daemon = True
self.__dequeue_thread.start()
def watch_block_add(self,
params,
filter_callback,
capacity=500,
callback=None):
"""
Internal method to add a watch block for starting event monitoring.
"""
if self.__handle.cookie is None:
return None
self.__wbs_lock.acquire()
watch_block = WatchBlock(params, filter_callback, capacity,
callback) # Add a List of Watchers
if watch_block is not None and watch_block.callback is None:
watch_block.callback = watch_block.dequeue_default_callback
self.__wbs.append(watch_block)
self.__wbs_lock.release()
return watch_block
def watch_block_remove(self, watch_block):
"""
Internal method to remove a watch block for
stopping event monitoring.
"""
if watch_block in self.__wbs:
self.__wbs.remove(watch_block)
def _add_class_id_watch(self, class_id):
if ucscentralcoreutils.find_class_id_in_mo_meta_ignore_case(
class_id) is None:
raise UcsCentralValidationException(
"Invalid ClassId %s specified." % class_id)
def watch__type_filter(mce):
"""
Callback method to work on events with a specific class_id.
"""
if mce.mo.get_class_id().lower() == class_id.lower():
return True
return False
return watch__type_filter
def _add_mo_watch(self,
managed_object,
prop=None,
success_value=[],
poll_sec=None):
if ucscentralcoreutils.find_class_id_in_mo_meta_ignore_case(
managed_object.get_class_id()) is None:
raise UcsCentralValidationException(
"Unknown ClassId %s provided." % managed_object.get_class_id())
if prop is not None:
mo_property_meta = ucscentralcoreutils.get_mo_property_meta(
managed_object.get_class_id(), prop)
if mo_property_meta is None:
raise UcsCentralValidationException(
"Unknown Property %s provided." % prop)
if not success_value:
raise UcsCentralValidationException(
"success_value parameter is not provided.")
if poll_sec is None:
def watch_mo_filter(mce):
"""
Callback method to work on events specific to respective
managed object.
"""
if mce.mo.dn == managed_object.dn:
return True
return False
return watch_mo_filter
else:
def watch_none_filter(mce):
"""
Callback method to ignore all events.
"""
return False
return watch_none_filter
def add(self,
class_id=None,
managed_object=None,
prop=None,
success_value=[],
failure_value=[],
transient_value=[],
poll_sec=None,
timeout_sec=None,
call_back=None,
context=None):
"""
Adds an event handler.
An event handler can be added using this method where an user can
subscribe for the event channel from UCSCENTRAL and can monitor those events
for any specific success value or failure value for a managed object.
Args:
class_id (str): managed object class id
managed_object (ManagedObject)
prop (str) - property of the managed object to monitor
success_value (list) - success values of a prop
failure_value (list) - failure values of a prop
transient_value (list) - transient values of a prop
poll_sec - specifies the time in seconds for polling event.
timeout_sec - time after which method should stop monitoring.
call_back - call back method
"""
if class_id is not None and managed_object is not None:
raise UcsCentralValidationException(
"Specify either class_id or managedObject, not both")
if class_id is not None:
filter_callback = self._add_class_id_watch(class_id)
elif managed_object is not None:
filter_callback = self._add_mo_watch(managed_object, prop,
success_value, poll_sec)
else:
def watch_all_filter(mce):
"""
Callback method to work on all events.
"""
return True
filter_callback = watch_all_filter
param_dict = {
'class_id': class_id,
'managed_object': managed_object,
'prop': prop,
'success_value': success_value,
'failure_value': failure_value,
'transient_value': transient_value,
'poll_sec': poll_sec,
'timeout_sec': timeout_sec,
'call_back': call_back,
'start_time': datetime.datetime.now(),
'context': context
}
if filter_callback is None:
raise UcsCentralValidationException("Error adding WatchBlock...")
watch_block = self.watch_block_add(params=param_dict,
filter_callback=filter_callback,
callback=call_back)
if watch_block is not None and len(self.__wbs) == 1:
if poll_sec is None:
self.__thread_enqueue_start()
self.__thread_dequeue_start()
return watch_block
def remove(self, watch_block):
"""
Removes an event handler.
"""
self.__wbs_lock.acquire()
if watch_block in self.__wbs:
self.watch_block_remove(watch_block)
else:
UcsCentralWarning("Event handler not found")
self.__wbs_lock.release()
def clean(self):
"""
Removes all the watch blocks from the event handler
"""
self.__wbs_lock.acquire()
for each in self.__wbs:
self.watch_block_remove(each)
self.__wbs_lock.release()
def get(self):
"""
Returns the list of event handlers.
"""
return self.__wbs
class Perception(RobotPart):
def __init__(self, robot_name, tf_listener, image_topic=None, projection_srv=None, camera_base_ns=''):
super(Perception, self).__init__(robot_name=robot_name, tf_listener=tf_listener)
if image_topic is None:
self.image_topic = "/" + self.robot_name + "/top_kinect/rgb/image"
else:
self.image_topic = image_topic
if projection_srv is None:
projection_srv_name = '/' + robot_name + '/top_kinect/project_2d_to_3d'
else:
projection_srv_name = projection_srv
self._camera_base_ns = camera_base_ns
self._camera_lazy_sub = None
self._camera_cv = Condition()
self._camera_last_image = None
self._annotate_srv = self.create_service_client(
'/' + robot_name + '/people_recognition/face_recognition/annotate', Annotate)
self._recognize_srv = self.create_service_client(
'/' + robot_name + '/people_recognition/face_recognition/recognize', Recognize)
self._clear_srv = self.create_service_client(
'/' + robot_name + '/people_recognition/face_recognition/clear', Empty)
self._image_data = (None, None, None)
self._face_properties_srv = self.create_service_client(
'/' + robot_name + '/people_recognition/face_recognition/get_face_properties', GetFaceProperties)
self._projection_srv = self.create_service_client(projection_srv_name, Project2DTo3D)
self._person_recognition_3d_srv = \
self.create_service_client('/' + robot_name + '/people_recognition/detect_people_3d', RecognizePeople3D)
def close(self):
pass
def _image_cb(self, image):
self._camera_cv.acquire()
self._camera_last_image = image
self._camera_cv.notify()
self._camera_cv.release()
def get_image(self, timeout=5):
# lazy subscribe to the rgb(d) camera
if not self._camera_lazy_sub:
rospy.loginfo("Creating subscriber")
self._camera_lazy_sub = rospy.Subscriber(self.image_topic, Image, self._image_cb)
rospy.loginfo('lazy subscribe to %s', self._camera_lazy_sub.name)
rospy.loginfo("getting one image...")
self._camera_cv.acquire()
self._camera_last_image = None
for i in range(timeout):
if self._camera_last_image:
rospy.loginfo("len(self._camera_last_image): {}".format(len(self._camera_last_image.data)))
break
else:
rospy.loginfo("self._camera_last_image: {}".format(self._camera_last_image))
if rospy.is_shutdown():
return
self._camera_cv.wait(timeout=1)
else:
raise Exception('no image received from %s' % self._camera_lazy_sub.name)
image = self._camera_last_image
self._camera_cv.release()
rospy.loginfo("got %d bytes of image data", len(image.data))
return image
def project_roi(self, roi, frame_id=None):
""" Projects a region of interest of a depth image to a 3D Point. Hereto, a service is used
:param roi: sensor_msgs/RegionOfInterest
:param frame_id: if specified, the result is transformed into this frame id
:return: VectorStamped object
"""
response = self.project_rois(rois=[roi]).points[0]
# Convert to VectorStamped
result = VectorStamped(x=response.point.x, y=response.point.y, z=response.point.z,
frame_id=response.header.frame_id)
# If necessary, transform the point
if frame_id is not None:
rospy.loginfo("Transforming roi to {}".format(frame_id))
result = result.projectToFrame(frame_id=frame_id, tf_listener=self.tf_listener)
# Return the result
return result
def project_rois(self, rois):
# Call the service with the provided Region of Interest
try:
points = self._projection_srv(rois=rois)
except rospy.ServiceException as e:
raise ValueError('project_roi failed', e)
else:
rospy.loginfo('project_rois response: %s', points)
return points
# OpenFace
def _get_faces(self, image=None):
if not image:
image = self.get_image()
try:
r = self._recognize_srv(image=image)
rospy.loginfo('Found %d face(s) in the image', len(r.recognitions))
except rospy.ServiceException as e:
rospy.logerr("Can't connect to face recognition service: {}".format(e))
r = RecognizeResponse()
except Exception as e:
rospy.logerr("Can't detect faces: {}".format(e))
return r
def learn_person(self, name='operator'):
HEIGHT_TRESHOLD = 88
WIDTH_TRESHOLD = 88
try:
image = self.get_image()
except Exception as e:
rospy.logerr("Can't get image: {}".format(e))
return False
raw_recognitions = self._get_faces(image).recognitions
recognitions = [r for r in raw_recognitions if r.roi.height > HEIGHT_TRESHOLD and r.roi.width > WIDTH_TRESHOLD]
rospy.loginfo('Found %d valid face(s)', len(recognitions))
if len(recognitions) != 1:
rospy.loginfo("Too many faces: {}".format(len(recognitions)))
return False
recognition = recognitions[0]
rospy.loginfo('annotating that face as %s', name)
try:
self._annotate_srv(image=image, annotations=[Annotation(label=name, roi=recognition.roi)])
except rospy.ServiceException as e:
rospy.logerr("Can't connect to person learning service: {}".format(e))
return False
except Exception as e:
rospy.logerr("Can't learn a person: {}".format(e))
return False
return True
def detect_faces(self, image=None, stamp=False):
"""
Snap an image with the camera and return the recognized faces.
:param image: image to use for recognition
:type image: sensor_msgs/Image
:param stamp: Return recognitions and stamp
:type stamp: bool
:return: recognitions of the faces
:rtype: list[image_recognition_msgs/Recognition]
"""
if not image:
image = self.get_image()
if stamp:
return self._get_faces(image).recognitions, image.header.stamp
else:
return self._get_faces(image).recognitions
@staticmethod
def get_best_face_recognition(recognitions, desired_label, probability_threshold=4.0):
"""
Returns the Recognition with the highest probability of having the desired_label.
Assumes that the probability distributions in Recognition are already sorted by probability (descending, highest first)
:param recognitions: The recognitions to select the best one with desired_label from
:type recognitions: list[image_recognition_msgs/Recognition]
:param desired_label: what label to look for in the recognitions
:type desired_label: str
:param probability_threshold: only accept recognitions with probability higher than threshold
:type probability_threshold: double
:return: the best recognition matching the given desired_label
:rtype image_recognition_msgs/Recognition
"""
rospy.logdebug("get_best_face_recognition: recognitions = {}".format(recognitions))
# Only take detections with operator
# detections = []
# The old implementation took, for each recognition, the (label, prob) pairs where label==desired_label.
# Other pairs in the same distribution may have higher probability.
# When the best_recognition is picked, it picked the recognition where the probability for the desired_label is hhighest comapared to other recognitions. BUT: a recognitions highest probability may be for a different label
# because the selection only compares matching labels, not looking at the probability of non-matching pairs.
# For example: we have 2 recognitions.
# in recognition 1, A has 50%, desired_label has 30%, B has 20%.
# in recognition 2, B has 60%, desired_label has 35%, A has 5%.
# Then, recognition 2 has the highest probability for the desired_label and is thus picked.
# Because we take the [0]'th index of the distribution, that name is B
#
# Solution: because the probability distributions are sorted, just take the probability distribution where the desired label has the highest probability.
#for recog in recognitions:
# for cp in recog.categorical_distribution.probabilities:
# if cp.label == desired_label:
# detections.append((recog, cp.probability))
# Sort based on probability
#if detections:
# sorted_detections = sorted(detections, key=lambda det: det[1])
# best_detection = sorted_detections[0][0] # A CategoricalDistribution in a Recognition is already ordered, max prob is at [0]
#else:
# best_detection = None
rospy.loginfo("Probability threshold %.2f", probability_threshold)
for index, recog in enumerate(recognitions):
rospy.loginfo("{index}: {dist}".format(index=index,
dist=[(cp.label, "{:.2f}".format(cp.probability)) for cp in recog.categorical_distribution.probabilities]))
matching_recognitions = [recog for recog in recognitions if \
recog.categorical_distribution.probabilities and \
recog.categorical_distribution.probabilities[0].label == desired_label]
if matching_recognitions:
best_recognition = max(matching_recognitions, key=lambda recog: recog.categorical_distribution.probabilities[0].probability)
return best_recognition if best_recognition.categorical_distribution.probabilities[0].probability > probability_threshold else None
else:
return None # TODO: Maybe so something smart with selecting a recognition where the desired_label is not the most probable for a recognition?
def clear_face(self):
"""
clearing all faces from the OpenFace node.
:return: no return
"""
rospy.loginfo('clearing all learned faces')
self._clear_srv()
# Skybiometry
def get_face_properties(self, faces=None, image=None):
"""
Get the face properties of all faces or in an image. If faces is provided, image is ignored. If both aren't
provided, an image is collected.
:param faces: images of all faces
:type faces: list[sensor_msgs/Image]
:param image: image containing the faces
:type image: sensor_msgs/Image
:return: list of face properties
:rtype: list[image_recognition_msgs/FaceProperties]
"""
if not faces:
if not image:
image = self.get_image()
face_recognitions = self.detect_faces(image=image)
rois = img_recognitions_to_rois(face_recognitions)
faces = img_cutout(image, rois)
face_properties = []
try:
face_properties_response = self._face_properties_srv(faces)
face_properties = face_properties_response.properties_array
except Exception as e:
rospy.logerr(e)
return [None] * len(faces)
face_log = '\n - '.join([''] + [repr(s) for s in face_properties])
rospy.loginfo('face_properties:%s', face_log)
return face_properties
def get_rgb_depth_caminfo(self, timeout=5):
"""
Get an rgb image and and depth image, along with camera info for the depth camera.
The returned tuple can serve as input for world_model_ed.ED.detect_people.
:param timeout: How long to wait until the images are all collected.
:return: tuple(rgb, depth, depth_info) or a None if no images could be gathered.
"""
event = Event()
def callback(rgb, depth, depth_info):
rospy.loginfo('Received rgb, depth, cam_info')
self._image_data = (rgb, depth, depth_info)
event.set()
# camera topics
depth_info_sub = message_filters.Subscriber('{}/depth_registered/camera_info'.format(self._camera_base_ns),
CameraInfo)
depth_sub = message_filters.Subscriber('{}/depth_registered/image'.format(self._camera_base_ns), Image)
rgb_sub = message_filters.Subscriber('{}/rgb/image_raw'.format(self._camera_base_ns), Image)
ts = message_filters.ApproximateTimeSynchronizer([rgb_sub, depth_sub, depth_info_sub],
queue_size=1,
slop=10)
ts.registerCallback(callback)
event.wait(timeout)
ts.callbacks.clear()
del ts, depth_info_sub, depth_sub, rgb_sub, callback
if any(self._image_data):
return self._image_data
else:
return None
def detect_person_3d(self, rgb, depth, depth_info):
return self._person_recognition_3d_srv(image_rgb=rgb, image_depth=depth, camera_info_depth=depth_info).people
def recv_ack(s, host, port, window, lock, lasted_ack, packages_time):
# Variaveis compartilhadas
global max_packages
global aux_time
global aux_lock
aux_lock = Condition()
global print_lock
print_lock = Condition()
rtt = 2.0
while True:
try:
# Tempo que o cliente espera pela confirmação do ack
s.settimeout(rtt)
# Recebe o pacote e o endereço do servidor
data, add = s.recvfrom(4096)
# Adiciona o tempo de envio dos pacotes
aux_lock.acquire()
packages_time.append(time() - aux_time)
aux_time = time()
aux_lock.release()
# Transforma o objeto de bytes no pacotes original
packet = pickle.loads(data)
print_lock.acquire()
print("Ack com número de sequência {} recebido".format(packet.ack))
print_lock.release()
# Flag para ver se o ack recebido é o esperado na janela
ack_awaited = False
# Percorre a janela de transmissão
for i in range(lasted_ack, max_packages + 1):
# Armazena o elemento (pacote, seqnumber) na variavel
lock.acquire()
index = window[i % 7]
lock.release()
# Verifica se o ack recebido é igual ao numero de sequencia do pacote armazenado na janela
if (packet.ack == index[1]):
# Ack recebido é o esperado
ack_awaited = True
lock.acquire()
# Percorre a janela transformando em None os pacotes confirmados
for j in range(lasted_ack, i + 1):
window[j % 7] = (None, None)
# Atualiza o ultimo ack recebido
lasted_ack += 1
max_packages += 1
lock.notify()
lock.release()
# Ack não é o esperado
if (ack_awaited == False):
print_lock.acquire()
print("Ack duplicado")
print_lock.release()
# Como foram recebidos ack duplicados a janela é reenviada
resend_window(s, host, port, window, lock, lasted_ack)
except socket.error as error:
print_lock.acquire()
print("Timeout")
print_lock.release()
# Se o houver um timeout a janela será reenviada
resend_window(s, host, port, window, lock, lasted_ack)
# Variavel para verificar se todos os pacotes da janela foram confirmados
empty_window = True
# Percorre a janela verificando se existi pacotes não confirmados
for i in range(lasted_ack, max_packages + 1):
if window[i % 7] != (None, None):
# Se houver atualiza a variavel para falso
empty_window = False
# Verifica se a janela está vazia e se o servidor parou de enviar pacotes
if end == True and empty_window == True:
break
class TimerClock(Clock):
"""
:py:class:`pyctrl.block.clock.TimerClock` provides a clock that
reads the current time periodically.
:param float period: period in seconds
"""
def __init__(self, **kwargs):
self.period = kwargs.pop('period', 0.01)
super().__init__(**kwargs)
self.condition = Condition()
self.timer = None
self.running = False
if self.enabled:
self.enabled = False
self.set_enabled(True)
def set(self, exclude=(), **kwargs):
"""
Set properties of :py:class:`pyctrl.block.clock.TimerClock`.
:param tuple exclude: attributes to exclude
:param float period: clock period
:param kwargs kwargs: other keyword arguments
:raise: :py:class:`pyctrl.block.BlockException` if any of the :py:attr:`kwargs` is left unprocessed
"""
if 'period' in kwargs:
self.period = kwargs.pop('period')
# call super
return super().set(exclude, **kwargs)
def get(self, *keys, exclude=()):
"""
Get properties of :py:class:`pyctrl.block.clock.TimerClock`.
Available attributes are those from :py:meth:`pyctrl.block.clock.Clock.get` and:
1. :py:attr:`period`
The elapsed time since initialization or last reset can be
obtained using the method :py:meth:`pyctrl.block.clock.TimerClock.read`.
:param keys: string or tuple of strings with property names
:param tuple exclude: keys never to be returned (Default ())
"""
# call super excluding time and last
return super().get(*keys,
exclude=exclude +
('condition', 'timer', 'running', 'thread'))
def tick(self):
# Acquire lock
self.condition.acquire()
# Got a tick
time = perf_counter()
#print('> TICK: {}, {}'.format(time, self))
dwell = time - self.time
if (dwell < 0.9 * self.period):
# need more time
#print('> MORE TIME')
# Setup new timer
self.timer = Timer(self.period - dwell, self.tick)
self.timer.start()
# Release lock
self.condition.release()
# and return
return
# Close to period
self.time = time
# Add to count
self.count += 1
# Notify lock
self.condition.notify_all()
# Release lock
self.condition.release()
def run(self):
#print('> run')
self.running = True
while self.enabled and self.running:
# Acquire condition
self.condition.acquire()
#print('> WILL TICK')
# Setup timer
self.timer = Timer(self.period, self.tick)
self.timer.start()
#print('> WAITING')
# Wait
self.condition.wait()
# and release
self.condition.release()
self.running = False
# print('> END OF RUN!')
def set_enabled(self, enabled=True):
"""
Set :py:class:`pyctrl.block.clock.TimerClock` :py:attr:`enabled` state.
:param bool enabled: True or False (default True)
"""
# quick return
if enabled == self.enabled:
return
# enable
if enabled:
# print('> Enabling TimerClock')
# set enabled
super().set_enabled(enabled)
# Start thread
self.thread = Thread(target=self.run)
self.thread.start()
# disable
else:
# Acquire condition
self.condition.acquire()
# print('> Disabling TimerClock')
# Prepare to stop
self.running = False
# Notify lock
self.condition.notify_all()
# set enabled
super().set_enabled(enabled)
# and release
self.condition.release()
def read(self):
"""
Read from :py:class:`pyctrl.block.clock.TimerClock`.
:return: tuple with elapsed time since initialization or last reset
"""
#print('> read')
if self.enabled:
# Acquire condition
self.condition.acquire()
# wait
self.condition.wait()
# and release
self.condition.release()
return (self.time - self.time_origin, )
class CommunicationTokenCredential(object):
"""Credential type used for authenticating to an Azure Communication service.
:param str token: The token used to authenticate to an Azure Communication service
:keyword token_refresher: The token refresher to provide capacity to fetch fresh token
:raises: TypeError
"""
_ON_DEMAND_REFRESHING_INTERVAL_MINUTES = 2
def __init__(self,
token, # type: str
**kwargs
):
token_refresher = kwargs.pop('token_refresher', None)
communication_token_refresh_options = CommunicationTokenRefreshOptions(token=token,
token_refresher=token_refresher)
self._token = communication_token_refresh_options.get_token()
self._token_refresher = communication_token_refresh_options.get_token_refresher()
self._lock = Condition(Lock())
self._some_thread_refreshing = False
def get_token(self):
# type () -> ~azure.core.credentials.AccessToken
"""The value of the configured token.
:rtype: ~azure.core.credentials.AccessToken
"""
if not self._token_refresher or not self._token_expiring():
return self._token
should_this_thread_refresh = False
with self._lock:
while self._token_expiring():
if self._some_thread_refreshing:
if self._is_currenttoken_valid():
return self._token
self._wait_till_inprogress_thread_finish_refreshing()
else:
should_this_thread_refresh = True
self._some_thread_refreshing = True
break
if should_this_thread_refresh:
try:
newtoken = self._token_refresher() # pylint:disable=not-callable
with self._lock:
self._token = newtoken
self._some_thread_refreshing = False
self._lock.notify_all()
except:
with self._lock:
self._some_thread_refreshing = False
self._lock.notify_all()
raise
return self._token
def _wait_till_inprogress_thread_finish_refreshing(self):
self._lock.release()
self._lock.acquire()
def _token_expiring(self):
return self._token.expires_on - self._get_utc_now() <\
timedelta(minutes=self._ON_DEMAND_REFRESHING_INTERVAL_MINUTES)
def _is_currenttoken_valid(self):
return self._get_utc_now() < self._token.expires_on
@classmethod
def _get_utc_now(cls):
return datetime.now().replace(tzinfo=TZ_UTC)
class Serial:
"""
A Serial object offers a way to send and data using a HDLC-like
formating.
"""
HDLC_FLAG_BYTE = 0x7e
HDLC_CTLESC_BYTE = 0x7d
TOS_SERIAL_ACTIVE_MESSAGE_ID = 0
TOS_SERIAL_CC1000_ID = 1
TOS_SERIAL_802_15_4_ID = 2
TOS_SERIAL_UNKNOWN_ID = 255
SERIAL_PROTO_ACK = 67
SERIAL_PROTO_PACKET_ACK = 68
SERIAL_PROTO_PACKET_NOACK = 69
SERIAL_PROTO_PACKET_UNKNOWN = 255
def __init__(self, port, baudrate, flush=False, debug=False, qsize=10):
self._debug = debug
self._in_queue = Queue.Queue(qsize)
self._out_lock = Lock()
self._out_ack = Condition()
self._seqno = 0
self._ack = None
self._write_counter = 0
self._write_counter_failures = 0
self._read_counter = 0
self._ts = None
self._s = serial.Serial(port, baudrate, rtscts=0, timeout=0.5)
self._s.flushInput()
start = time.time()
if flush:
print >> sys.stdout, "Flushing the serial port",
while time.time() - start < 1:
p = self._read()
sys.stdout.write(".")
if not self._debug:
sys.stdout.write("\n")
self._s.close()
self._s = serial.Serial(port, baudrate, rtscts=0, timeout=None)
thread.start_new_thread(self.run, ())
def run(self):
while True:
p = self._read()
self._read_counter += 1
if self._debug:
print "Serial:run: got a packet(%d): %s" % (self._read_counter,
p)
ack = AckFrame(p.data)
if ack.protocol == self.SERIAL_PROTO_ACK:
if not self._ack:
self._ack = ack
if self._debug:
print "Serial:run: got an ack:", ack
self._ack = ack
# Wake up the writer
self._out_ack.acquire()
self._out_ack.notify()
self._out_ack.release()
else:
ampkt = ActiveMessage(NoAckDataFrame(p.data).data)
if ampkt.type == 100:
for t in "".join([chr(i) for i in ampkt.data
]).strip('\n\0').split('\n'):
print "PRINTF:", t.strip('\n')
else:
if self._in_queue.full():
print "Warning: Buffer overflow"
self._in_queue.get()
self._in_queue.put(p, block=False)
# Returns the next incoming serial packet
def _read(self):
"""Wait for a packet and return it as a RawPacket."""
try:
d = self._get_byte()
ts = time.time()
while d != self.HDLC_FLAG_BYTE:
d = self._get_byte()
ts = time.time()
packet = [d]
d = self._get_byte()
if d == self.HDLC_FLAG_BYTE:
d = self._get_byte()
ts = time.time()
else:
packet.append(d)
while d != self.HDLC_FLAG_BYTE:
d = self._get_byte()
packet.append(d)
if self._debug == True:
print "Serial:_read: unescaped", packet
packet = self._unescape(packet)
crc = self._crc16(0, packet[1:-3])
packet_crc = self._decode(packet[-3:-1])
if crc != packet_crc:
print "Warning: wrong CRC! %x != %x %s" % (
crc, packet_crc, ["%2x" % i for i in packet])
if self._debug:
if self._ts == None:
self._ts = ts
else:
print "Serial:_read: %.4f (%.4f) Recv:" % (
ts, ts - self._ts), self._format_packet(packet[1:-3])
self._ts = ts
return RawPacket(ts, packet[1:-3], crc == packet_crc)
except socket.timeout:
return None
def read(self, timeout=None):
start = time.time()
done = False
while not done:
p = None
while p == None:
if timeout == 0 or time.time() - start < timeout:
try:
p = self._in_queue.get(True, timeout)
except Queue.Empty:
return None
else:
return None
if p.crc:
done = True
else:
p = None
# In the current TinyOS the packets from the mote are always NoAckDataFrame
return NoAckDataFrame(p.data)
def write(self, payload):
"""
Write a packet. If the payload argument is a list, it is
assumed to be exactly the payload. Otherwise the payload is
assume to be a Packet and the real payload is obtain by
calling the .payload().
"""
if type(payload) != type([]):
# Assume this will be derived from Packet
payload = payload.payload()
self._out_lock.acquire()
self._seqno = (self._seqno + 1) % 100
packet = DataFrame()
packet.protocol = self.SERIAL_PROTO_PACKET_ACK
packet.seqno = self._seqno
packet.dispatch = 0
packet.data = payload
packet = packet.payload()
crc = self._crc16(0, packet)
packet.append(crc & 0xff)
packet.append((crc >> 8) & 0xff)
packet = [self.HDLC_FLAG_BYTE
] + self._escape(packet) + [self.HDLC_FLAG_BYTE]
while True:
self._put_bytes(packet)
self._write_counter += 1
if self._debug == True:
print "Send(%d/%d): %s" % (
self._write_counter, self._write_counter_failures, packet)
print "Wait for ack %d ..." % (self._seqno)
self._out_ack.acquire()
self._out_ack.wait(0.2)
if self._debug:
print "Wait for ack %d done. Latest ack:" % (
self._seqno), self._ack
self._out_ack.release()
if self._ack and self._ack.seqno == self._seqno:
if self._debug:
print "The packet was acked."
self._out_lock.release()
if self._debug:
print "Returning from Serial.write..."
return True
else:
self._write_counter_failures += 1
if self._debug:
print "The packet was not acked. Try again."
# break # make only one sending attempt
self._out_lock.release()
return False
def _format_packet(self, payload):
f = NoAckDataFrame(payload)
if f.protocol == self.SERIAL_PROTO_ACK:
rpacket = AckFrame(payload)
return "Ack seqno: %d" % (rpacket.seqno)
else:
rpacket = ActiveMessage(f.data)
return "D: %04x S: %04x L: %02x G: %02x T: %02x | %s" % \
(rpacket.destination, rpacket.source,
rpacket.length, rpacket.group, rpacket.type,
list2hex(rpacket.data))
def _crc16(self, base_crc, frame_data):
crc = base_crc
for b in frame_data:
crc = crc ^ (b << 8)
for i in range(0, 8):
if crc & 0x8000 == 0x8000:
crc = (crc << 1) ^ 0x1021
else:
crc = crc << 1
crc = crc & 0xffff
return crc
def _encode(self, val, dim):
output = []
for i in range(dim):
output.append(val & 0xFF)
val = val >> 8
return output
def _decode(self, v):
r = long(0)
for i in v[::-1]:
r = (r << 8) + i
return r
def _get_byte(self):
try:
r = struct.unpack("B", self._s.read())[0]
return r
except struct.error:
# Serial port read timeout
raise socket.timeout
def _put_bytes(self, data):
#print "DEBUG: _put_bytes:", data
for b in data:
self._s.write(struct.pack('B', b))
def _unescape(self, packet):
r = []
esc = False
for b in packet:
if esc:
r.append(b ^ 0x20)
esc = False
elif b == self.HDLC_CTLESC_BYTE:
esc = True
else:
r.append(b)
return r
def _escape(self, packet):
r = []
for b in packet:
if b == self.HDLC_FLAG_BYTE or b == self.HDLC_CTLESC_BYTE:
r.append(self.HDLC_CTLESC_BYTE)
r.append(b ^ 0x20)
else:
r.append(b)
return r
def debug(self, debug):
self._debug = debug
class ReadWriteLock(object):
"""Read-Write lock class. A read-write lock differs from a standard
threading.RLock() by allowing multiple threads to simultaneously hold a
read lock, while allowing only a single thread to hold a write lock at the
same point of time.
When a read lock is requested while a write lock is held, the reader
is blocked; when a write lock is requested while another write lock is
held or there are read locks, the writer is blocked.
Writers are always preferred by this implementation: if there are blocked
threads waiting for a write lock, current readers may request more read
locks (which they eventually should free, as they starve the waiting
writers otherwise), but a new thread requesting a read lock will not
be granted one, and block. This might mean starvation for readers if
two writer threads interweave their calls to acquireWrite() without
leaving a window only for readers.
In case a current reader requests a write lock, this can and will be
satisfied without giving up the read locks first, but, only one thread
may perform this kind of lock upgrade, as a deadlock would otherwise
occur. After the write lock has been granted, the thread will hold a
full write lock, and not be downgraded after the upgrading call to
acquireWrite() has been match by a corresponding release().
"""
def __init__(self):
"""Initialize this read-write lock."""
# Condition variable, used to signal waiters of a change in object
# state.
self.__condition = Condition(Lock())
# Initialize with no writers.
self.__writer = None
self.__upgradewritercount = 0
self.__pendingwriters = []
# Initialize with no readers.
self.__readers = {}
def acquireRead(self, timeout=None):
"""Acquire a read lock for the current thread, waiting at most
timeout seconds or doing a non-blocking check in case timeout is <= 0.
In case timeout is None, the call to acquireRead blocks until the
lock request can be serviced.
In case the timeout expires before the lock could be serviced, a
RuntimeError is thrown."""
if timeout is not None:
endtime = time() + timeout
me = currentThread()
self.__condition.acquire()
try:
if self.__writer is me:
# If we are the writer, grant a new read lock, always.
self.__writercount += 1
return
while True:
if self.__writer is None:
# Only test anything if there is no current writer.
if self.__upgradewritercount or self.__pendingwriters:
if me in self.__readers:
# Only grant a read lock if we already have one
# in case writers are waiting for their turn.
# This means that writers can't easily get starved
# (but see below, readers can).
self.__readers[me] += 1
return
# No, we aren't a reader (yet), wait for our turn.
else:
# Grant a new read lock, always, in case there are
# no pending writers (and no writer).
self.__readers[me] = self.__readers.get(me, 0) + 1
return
if timeout is not None:
remaining = endtime - time()
if remaining <= 0:
# Timeout has expired, signal caller of this.
raise RuntimeError("Acquiring read lock timed out")
self.__condition.wait(remaining)
else:
self.__condition.wait()
finally:
self.__condition.release()
def acquireWrite(self, timeout=None):
"""Acquire a write lock for the current thread, waiting at most
timeout seconds or doing a non-blocking check in case timeout is <= 0.
In case the write lock cannot be serviced due to the deadlock
condition mentioned above, a ValueError is raised.
In case timeout is None, the call to acquireWrite blocks until the
lock request can be serviced.
In case the timeout expires before the lock could be serviced, a
RuntimeError is thrown."""
if timeout is not None:
endtime = time() + timeout
me, upgradewriter = currentThread(), False
self.__condition.acquire()
try:
if self.__writer is me:
# If we are the writer, grant a new write lock, always.
self.__writercount += 1
return
elif me in self.__readers:
# If we are a reader, no need to add us to pendingwriters,
# we get the upgradewriter slot.
if self.__upgradewritercount:
# If we are a reader and want to upgrade, and someone
# else also wants to upgrade, there is no way we can do
# this except if one of us releases all his read locks.
# Signal this to user.
raise ValueError(
"Inevitable dead lock, denying write lock")
upgradewriter = True
self.__upgradewritercount = self.__readers.pop(me)
else:
# We aren't a reader, so add us to the pending writers queue
# for synchronization with the readers.
self.__pendingwriters.append(me)
while True:
if not self.__readers and self.__writer is None:
# Only test anything if there are no readers and writers.
if self.__upgradewritercount:
if upgradewriter:
# There is a writer to upgrade, and it's us. Take
# the write lock.
self.__writer = me
self.__writercount = self.__upgradewritercount + 1
self.__upgradewritercount = 0
return
# There is a writer to upgrade, but it's not us.
# Always leave the upgrade writer the advance slot,
# because he presumes he'll get a write lock directly
# from a previously held read lock.
elif self.__pendingwriters[0] is me:
# If there are no readers and writers, it's always
# fine for us to take the writer slot, removing us
# from the pending writers queue.
# This might mean starvation for readers, though.
self.__writer = me
self.__writercount = 1
self.__pendingwriters = self.__pendingwriters[1:]
return
if timeout is not None:
remaining = endtime - time()
if remaining <= 0:
# Timeout has expired, signal caller of this.
if upgradewriter:
# Put us back on the reader queue. No need to
# signal anyone of this change, because no other
# writer could've taken our spot before we got
# here (because of remaining readers), as the test
# for proper conditions is at the start of the
# loop, not at the end.
self.__readers[me] = self.__upgradewritercount
self.__upgradewritercount = 0
else:
# We were a simple pending writer, just remove us
# from the FIFO list.
self.__pendingwriters.remove(me)
raise RuntimeError("Acquiring write lock timed out")
self.__condition.wait(remaining)
else:
self.__condition.wait()
finally:
self.__condition.release()
def release(self):
"""Release the currently held lock.
In case the current thread holds no lock, a ValueError is thrown."""
me = currentThread()
self.__condition.acquire()
try:
if self.__writer is me:
# We are the writer, take one nesting depth away.
self.__writercount -= 1
if not self.__writercount:
# No more write locks; take our writer position away and
# notify waiters of the new circumstances.
self.__writer = None
self.__condition.notifyAll()
elif me in self.__readers:
# We are a reader currently, take one nesting depth away.
self.__readers[me] -= 1
if not self.__readers[me]:
# No more read locks, take our reader position away.
del self.__readers[me]
if not self.__readers:
# No more readers, notify waiters of the new
# circumstances.
self.__condition.notifyAll()
else:
raise ValueError("Trying to release unheld lock")
finally:
self.__condition.release()
def send(s, host, port, window, lock):
# Variaveis compartilhadas
global max_packages
max_packages = 2
global aux_time
aux_time = time()
global aux_lock
aux_lock = Condition()
global print_lock
print_lock = Condition()
# Variavel para controlar o fim do programa
global end
end = False
# Número de sequencia do pacote
seqNumber = 0
# Pacotes enviados
i = 0
payloads = [
'1', '12', '123', '1234', '12345', '123456', '1234567', '12345678',
'123456789'
]
# Envia 15 pacotes ao servidor
while i != 15:
# Demora aleatóriamente entre 0.6 e 1.2 para enviar um novo pacote
sleep(random.uniform(0.6, 1.2))
# Envia pacotes com tamanho aleatório entre 1 e 9 de acordo com o indice da lista "payloads"
data = payloads[random.randint(0, 8)]
# Constroi o pacote para ser enviado
packet = p_client(data, seqNumber)
# Transforma o pacote em um objeto de bytes
packet = pickle.dumps(packet)
lock.acquire()
# Chega se a quantidades de pacotes enviados é maior do que max_package
if i > max_packages:
lock.wait()
lock.release()
s.sendto(packet, (host, port))
print_lock.acquire()
print("Pacote com número de sequência {} enviado".format(seqNumber))
print_lock.release()
# Atualiza o número de sequencia
seqNumber += len(data)
# Adiciona o pacote na janela
lock.acquire()
window[i % 7] = (packet, seqNumber)
lock.release()
# Incrementa os pacotes enviados
i += 1
# Atualiza end
end = True
class PooledDB:
"""Pool for DB-API 2 connections.
After you have created the connection pool, you can use
connection() to get pooled, steady DB-API 2 connections.
"""
version = __version__
def __init__(self,
creator,
mincached=0,
maxcached=0,
maxshared=0,
maxconnections=0,
blocking=False,
maxusage=None,
setsession=None,
reset=True,
failures=None,
ping=1,
*args,
**kwargs):
"""Set up the DB-API 2 connection pool.
creator: either an arbitrary function returning new DB-API 2
connection objects or a DB-API 2 compliant database module
mincached: initial number of idle connections in the pool
(0 means no connections are made at startup)
maxcached: maximum number of idle connections in the pool
(0 or None means unlimited pool size)
maxshared: maximum number of shared connections
(0 or None means all connections are dedicated)
When this maximum number is reached, connections are
shared if they have been requested as shareable.
maxconnections: maximum number of connections generally allowed
(0 or None means an arbitrary number of connections)
blocking: determines behavior when exceeding the maximum
(if this is set to true, block and wait until the number of
connections decreases, otherwise an error will be reported)
maxusage: maximum number of reuses of a single connection
(0 or None means unlimited reuse)
When this maximum usage number of the connection is reached,
the connection is automatically reset (closed and reopened).
setsession: optional list of SQL commands that may serve to prepare
the session, e.g. ["set datestyle to ...", "set time zone ..."]
reset: how connections should be reset when returned to the pool
(False or None to rollback transcations started with begin(),
True to always issue a rollback for safety's sake)
failures: an optional exception class or a tuple of exception classes
for which the connection failover mechanism shall be applied,
if the default (OperationalError, InternalError) is not adequate
ping: determines when the connection should be checked with ping()
(0 = None = never, 1 = default = whenever fetched from the pool,
2 = when a cursor is created, 4 = when a query is executed,
7 = always, and all other bit combinations of these values)
args, kwargs: the parameters that shall be passed to the creator
function or the connection constructor of the DB-API 2 module
"""
try:
threadsafety = creator.threadsafety
except AttributeError:
try:
if not callable(creator.connect):
raise AttributeError
except AttributeError:
threadsafety = 2
else:
threadsafety = 0
if not threadsafety:
raise NotSupportedError("Database module is not thread-safe.")
self._creator = creator
self._args, self._kwargs = args, kwargs
self._maxusage = maxusage
self._setsession = setsession
self._reset = reset
self._failures = failures
self._ping = ping
if mincached is None:
mincached = 0
if maxcached is None:
maxcached = 0
if maxconnections is None:
maxconnections = 0
if maxcached:
if maxcached < mincached:
maxcached = mincached
self._maxcached = maxcached
else:
self._maxcached = 0
if threadsafety > 1 and maxshared:
self._maxshared = maxshared
self._shared_cache = [] # the cache for shared connections
else:
self._maxshared = 0
if maxconnections:
if maxconnections < maxcached:
maxconnections = maxcached
if maxconnections < maxshared:
maxconnections = maxshared
self._maxconnections = maxconnections
else:
self._maxconnections = 0
self._idle_cache = [] # the actual pool of idle connections
self._condition = Condition()
if not blocking:
def wait():
raise TooManyConnections
self._condition.wait = wait
self._connections = 0
# Establish an initial number of idle database connections:
idle = [self.dedicated_connection() for i in range(mincached)]
while idle:
idle.pop().close()
def steady_connection(self):
"""Get a steady, unpooled DB-API 2 connection."""
return connect(self._creator, self._maxusage, self._setsession,
self._failures, self._ping, True, *self._args,
**self._kwargs)
def connection(self, shareable=True):
"""Get a steady, cached DB-API 2 connection from the pool.
If shareable is set and the underlying DB-API 2 allows it,
then the connection may be shared with other threads.
"""
if shareable and self._maxshared:
self._condition.acquire()
try:
while (not self._shared_cache and self._maxconnections
and self._connections >= self._maxconnections):
self._condition.wait()
if len(self._shared_cache) < self._maxshared:
# shared cache is not full, get a dedicated connection
try: # first try to get it from the idle cache
con = self._idle_cache.pop(0)
except IndexError: # else get a fresh connection
con = self.steady_connection()
else:
con._ping_check() # check this connection
con = SharedDBConnection(con)
self._connections += 1
else: # shared cache full or no more connections allowed
self._shared_cache.sort() # least shared connection first
con = self._shared_cache.pop(0) # get it
while con.con._transaction:
# do not share connections which are in a transaction
self._shared_cache.insert(0, con)
self._condition.wait()
self._shared_cache.sort()
con = self._shared_cache.pop(0)
con.con._ping_check() # check the underlying connection
con.share() # increase share of this connection
# put the connection (back) into the shared cache
self._shared_cache.append(con)
self._condition.notify()
finally:
self._condition.release()
con = PooledSharedDBConnection(self, con)
else: # try to get a dedicated connection
self._condition.acquire()
try:
while (self._maxconnections
and self._connections >= self._maxconnections):
self._condition.wait()
# connection limit not reached, get a dedicated connection
try: # first try to get it from the idle cache
con = self._idle_cache.pop(0)
except IndexError: # else get a fresh connection
con = self.steady_connection()
else:
con._ping_check() # check connection
con = PooledDedicatedDBConnection(self, con)
self._connections += 1
finally:
self._condition.release()
return con
def dedicated_connection(self):
"""Alias for connection(shareable=False)."""
return self.connection(False)
def unshare(self, con):
"""Decrease the share of a connection in the shared cache."""
self._condition.acquire()
try:
con.unshare()
shared = con.shared
if not shared: # connection is idle,
try: # so try to remove it
self._shared_cache.remove(con) # from shared cache
except ValueError:
pass # pool has already been closed
finally:
self._condition.release()
if not shared: # connection has become idle,
self.cache(con.con) # so add it to the idle cache
def cache(self, con):
"""Put a dedicated connection back into the idle cache."""
self._condition.acquire()
try:
if not self._maxcached or len(self._idle_cache) < self._maxcached:
con._reset(force=self._reset) # rollback possible transaction
# the idle cache is not full, so put it there
self._idle_cache.append(con) # append it to the idle cache
else: # if the idle cache is already full,
con.close() # then close the connection
self._connections -= 1
self._condition.notify()
finally:
self._condition.release()
def close(self):
"""Close all connections in the pool."""
self._condition.acquire()
try:
while self._idle_cache: # close all idle connections
con = self._idle_cache.pop(0)
try:
con.close()
except Exception:
pass
if self._maxshared: # close all shared connections
while self._shared_cache:
con = self._shared_cache.pop(0).con
try:
con.close()
except Exception:
pass
self._connections -= 1
self._condition.notifyAll()
finally:
self._condition.release()
def __del__(self):
"""Delete the pool."""
try:
self.close()
except Exception:
pass
class VODTransporter(MovieTransport):
def __init__(self, dd, dlhash, fileinfo, vodeventfunc):
self.wait_sufficient_speed = dd.config.get('wait_sufficient_speed',
False)
self.player_buffer_time = dd.config.get('player_buffer_time', 5)
self.live_buffer_time = dd.config.get('live_buffer_time', 10)
self.fileinfo = fileinfo
self.dd = dd
self.rawserver = dd.rawserver
self.downloader = dd.downloader
self.storage = dd.storage
self.vodeventfunc = vodeventfunc
self.log_prefix = 'dd-vod::' + binascii.hexlify(dlhash) + ':'
if DEBUG:
log(self.log_prefix + '__init__: fileinfo', self.fileinfo,
'wait_sufficient_speed', self.wait_sufficient_speed,
'player_buffer_time', self.player_buffer_time)
self.set_mimetype(self.fileinfo['mimetype'])
bitrate = self.fileinfo.get('bitrate', None)
if bitrate is None:
self.bitrate_set = False
self.bitrate = 102400
if DEBUG:
log(self.log_prefix + '__init__: set fake bitrate: bitrate',
self.bitrate)
else:
self.bitrate_set = True
self.bitrate = bitrate
if DEBUG:
log(self.log_prefix + '__init__: got bitrate: bitrate',
self.bitrate)
self.data_ready = Condition()
self._complete = False
self.filestream = None
self.prebufprogress = 0.0
self.prebufstart = time.time()
self.playable = False
self.usernotified = False
self.playing = False
self.prebuffering = True
self.stream_pos = 0
self.outbuf = []
self.stat_outbuf = []
self.outbuflen = 0
self.outbufpos = 0
self.update_prebuffering()
self.refill_buffer_task()
def set_bitrate(self, bitrate):
if DEBUG:
log(self.log_prefix + 'set_bitrate: bitrate', bitrate, 'fileinfo',
self.fileinfo)
self.bitrate = bitrate
self.bitrate_set = True
def is_playable(self):
if not self.playable or self.prebuffering:
self.playable = self.prebufprogress == 1.0 and self.enough_buffer()
return self.playable
def get_prebuffering_progress(self):
return self.prebufprogress
def complete(self):
if DEBUG:
log(self.log_prefix + 'complete: ---')
if not self._complete:
self._complete = True
path = self.storage.get_dest_path()
self.data_ready.acquire()
try:
self.filestream = open(path, 'rb')
self.filestream.seek(self.stream_pos)
if DEBUG:
log(self.log_prefix + 'complete: open file and seek: path',
path, 'stream_pos', self.stream_pos)
self.data_ready.notify()
finally:
self.data_ready.release()
def got_data_observer(self, pos, length):
if self.prebuffering:
self.rawserver.add_task(self.update_prebuffering, 0.1)
return True
else:
return False
def update_prebuffering(self):
if not self.prebuffering:
return
want_len = min(self.bitrate * self.player_buffer_time,
self.storage.get_content_length())
avail_len = self.storage.get_available_length(0)
if avail_len >= want_len and self.enough_buffer():
self.data_ready.acquire()
try:
if DEBUG:
log(self.log_prefix + 'update_prebuffering: ready: want',
want_len, 'avail', avail_len)
self.prebuffering = False
self.notify_playable()
self.data_ready.notify()
return True
finally:
self.data_ready.release()
else:
if DEBUG:
log(self.log_prefix + 'update_prebuffering: not ready: want',
want_len, 'avail', avail_len)
return False
def expected_download_time(self):
bytes_left = self.storage.get_amount_left()
if bytes_left == 0:
return True
rate = self.downloader.measure.get_rate()
if rate < 0.1:
return float(2147483648L)
time_left = bytes_left / float(rate)
return time_left
def expected_playback_time(self):
bytes_to_play = self.storage.get_content_length() - self.outbufpos
if bytes_to_play <= 0:
return 0
if not self.bitrate_set:
return float(2147483648L)
playback_time = bytes_to_play / float(self.bitrate)
return playback_time
def enough_buffer(self):
try:
if not self.bitrate_set:
return True
if not self.wait_sufficient_speed:
return True
expected_download_time = self.expected_download_time()
expected_playback_time = self.expected_playback_time()
if DEBUG:
log(self.log_prefix + 'enough_buffer: expected_download_time',
expected_download_time, 'expected_playback_time',
expected_playback_time)
return max(0.0,
expected_download_time - expected_playback_time) == 0.0
except:
log_exc()
return True
def expected_buffering_time(self):
download_time = self.expected_download_time()
playback_time = self.expected_playback_time()
if download_time > float(1073741824) and playback_time > float(
1073741824):
return float(2147483648L)
return abs(download_time - playback_time)
def get_playable_after(self):
return self.expected_buffering_time()
def notify_playable(self):
self.prebufprogress = 1.0
self.playable = True
if self.usernotified:
return
mimetype = self.get_mimetype()
complete = self.storage.is_finished()
if complete:
stream = None
filename = self.storage.get_dest_path()
else:
stream = MovieTransportStreamWrapper(self)
filename = None
try:
self.vodeventfunc(
self.fileinfo, VODEVENT_START, {
'complete': complete,
'filename': filename,
'mimetype': mimetype,
'stream': stream,
'length': self.storage.get_content_length(),
'bitrate': self.bitrate
})
except:
log_exc()
def get_mimetype(self):
return self.mimetype
def set_mimetype(self, mimetype):
self.mimetype = mimetype
def start(self, bytepos=0, force=False):
if DEBUG:
log(self.log_prefix + 'start: bytepos', bytepos, 'playing',
self.playing, 'force', force)
if self.playing and not force:
return
self.downloader.start(bytepos)
self.data_ready.acquire()
try:
self.stream_pos = bytepos
self.playing = True
if self._complete:
if self.filestream is None:
path = self.storage.get_dest_path()
if DEBUG:
log(self.log_prefix + 'start: open file: path', path)
self.filestream = open(path, 'rb')
if DEBUG:
log(self.log_prefix + 'start: seek file: pos', bytepos)
self.filestream.seek(bytepos)
else:
self.outbuf = []
self.stat_outbuf = []
self.outbuflen = 0
self.outbufpos = self.stream_pos
finally:
self.data_ready.release()
self.update_prebuffering()
self.refill_buffer()
def shutdown(self):
if DEBUG:
log(self.log_prefix + 'shutdown: ---')
self.stop()
def stop(self, seek=False):
if DEBUG:
log(self.log_prefix + 'stop: playing', self.playing, 'seek', seek)
if not self.playing:
return
self.playing = False
self.data_ready.acquire()
try:
self.outbuf = []
self.stat_outbuf = []
self.outbuflen = 0
self.outbufpos = 0
self.prebuffering = False
if not seek:
if self.filestream is not None:
if DEBUG:
log(self.log_prefix + 'stop: close filestream')
self.filestream.close()
self.filestream = None
self.data_ready.notify()
finally:
self.data_ready.release()
def seek(self, pos, whence=os.SEEK_SET):
length = self.storage.get_content_length()
self.data_ready.acquire()
try:
if whence == os.SEEK_SET:
abspos = pos
elif whence == os.SEEK_END:
if pos > 0:
raise ValueError('seeking beyond end of stream')
else:
abspos = length + pos
else:
raise ValueError('seeking does not currently support SEEK_CUR')
if DEBUG:
log(self.log_prefix + 'seek: pos', pos, 'whence', whence,
'length', length, 'abspos', abspos)
if self._complete:
if self.filestream is None:
path = self.storage.get_dest_path()
if DEBUG:
log(self.log_prefix + 'seek: open file:', path)
self.filestream = open(path, 'rb')
if DEBUG:
log(self.log_prefix + 'seek: seek file: abspos', abspos)
self.filestream.seek(abspos)
self.stream_pos = abspos
else:
self.stop(seek=True)
self.start(abspos)
finally:
self.data_ready.release()
def read(self, numbytes=None):
self.data_ready.acquire()
try:
data = self.pop(numbytes)
if data is None:
return
self.stream_pos += len(data)
if DEBUG:
log(self.log_prefix + 'read: update stream pos: stream_pos',
self.stream_pos, 'datalen', len(data))
return data
except:
print_exc()
finally:
self.data_ready.release()
def pop(self, max_size=None):
while self.prebuffering and not self.done():
self.data_ready.wait()
while not self._complete and not self.outbuf and not self.done():
self.data_ready.wait()
if self._complete:
if max_size is None:
max_size = DEFAULT_READ_SIZE
if DEBUG:
log(self.log_prefix + 'pop: read from filestream: max_size',
max_size)
data = self.filestream.read(max_size)
return data
if not self.outbuf:
if DEBUG:
log(
self.log_prefix +
'pop: empty buffer, return None: _complete',
self._complete)
return
while True:
bad_chunk = False
start_pos = None
total_length = 0
total_data = ''
if DEBUG:
log(self.log_prefix + 'pop: read data from outbuf')
while self.outbuf:
pos, data = self.outbuf.pop(0)
self.stat_outbuf.pop(0)
length = len(data)
if max_size is not None and total_length + length > max_size:
offset = total_length + length - max_size
newpos = pos + length - offset
newdata = data[-offset:]
data = data[:-offset]
oldlength = length
length = len(data)
if DEBUG:
log(
self.log_prefix +
'pop: trim chunk to max size: pos', pos, 'oldlen',
oldlength, 'max_size', max_size, 'offset', offset,
'total_len', total_length, 'newlen', length,
'newpos', newpos)
self.outbuf.insert(0, (newpos, newdata))
self.stat_outbuf.insert(0, (newpos, len(newdata)))
if start_pos is None:
start_pos = pos
total_data += data
total_length += length
self.outbuflen -= length
if DEBUG:
log(self.log_prefix + 'pop: outbufpos', self.outbufpos,
'pos', pos, 'start_pos', start_pos, 'len', length,
'total_len', total_length, 'max_size', max_size,
'outbuflen', self.outbuflen)
if not start_pos <= self.stream_pos <= start_pos + total_length:
if DEBUG:
log(
self.log_prefix +
'pop: wrong chunk popped, discard: stream_pos',
self.stream_pos, 'start_pos', start_pos,
'total_len', total_length)
bad_chunk = True
break
if max_size is None:
break
if total_length >= max_size:
if DEBUG:
log(
self.log_prefix +
'pop: stop popping, max size reached: total_length',
total_length, 'max_size', max_size)
break
if bad_chunk:
continue
break
return total_data
def max_buffer_size(self):
return int(self.player_buffer_time * self.bitrate * 2)
def done(self):
if not self.playing:
return True
return self.outbufpos == self.storage.get_content_length() and len(
self.outbuf) == 0
def refill_buffer(self):
self.data_ready.acquire()
try:
if self.prebuffering or self._complete or not self.playing or self.done(
):
return
mx = self.max_buffer_size()
length = self.storage.get_content_length()
while self.outbuflen < mx and self.outbufpos < length:
numbytes = mx - self.outbuflen
if DEBUG:
log(
self.log_prefix +
'refill_buffer: read from storage: pos',
self.outbufpos, 'numbytes', numbytes, 'outbuflen',
self.outbuflen, 'mx', mx)
data = self.storage.read(self.outbufpos, numbytes)
if not data:
if DEBUG:
log(
self.log_prefix +
'refill_buffer: no data available: pos',
self.outbufpos)
break
datalen = len(data)
self.outbuf.append((self.outbufpos, data))
self.stat_outbuf.append((self.outbufpos, datalen))
self.outbuflen += datalen
self.outbufpos += datalen
self.data_ready.notify()
if DEBUG:
log(
self.log_prefix +
'refill_buffer: got data from storage: datalen',
datalen, 'outbufpos', self.outbufpos, 'outbuflen',
self.outbuflen)
except:
log_exc()
finally:
self.data_ready.release()
def refill_buffer_task(self):
self.refill_buffer()
self.rawserver.add_task(self.refill_buffer_task,
REFILL_BUFFER_INTERVAL)
def set_wait_sufficient_speed(self, value):
if DEBUG:
log(self.log_prefix + 'set_wait_sufficient_speed:', value)
self.wait_sufficient_speed = value
def set_player_buffer_time(self, value):
if DEBUG:
log(self.log_prefix + 'set_player_buffer_time:', value)
self.player_buffer_time = value
class farm:
'''init returns an object, we'll call o.run(i), then del o in the end.'''
def __init__(self,
num_threads,
init,
it,
extendable=False,
tn_tmpl=None,
reuse=None,
handler=None):
'''When extendable is True, it means that we need to leave threads ready in case the input list is extended (with extend()). Also the run() function can be invoked several times. The drawback is that underneath the input it is converted to a list, and then manipulated, so it's feasible to start with relatively small input data. Don't forget to call close(), it will clean up all the threads. Or you can use it as a context manager, so this will be called automatically upon __exit__().
If extendable is False, the code is simpler, but it supports iterables however big.
tn_tmpl is format() template with {} to be changed to thread's number.
reuse tells whether to stash worked sessions for future reuse. If it's a list, it's a global list of warm sessions.
A function in the handler parameter is invoked each second, while the processing is postponed, so it shouldn't take long to complete'''
self.waiting = 0
self.extendable = extendable
if self.extendable:
self.arr = list(
it) # If it was a generator for example. Make it real.
lg.info("Total: {}".format(len(self.arr)))
else: # Otherwise leave it as it is, we'll treat it as iterator
self.arr = iter(it)
self.cond = Condition()
self.num_threads = num_threads if num_threads else len(self.arr)
# Barrier to wait on for the restart (when extendable)
self.barr = Barrier(self.num_threads + 1)
self.init = init
self.tn_tmpl = tn_tmpl
self.reuse = reuse
self.handler = handler
# Objects living within threads, used to signal them to quit
# (by setting quit_flag)
self.objects = []
if type(reuse) is list:
self.reuse_pool = reuse
else:
self.reuse_pool = None
#if self.reuse: self.reuse_pool=[]
self.q = Queue()
self.tlist = []
def __del__(self):
# Let the user delete them if it was user-supplied
if type(self.reuse) is not list:
if self.reuse:
for i in self.reuse_pool:
del i
def close(self):
# Join our threads
if self.extendable:
lg.debug('Cancelling threads (break the barrier)')
self.barr.abort() # Let those waiting on the barrier to quit
lg.info('Joining threads')
for i in self.tlist:
i.join()
lg.info('Finished joining threads')
def __enter__(self):
return self
def __exit__(self, *exc):
self.close()
def extend(self, arr, end=False):
'''If end is True, add to the end'''
if not self.extendable:
lg.error(
'The farm is not extendable, "extendable" parameter is False')
return
with self.cond:
orig_len = len(self.arr)
#if type(arr) is GeneratorType: arr=tuple(arr)
if end or self.arr and self.arr[-1] is not None:
self.arr += arr
else: # When we're quitting
self.arr[:
0] = arr # Insert in the beginning, so poison pills won't get lost
lg.info("Total after extend: {}".format(len(self.arr)))
#lg.info("Notifying: {}".format(len(self.arr)-orig_len))
self.cond.notify(len(self.arr) - orig_len)
def print_arr(self):
'''Will be printed on farm exit. Beware to call it while the threads are running if you need the actual list.'''
with self.cond:
print(self.arr)
def reusing(self):
return type(self.reuse_pool) is list
def handle_item(o, i):
'''Handles the item with do() function of the class, passing parameters depending on the nature of the argument: can be presented as several arguments to make things easy. do() function must yield one or several results, that will be returned by farm.run()'''
lg.info('Item: {}'.format(i))
if isinstance(i, (tuple, list, set)): # Present as arguments
yield from o.do(*i)
else:
yield from o.do(i)
#lg.debug('do: {}'.format(res))
#return res
def do_extendable(self):
'''We need to leave threads ready in case the array is extended. Otherwise we can quit right after do() has completed. Also we can use this farm several times, the objects remain live, so we can extend and invoke another run() to gather the results as many times as we want.'''
o = self.init()
with suppress(AttributeError):
if not o.f: o.f = self # Set to the current farm
with self.cond:
self.objects.append(o)
while True:
self.cond.acquire()
self.waiting += 1
lg.debug('waiting incremented: {}, len={}'.format(
self.waiting, len(self.arr)))
if not len(self.arr):
if self.waiting == self.num_threads:
# No threads left to replenish the array, we should all quit
# Adding poison pills
if 1:
lg.info('Killing all')
# Put poison pills for everyone including us
self.arr += [None] * (self.num_threads)
self.cond.notify(
self.num_threads) # Wake up other threads
else:
lg.info('Killing all')
self.arr += [None] * (self.num_threads - 1)
self.cond.notify(self.num_threads - 1)
self.cond.release()
break
else:
self.cond.wait() # Someone else will kill us
lm = len(self.arr)
if lm: # Another check for those who have left cond.wait()
i = self.arr.pop()
self.waiting -= 1
lg.debug('waiting decremented: ' + str(self.waiting))
self.cond.release()
if not lm: continue # Someone has stolen our item, snap!
if i is None:
self.q.put(None) # Mark we're done
# Sleep on the condition to let other threads get their pills
lg.debug('Sleeping on barrier')
try:
self.barr.wait()
except BrokenBarrierError: # We're to quit
break
lg.debug('Continuing after barrier')
continue # then restart processing the queue
for j in farm.handle_item(o, i):
self.q.put(j)
with self.cond:
self.objects.remove(o)
del o
lg.info("has finished")
def do(self):
'''if an item from the iterator is a tuple, we explode it to be arguments to do(). Otherwise we pass it verbatim'''
#tracker = SummaryTracker()
o = None
if self.reusing(): # Try to get warm session
with self.cond:
if len(self.reuse_pool): o = self.reuse_pool.pop()
if not o: o = self.init()
with suppress(AttributeError):
if not o.f: o.f = self # Set to the current farm
with self.cond:
self.objects.append(o)
#lg.warning(len(self.arr))
while True:
with self.cond:
try:
i = next(self.arr)
except StopIteration: # empty
break
if i is None: break # End of queue marker
for j in farm.handle_item(o, i):
self.q.put(j)
#lg.error(asizeof.asizeof(o))
with self.cond:
self.objects.remove(o)
if self.reusing():
self.reuse_pool.append(o)
else:
del o # This will release proxy back to the pool
self.q.put(None) # Mark the thread has finished
lg.info("has finished")
#tracker.print_diff()
def cancel(self, cnt=0):
'''Cancels all the threads in the farm'''
# Put poison pills and then signal the threads to stop
if not cnt: cnt = self.num_threads # That many threads are running
with self.cond:
if self.extendable:
self.arr += [None] * cnt
else:
self.arr = chain(repeat(None, cnt), self.arr)
self.cond.notify(cnt)
for _ in self.objects:
_.quit_flag = True
def run(self):
'''Main function to invoke. When KeyboardInterrupt is received, it sets the quit_flag in all the objects present, retrievers then raise an exception. It's the problem of the do() function to handle it and possibly extend the main list with the item that wasn't handled to show it in the end (for possible restart)
self.handler() function is invoked each second if there are no items in the queue to allow for some rudimental auxiliary activity'''
# Now start the threads, only once
if self.tlist:
lg.debug('Restarting threads')
self.barr.wait()
else:
for i in range(self.num_threads):
tn = self.tn_tmpl.format(i) if self.tn_tmpl else None
t = Thread(
target=self.do_extendable if self.extendable else self.do,
name=tn)
self.tlist.append(t)
t.start()
cnt = self.num_threads # That many threads are running
while True:
try:
res = self.q.get(timeout=1)
except Empty:
if self.handler: self.handler()
continue # Go back to suck the queue
except KeyboardInterrupt:
lg.warning(
'Trying to kill nicely, putting {} None'.format(cnt))
self.cancel(cnt)
res = None
#lg.warning('run: {}'.format(res))
if res != None:
yield res
continue
cnt -= 1 # One thread has completed
if not cnt: # All threads are completed (but may still be processing the quit_flag), we'll join them in close()
return
class URLHandler(object):
def __init__(self, *args, **kwargs):
log.debug("create urlhandler {} {}".format(args, kwargs))
self.pending = {}
self.name = kwargs.pop('name', None)
self.content_handler = kwargs.pop('content_handler', None)
self._setup()
def _setup(self):
self.done = Condition()
self.lock = Lock()
self.fetcher = make_fetcher(name=self.name,
content_handler=self.content_handler)
self.fetcher.add_watcher(self)
def __getstate__(self):
return dict(name=self.name)
def __setstate__(self, state):
self.__dict__.update(state)
self._setup()
def is_done(self):
return self.count == 0
def thing_to_url(self, t):
return t
@property
def count(self):
return len(self.pending)
def schedule(self, things):
try:
self.lock.acquire()
self.i_schedule(things)
finally:
self.lock.release()
def i_schedule(self, things):
for t in things:
self.pending[self.thing_to_url(t)] = t
self.fetcher.schedule(self.thing_to_url(t))
def i_handle(self,
t,
url=None,
response=None,
exception=None,
last_fetched=None):
raise NotImplementedError()
def __call__(self,
watched=None,
url=None,
response=None,
exception=None,
last_fetched=None):
if url in self.pending:
t = self.pending[url]
with self.lock:
log.debug("RESPONSE url={}, exception={} @ {}".format(
url, exception, self.count))
self.i_handle(t,
url=url,
response=response,
exception=exception,
last_fetched=last_fetched)
del self.pending[url]
if self.is_done():
try:
self.done.acquire()
self.done.notify()
finally:
self.done.release()
class Application(object):
def __init__(self):
self.is_authenticating = Condition()
self.authorization = None
# Bind trakt events
Trakt.on('oauth.token_refreshed', self.on_token_refreshed)
def authenticate(self):
if not self.is_authenticating.acquire(blocking=False):
print('Authentication has already been started')
return False
# Request new device code
code = Trakt['oauth/device'].code()
print('Enter the code "%s" at %s to authenticate your account' %
(code.get('user_code'), code.get('verification_url')))
# Construct device authentication poller
poller = Trakt['oauth/device'].poll(**code)\
.on('aborted', self.on_aborted)\
.on('authenticated', self.on_authenticated)\
.on('expired', self.on_expired)\
.on('poll', self.on_poll)
# Start polling for authentication token
poller.start(daemon=False)
# Wait for authentication to complete
return self.is_authenticating.wait()
def run(self):
self.authenticate()
if not self.authorization:
print('ERROR: Authentication required')
exit(1)
# Simulate expired token
self.authorization['expires_in'] = 0
# Test authenticated calls
with Trakt.configuration.oauth.from_response(self.authorization):
# Expired token, requests will return `None`
print(Trakt['sync/collection'].movies())
with Trakt.configuration.oauth.from_response(self.authorization,
refresh=True):
# Expired token will be refreshed automatically (as `refresh=True`)
print(Trakt['sync/collection'].movies())
with Trakt.configuration.oauth.from_response(self.authorization):
# Current token is still valid
print(Trakt['sync/collection'].movies())
def on_aborted(self):
"""Device authentication aborted.
Triggered when device authentication was aborted (either with `DeviceOAuthPoller.stop()`
or via the "poll" event)
"""
print('Authentication aborted')
# Authentication aborted
self.is_authenticating.acquire()
self.is_authenticating.notify_all()
self.is_authenticating.release()
def on_authenticated(self, authorization):
"""Device authenticated.
:param authorization: Authentication token details
:type authorization: dict
"""
# Acquire condition
self.is_authenticating.acquire()
# Store authorization for future calls
self.authorization = authorization
print('Authentication successful - authorization: %r' %
self.authorization)
# Authentication complete
self.is_authenticating.notify_all()
self.is_authenticating.release()
def on_expired(self):
"""Device authentication expired."""
print('Authentication expired')
# Authentication expired
self.is_authenticating.acquire()
self.is_authenticating.notify_all()
self.is_authenticating.release()
def on_poll(self, callback):
"""Device authentication poll.
:param callback: Call with `True` to continue polling, or `False` to abort polling
:type callback: func
"""
# Continue polling
callback(True)
def on_token_refreshed(self, authorization):
# OAuth token refreshed, store authorization for future calls
self.authorization = authorization
print('Token refreshed - authorization: %r' % self.authorization)
class ForkingBase(object):
'''Base class for classes which fork children and wait for them to exit.
Sub-classes must provide the following data attributes and methods:
log - an object of type getmailcore.logging.Logger()
'''
def _child_handler(self, sig, stackframe):
def notify():
self.__child_exited.acquire()
self.__child_exited.notify_all()
self.__child_exited.release()
self.log.trace('handler called for signal %s' % sig)
try:
pid, r = os.waitpid(self.child.childpid, 0)
except OSError as o:
# No children on SIGCHLD. Can't happen?
self.log.warning('handler called, but no children (%s)' % o)
notify()
return
signal.signal(signal.SIGCHLD, self.__orig_handler)
self.__child_pid = pid
self.__child_status = r
self.log.trace('handler reaped child %s with status %s' % (pid, r))
notify()
def _prepare_child(self):
self.log.trace('')
self.__child_exited = Condition()
self.__child_pid = 0
self.__child_status = None
self.__orig_handler = signal.signal(signal.SIGCHLD,
self._child_handler)
def _wait_for_child(self, childpid):
self.__child_exited.acquire()
if self.__child_exited.wait(
socket.getdefaulttimeout()
or 60) == False: # Py2, <Py3.2: always None
raise getmailOperationError('waiting child pid %d timed out' %
childpid)
self.__child_exited.release()
if self.__child_pid != childpid:
#self.log.error('got child pid %d, not %d' % (pid, childpid))
raise getmailOperationError('got child pid %d, not %d' %
(self.__child_pid, childpid))
if os.WIFSTOPPED(self.__child_status):
raise getmailOperationError(
'child pid %d stopped by signal %d' %
(self.__child_pid, os.WSTOPSIG(self.__child_status)))
if os.WIFSIGNALED(self.__child_status):
raise getmailOperationError(
'child pid %d killed by signal %d' %
(self.__child_pid, os.WTERMSIG(self.__child_status)))
if not os.WIFEXITED(self.__child_status):
raise getmailOperationError('child pid %d failed to exit' %
self.__child_pid)
exitcode = os.WEXITSTATUS(self.__child_status)
return exitcode
def _pipemail(self, msg, delivered_to, received, unixfrom, stdout, stderr):
# Write out message
msgfile = TemporaryFile23()
msgfile.write(
msg.flatten(delivered_to, received, include_from=unixfrom))
msgfile.flush()
os.fsync(msgfile.fileno())
# Rewind
msgfile.seek(0)
# Set stdin to read from this file
os.dup2(msgfile.fileno(), 0)
# Set stdout and stderr to write to files
os.dup2(stdout.fileno(), 1)
os.dup2(stderr.fileno(), 2)
def child_replace_me(self,
msg,
delivered_to,
received,
unixfrom,
stdout,
stderr,
args,
nolog=False):
self._pipemail(msg, delivered_to, received, unixfrom, stdout, stderr)
nolog or self.log.debug('about to execl() with args %s\n' % str(args))
os.execl(*args)
def forkchild(self, childfun, with_out=True):
self.child = child = Namespace()
child.stdout = TemporaryFile23()
child.stderr = TemporaryFile23()
child.childpid = os.fork()
if child.childpid != 0: # here (in the parent)
self._prepare_child()
self.log.debug('spawned child %d\n' % child.childpid)
child.exitcode = self._wait_for_child(child.childpid)
child.stderr.seek(0)
child.err = child.stderr.read().strip().decode()
child.stdout.seek(0)
if with_out:
child.out = child.stdout.read().strip()
return child
else: #== 0 in the child
# calls child_replace_me to execl external command
childfun(child.stdout, child.stderr)
def get_msginfo(self, msg):
msginfo = {}
msginfo['sender'] = msg.sender.strip()
if msg.recipient != None:
rcpnt = msg.recipient.strip()
msginfo['recipient'] = rcpnt
msginfo['domain'] = rcpnt.lower().split('@')[-1]
msginfo['local'] = '@'.join(rcpnt.split('@')[:-1])
self.log.debug('msginfo "%s"\n' % msginfo)
return msginfo
class SrRecver:
def __init__(self, addr):
self.__sock = socket(AF_INET, SOCK_DGRAM)
self.__sock.bind(addr)
self.__window = []
# 定义窗口中最前端的数据的seq
self.__seq_range = []
self.__max_seq = 255
self.__window_size = 10
self.__notify_recv = Condition()
self.__recv_cache = []
t = Thread(target=self.__recv_thread)
t.start()
def recv(self):
"""
提供给上层应用调用的接口，用来接收数据。
:return: bytes类型的数据
"""
self.__notify_recv.acquire()
while len(self.__recv_cache) == 0:
self.__notify_recv.wait()
self.__notify_recv.release()
data = self.__recv_cache[0]
del self.__recv_cache[0]
return data
def __find_index(self, seq):
"""
查找seq是不是在窗口中，如果是的话返回其在窗口中的index，如果不在的话返回None
:param seq: 要查找的seq
:return:
"""
for i in range(self.__window_size):
if self.__seq_range[i] == seq:
return i
return None
def __recv_thread(self):
# 初始化seq_range和window为固定大小的数组
for i in range(self.__window_size):
self.__seq_range.append(i + 1)
self.__window.append(None)
while True:
data, addr = self.__sock.recvfrom(4096)
if lib.checksum(data) == 0xffff:
seq = data[1]
if seq in self.__seq_range:
ack = bytes([255, seq])
ack += (lib.checksum(ack) ^ 0xffff).to_bytes(2, 'big')
self.__sock.sendto(ack, addr)
index = self.__find_index(seq)
self.__window[index] = data
# 移动窗口，将接收到的数据移动到缓存中
while self.__window[0] is not None:
data = self.__window[0]
self.__min_seq = data[1] + 1
if self.__min_seq == self.__max_seq:
self.__min_seq = 0
data_len = int.from_bytes(data[2:6], 'big')
self.__recv_cache.append(self.__window[0][6:6+data_len])
self.__notify_recv.acquire()
self.__notify_recv.notify()
self.__notify_recv.release()
self.__window.pop(0)
self.__seq_range.pop(0)
self.__window.append(None)
# 维护可以接收的seq范围
seq = self.__seq_range[self.__window_size - 2] + 1
if seq > self.__max_seq:
seq = 0
self.__seq_range.append(seq)
class X6500FPGA(BaseWorker):
# Constructor, gets passed a reference to the miner core, the X6500Worker,
# its FPGA id, and the bitstream version currently running on that FPGA
def __init__(self, core, parent, fpga, bitstreamversion):
self.parent = parent
self.fpga = fpga
self.firmware_rev = bitstreamversion
# Let our superclass do some basic initialization
super(X6500FPGA, self).__init__(core, None)
# Initialize wakeup flag for the main thread.
# This serves as a lock at the same time.
self.wakeup = Condition()
# Validate settings, mostly coping them from our parent
# Called from the constructor and after every settings change on the parent.
def apply_settings(self):
self.settings.name = "%s FPGA%d" % (self.parent.settings.name,
self.fpga)
# Let our superclass handle everything that isn't specific to this worker module
super(X6500FPGA, self).apply_settings()
# Reset our state. Called both from the constructor and from self.start().
def _reset(self):
# Let our superclass handle everything that isn't specific to this worker module
super(X6500FPGA, self)._reset()
self.stats.temperature = None
self.stats.speed = None
self.initialramp = True
# Start up the worker module. This is protected against multiple calls and concurrency by a wrapper.
def _start(self):
# Let our superclass handle everything that isn't specific to this worker module
super(X6500FPGA, self)._start()
# Assume a default job interval to make the core start fetching work for us.
# The actual hashrate will be measured (and this adjusted to the correct value) later.
self.jobs_per_second = 1. / self.parent.settings.jobinterval
# This worker will only ever process one job at once. The work fetcher needs this information
# to estimate how many jobs might be required at once in the worst case (after a block was found).
self.parallel_jobs = 1
# Reset the shutdown flag for our threads
self.shutdown = False
# Start up the main thread, which handles pushing work to the device.
self.mainthread = Thread(None, self.main, self.settings.name + "_main")
self.mainthread.daemon = True
self.mainthread.start()
# Stut down the worker module. This is protected against multiple calls and concurrency by a wrapper.
def _stop(self):
# Let our superclass handle everything that isn't specific to this worker module
super(X6500FPGA, self)._stop()
# Set the shutdown flag for our threads, making them terminate ASAP.
self.shutdown = True
# Trigger the main thread's wakeup flag, to make it actually look at the shutdown flag.
with self.wakeup:
self.wakeup.notify()
# Wait for the main thread to terminate.
self.mainthread.join(2)
# This function should interrupt processing of the specified job if possible.
# This is necesary to avoid producing stale shares after a new block was found,
# or if a job expires for some other reason. If we don't know about the job, just ignore it.
# Never attempts to fetch a new job in here, always do that asynchronously!
# This needs to be very lightweight and fast. We don't care whether it's a
# graceful cancellation for this module because the work upload overhead is low.
def notify_canceled(self, job, graceful):
# Acquire the wakeup lock to make sure that nobody modifies job/nextjob while we're looking at them.
with self.wakeup:
# If the currently being processed, or currently being uploaded job are affected,
# wake up the main thread so that it can request and upload a new job immediately.
if self.job == job: self.wakeup.notify()
# Report custom statistics.
def _get_statistics(self, stats, childstats):
# Let our superclass handle everything that isn't specific to this worker module
super(X6500FPGA, self)._get_statistics(stats, childstats)
stats.temperature = self.stats.temperature
# Main thread entry point
# This thread is responsible for fetching work and pushing it to the device.
def main(self):
# If we're currently shutting down, just die. If not, loop forever,
# to recover from possible errors caught by the huge try statement inside this loop.
# Count how often the except for that try was hit recently. This will be reset if
# there was no exception for at least 5 minutes since the last one.
tries = 0
while not self.shutdown:
try:
# Record our starting timestamp, in order to back off if we repeatedly die
starttime = time.time()
# Initialize megahashes per second to zero, will be measured later.
self.stats.mhps = 0
# Job that the device is currently working on, or that is currently being uploaded.
# This variable is used by BaseWorker to figure out the current work source for statistics.
self.job = None
# Job that was previously being procesed. Has been destroyed, but there might be some late nonces.
self.oldjob = None
# We keep control of the wakeup lock at all times unless we're sleeping
self.wakeup.acquire()
# Eat up leftover wakeups
self.wakeup.wait(0)
# Honor shutdown flag (in case it was a real wakeup)
if self.shutdown: break
# Set validation success flag to false
self.checksuccess = False
# Set validation job second iteration flag to false
self.seconditeration = False
# Initialize hash rate tracking data
self.lasttime = None
self.lastnonce = None
# Initialize malfunction tracking data
self.recentshares = 0
self.recentinvalid = 0
# Configure core clock, if the bitstream supports that
if self.firmware_rev > 0:
self._set_speed(self.parent.settings.initialspeed)
# Clear FPGA's nonce queue
self.parent.clear_queue(self.fpga)
# Send validation job to device
job = ValidationJob(
self.core,
unhexlify(
b"00000001c3bf95208a646ee98a58cf97c3a0c4b7bf5de4c89ca04495000005200000000024d1fff8d5d73ae11140e4e48032cd88ee01d48c67147f9a09cd41fdec2e25824f5c038d1a0b350c5eb01f04"
))
self._sendjob(job)
# Wait for the validation job to complete. The wakeup flag will be set by the listener
# thread when the validation job completes. 180 seconds should be sufficient for devices
# down to about 50MH/s, for slower devices this timeout will need to be increased.
if self.stats.speed:
self.wakeup.wait(
(2**32 / 1000000. / self.stats.speed) * 1.1)
else:
self.wakeup.wait(180)
# Honor shutdown flag
if self.shutdown: break
# We woke up, but the validation job hasn't succeeded in the mean time.
# This usually means that the wakeup timeout has expired.
if not self.checksuccess:
raise Exception(
"Timeout waiting for validation job to finish")
# self.stats.mhps has now been populated by the listener thread
self.core.log(self, "Running at %f MH/s\n" % self.stats.mhps,
300, "B")
self._update_job_interval()
# Main loop, continues until something goes wrong or we're shutting down.
while not self.shutdown:
# Fetch a job, add 2 seconds safety margin to the requested minimum expiration time.
# Blocks until one is available. Because of this we need to release the
# wakeup lock temporarily in order to avoid possible deadlocks.
self.wakeup.release()
job = self.core.get_job(self, self.jobinterval + 2)
self.wakeup.acquire()
# If a new block was found while we were fetching that job, just discard it and get a new one.
if job.canceled:
job.destroy()
continue
# Upload the job to the device
self._sendjob(job)
# Go through the safety checks and reduce the clock if necessary
self.safetycheck()
# If the job was already caught by a long poll while we were uploading it,
# jump back to the beginning of the main loop in order to immediately fetch new work.
# Don't check for the canceled flag before the job was accepted by the device,
# otherwise we might get out of sync.
if self.job.canceled: continue
# Wait while the device is processing the job. If nonces are sent by the device, they
# will be processed by the listener thread. If the job gets canceled, we will be woken up.
self.wakeup.wait(self.jobinterval)
# If something went wrong...
except Exception as e:
# ...complain about it!
self.core.log(self, "%s\n" % traceback.format_exc(), 100, "rB")
finally:
# We're not doing productive work any more, update stats and destroy current job
self._jobend()
self.stats.mhps = 0
try:
self.wakeup.release()
except:
pass
# If we aren't shutting down, figure out if there have been many errors recently,
# and if yes, restart the parent worker as well.
if not self.shutdown:
tries += 1
if time.time() - starttime >= 300: tries = 0
with self.wakeup:
if tries > 5:
self.parent.async_restart()
return
else:
self.wakeup.wait(1)
# Restart (handled by "while not self.shutdown:" loop above)
def notify_nonce_found(self, now, nonce):
# Snapshot the current jobs to avoid race conditions
oldjob = self.oldjob
newjob = self.job
# If there is no job, this must be a leftover from somewhere, e.g. previous invocation
# or reiterating the keyspace because we couldn't provide new work fast enough.
# In both cases we can't make any use of that nonce, so just discard it.
if not oldjob and not newjob: return
# Pass the nonce that we found to the work source, if there is one.
# Do this before calculating the hash rate as it is latency critical.
job = None
if newjob:
if newjob.nonce_found(nonce, oldjob): job = newjob
if not job and oldjob:
if oldjob.nonce_found(nonce): job = oldjob
self.recentshares += 1
if not job: self.recentinvalid += 1
nonceval = struct.unpack("<I", nonce)[0]
if isinstance(newjob, ValidationJob):
# This is a validation job. Validate that the nonce is correct, and complain if not.
if newjob.nonce != nonce:
raise Exception(
"Mining device is not working correctly (returned %s instead of %s)"
% (hexlify(nonce).decode("ascii"), hexlify(
newjob.nonce).decode("ascii")))
else:
# The nonce was correct
if self.firmware_rev > 0:
# The FPGA is running overclocker firmware, so we don't need to use this method to calculate the hashrate.
# In fact, it will not work because the FPGA will go to sleep after working through all possible nonces.
with self.wakeup:
self.checksuccess = True
self.wakeup.notify()
else:
if self.seconditeration == True:
with self.wakeup:
# This is the second iteration. We now know the actual nonce rotation time.
delta = (now - newjob.starttime)
# Calculate the hash rate based on the nonce rotation time.
self.stats.mhps = 2**32 / 1000000. / delta
# Update hash rate tracking information
self.lasttime = now
self.lastnonce = nonceval
# Wake up the main thread
self.checksuccess = True
self.wakeup.notify()
else:
with self.wakeup:
# This was the first iteration. Wait for another one to figure out nonce rotation time.
newjob.starttime = now
self.seconditeration = True
else:
if self.firmware_rev == 0:
# Adjust hash rate tracking
delta = (now - self.lasttime)
estimatednonce = int(
round(self.lastnonce + self.stats.mhps * 1000000 * delta))
noncediff = nonceval - (estimatednonce & 0xffffffff)
if noncediff < -0x80000000: noncediff = noncediff + 0x100000000
elif noncediff > 0x7fffffff:
noncediff = noncediff - 0x100000000
estimatednonce = estimatednonce + noncediff
# Calculate the hash rate based on adjusted tracking information
currentmhps = (estimatednonce -
self.lastnonce) / 1000000. / delta
weight = min(0.5, delta / 100.)
self.stats.mhps = (
1 - weight) * self.stats.mhps + weight * currentmhps
# Update hash rate tracking information
self.lasttime = now
self.lastnonce = nonceval
# This function uploads a job to the device
def _sendjob(self, job):
# Move previous job to oldjob, and new one to job
self.oldjob = self.job
self.job = job
# Send it to the FPGA
start, now = self.parent.send_job(self.fpga, job)
# Calculate how long the old job was running
if self.oldjob:
if self.oldjob.starttime:
self.oldjob.hashes_processed(
(now - self.oldjob.starttime) * self.stats.mhps * 1000000)
self.oldjob.destroy()
self.job.starttime = now
# This function needs to be called whenever the device terminates working on a job.
# It calculates how much work was actually done for the job and destroys it.
def _jobend(self, now=None):
# Hack to avoid a python bug, don't integrate this into the line above
if not now: now = time.time()
# Calculate how long the job was actually running and multiply that by the hash
# rate to get the number of hashes calculated for that job and update statistics.
if self.job != None:
if self.job.starttime:
self.job.hashes_processed(
(now - self.job.starttime) * self.stats.mhps * 1000000)
# Destroy the job, which is neccessary to actually account the calculated amount
# of work to the worker and work source, and to remove the job from cancelation lists.
self.oldjob = self.job
self.job.destroy()
self.job = None
# Check the invalid rate and temperature, and reduce the FPGA clock if these exceed safe values
def safetycheck(self):
warning = False
critical = False
if self.recentinvalid >= self.parent.settings.invalidwarning:
warning = True
if self.recentinvalid >= self.parent.settings.invalidcritical:
critical = True
if self.stats.temperature:
if self.stats.temperature > self.parent.settings.tempwarning:
warning = True
if self.stats.temperature > self.parent.settings.tempcritical:
critical = True
threshold = self.parent.settings.warmupstepshares if self.initialramp and not self.recentinvalid else self.parent.settings.speedupthreshold
if warning:
self.core.log(self, "Detected overload condition!\n", 200, "y")
if critical:
self.core.log(self, "Detected CRITICAL condition!\n", 100, "rB")
if critical:
speedstep = -20
self.initialramp = False
elif warning:
speedstep = -2
self.initialramp = False
elif not self.recentinvalid and self.recentshares >= threshold:
speedstep = 2
else:
speedstep = 0
if self.firmware_rev > 0:
if speedstep: self._set_speed(self.stats.speed + speedstep)
elif warning or critical:
self.core.log(
self,
"Firmware too old, can not automatically reduce clock!\n", 200,
"yB")
if critical:
self.core.log(self, "Shutting down FPGA to protect it!\n", 100,
"rB")
self.parent.shutdown_fpga(self.fpga)
self.async_stop(2)
if speedstep or self.recentshares >= threshold:
self.recentinvalid = 0
self.recentshares = 0
def _set_speed(self, speed):
speed = min(max(speed, 4), self.parent.settings.maximumspeed)
if self.stats.speed == speed: return
if speed == self.parent.settings.maximumspeed: self.initialramp = False
self.core.log(
self, "%s: Setting clock speed to %.2f MHz...\n" %
("Warmup" if self.initialramp else "Tracking", speed), 500, "B")
self.parent.set_speed(self.fpga, speed)
self.stats.speed = self.parent.get_speed(self.fpga)
self.stats.mhps = self.stats.speed
self._update_job_interval()
if self.stats.speed != speed:
self.core.log(self, "Setting clock speed failed!\n", 100, "rB")
def _update_job_interval(self):
self.core.event(350, self, "speed", self.stats.mhps * 1000,
"%f MH/s" % self.stats.mhps, self)
# Calculate the time that the device will need to process 2**32 nonces.
# This is limited at 60 seconds in order to have some regular communication,
# even with very slow devices (and e.g. detect if the device was unplugged).
interval = min(60, 2**32 / 1000000. / self.stats.mhps)
# Add some safety margin and take user's interval setting (if present) into account.
self.jobinterval = min(self.parent.settings.jobinterval,
max(0.5, interval * 0.8 - 1))
self.core.log(self, "Job interval: %f seconds\n" % self.jobinterval,
400, "B")
# Tell the MPBM core that our hash rate has changed, so that it can adjust its work buffer.
self.jobs_per_second = 1. / self.jobinterval
self.core.notify_speed_changed(self)
class StreamPump(Thread):
def __init__(self, sink, size):
Thread.__init__(self)
self.rPos = 0
self.wPos = 0
self.buff = bytearray(size)
self.size = size
self.sink = sink
self.cond = Condition()
self.isRunning = True
def write(self, data):
nOfBytes = len(data)
overflow = self.wPos + nOfBytes - self.size
if overflow >= 0:
#print("%d > %d ov" %(overflow,self.rPos))
if self.wPos < self.rPos < self.size or 0 < self.rPos < overflow:
print("Error :-(")
raise BaseException("Buffer overflow")
self.buff[self.wPos:self.size] = data[0:nOfBytes-overflow]
self.buff[0:overflow] = data[nOfBytes-overflow:nOfBytes]
self.wPos = overflow
else:
newWpos = self.wPos+nOfBytes
#print("%d < %d ov" %(newWpos,self.rPos))
if self.wPos < self.rPos < newWpos:
print("Error :-(")
raise BaseException("Buffer overflow")
self.buff[self.wPos:newWpos] = data
self.wPos = newWpos
self.cond.acquire()
self.cond.notify()
self.cond.release()
return nOfBytes
def run(self):
self.isRunning = True
while True:
while self.rPos == self.wPos:
if self.isRunning:
self.cond.acquire()
self.cond.wait()
self.cond.release()
else:
return
# make sure to always use same wPos
wPos = self.wPos
if self.rPos > wPos:
self.sink.write(self.buff[self.rPos:self.size])
self.rPos = 0
self.sink.write(self.buff[self.rPos:wPos])
self.rPos = wPos
def finish(self):
#print("Finish")
self.cond.acquire()
self.isRunning = False
self.cond.notify()
self.cond.release()
self.join()
class IbApi(EWrapper):
""""""
data_filename = "ib_contract_data.db"
data_filepath = str(get_file_path(data_filename))
def __init__(self, gateway: BaseGateway):
""""""
super().__init__()
self.gateway = gateway
self.gateway_name = gateway.gateway_name
self.status = False
self.reqid = 0
self.orderid = 0
self.clientid = 0
self.account = ""
self.ticks = {}
self.orders = {}
self.accounts = {}
self.contracts = {}
self.tick_exchange = {}
self.history_req = None
self.history_condition = Condition()
self.history_buf = []
self.client = IbClient(self)
self.thread = Thread(target=self.client.run)
def connectAck(self): # pylint: disable=invalid-name
"""
Callback when connection is established.
"""
self.status = True
self.gateway.write_log("IB TWS连接成功")
self.load_contract_data()
def connectionClosed(self): # pylint: disable=invalid-name
"""
Callback when connection is closed.
"""
self.status = False
self.gateway.write_log("IB TWS连接断开")
def nextValidId(self, orderId: int): # pylint: disable=invalid-name
"""
Callback of next valid orderid.
"""
super().nextValidId(orderId)
if not self.orderid:
self.orderid = orderId
def currentTime(self, time: int): # pylint: disable=invalid-name
"""
Callback of current server time of IB.
"""
super().currentTime(time)
dt = datetime.fromtimestamp(time)
time_string = dt.strftime("%Y-%m-%d %H:%M:%S.%f")
msg = f"服务器时间: {time_string}"
self.gateway.write_log(msg)
def error(
self, reqId: TickerId, errorCode: int, errorString: str
): # pylint: disable=invalid-name
"""
Callback of error caused by specific request.
"""
super().error(reqId, errorCode, errorString)
msg = f"信息通知，代码：{errorCode}，内容: {errorString}"
self.gateway.write_log(msg)
def tickPrice( # pylint: disable=invalid-name
self, reqId: TickerId, tickType: TickType, price: float, attrib: TickAttrib
):
"""
Callback of tick price update.
"""
super().tickPrice(reqId, tickType, price, attrib)
if tickType not in TICKFIELD_IB2VT:
return
tick = self.ticks[reqId]
name = TICKFIELD_IB2VT[tickType]
setattr(tick, name, price)
# Update name into tick data.
contract = self.contracts.get(tick.vt_symbol, None)
if contract:
tick.name = contract.name
# Forex and spot product of IDEALPRO has no tick time and last price.
# We need to calculate locally.
exchange = self.tick_exchange[reqId]
if exchange is Exchange.IDEALPRO:
tick.last_price = (tick.bid_price_1 + tick.ask_price_1) / 2
tick.datetime = datetime.now()
self.gateway.on_tick(copy(tick))
def tickSize(
self, reqId: TickerId, tickType: TickType, size: int
): # pylint: disable=invalid-name
"""
Callback of tick volume update.
"""
super().tickSize(reqId, tickType, size)
if tickType not in TICKFIELD_IB2VT:
return
tick = self.ticks[reqId]
name = TICKFIELD_IB2VT[tickType]
setattr(tick, name, size)
self.gateway.on_tick(copy(tick))
def tickString(
self, reqId: TickerId, tickType: TickType, value: str
): # pylint: disable=invalid-name
"""
Callback of tick string update.
"""
super().tickString(reqId, tickType, value)
if tickType != TickTypeEnum.LAST_TIMESTAMP:
return
tick = self.ticks[reqId]
tick.datetime = datetime.fromtimestamp(value)
self.gateway.on_tick(copy(tick))
def orderStatus( # pylint: disable=invalid-name
self,
orderId: OrderId,
status: str,
filled: float,
remaining: float,
avgFillPrice: float,
permId: int,
parentId: int,
lastFillPrice: float,
clientId: int,
whyHeld: str,
mktCapPrice: float,
):
"""
Callback of order status update.
"""
super().orderStatus(
orderId,
status,
filled,
remaining,
avgFillPrice,
permId,
parentId,
lastFillPrice,
clientId,
whyHeld,
mktCapPrice,
)
orderid = str(orderId)
order = self.orders.get(orderid, None)
order.traded = filled
# To filter PendingCancel status
order_status = STATUS_IB2VT.get(status, None)
if order_status:
order.status = order_status
self.gateway.on_order(copy(order))
def openOrder( # pylint: disable=invalid-name
self,
orderId: OrderId,
ib_contract: Contract,
ib_order: Order,
orderState: OrderState,
):
"""
Callback when opening new order.
"""
super().openOrder(
orderId, ib_contract, ib_order, orderState
)
orderid = str(orderId)
order = OrderData(
symbol=ib_contract.conId,
exchange=EXCHANGE_IB2VT.get(
ib_contract.exchange, ib_contract.exchange),
type=ORDERTYPE_IB2VT[ib_order.orderType],
orderid=orderid,
direction=DIRECTION_IB2VT[ib_order.action],
volume=ib_order.totalQuantity,
gateway_name=self.gateway_name,
)
if order.type == OrderType.LIMIT:
order.price = ib_order.lmtPrice
elif order.type == OrderType.STOP:
order.price = ib_order.auxPrice
self.orders[orderid] = order
self.gateway.on_order(copy(order))
def updateAccountValue( # pylint: disable=invalid-name
self, key: str, val: str, currency: str, accountName: str
):
"""
Callback of account update.
"""
super().updateAccountValue(key, val, currency, accountName)
if not currency or key not in ACCOUNTFIELD_IB2VT:
return
accountid = f"{accountName}.{currency}"
account = self.accounts.get(accountid, None)
if not account:
account = AccountData(accountid=accountid,
gateway_name=self.gateway_name)
self.accounts[accountid] = account
name = ACCOUNTFIELD_IB2VT[key]
setattr(account, name, float(val))
def updatePortfolio( # pylint: disable=invalid-name
self,
contract: Contract,
position: float,
marketPrice: float,
marketValue: float,
averageCost: float,
unrealizedPNL: float,
realizedPNL: float,
accountName: str,
):
"""
Callback of position update.
"""
super().updatePortfolio(
contract,
position,
marketPrice,
marketValue,
averageCost,
unrealizedPNL,
realizedPNL,
accountName,
)
if contract.exchange:
exchange = EXCHANGE_IB2VT.get(contract.exchange, None)
elif contract.primaryExchange:
exchange = EXCHANGE_IB2VT.get(contract.primaryExchange, None)
else:
exchange = Exchange.SMART # Use smart routing for default
if not exchange:
msg = f"存在不支持的交易所持仓{contract.conId} {contract.exchange} {contract.primaryExchange}"
self.gateway.write_log(msg)
return
ib_size = contract.multiplier
if not ib_size:
ib_size = 1
price = averageCost / ib_size
pos = PositionData(
symbol=contract.conId,
exchange=exchange,
direction=Direction.NET,
volume=position,
price=price,
pnl=unrealizedPNL,
gateway_name=self.gateway_name,
)
self.gateway.on_position(pos)
def updateAccountTime(self, timeStamp: str): # pylint: disable=invalid-name
"""
Callback of account update time.
"""
super().updateAccountTime(timeStamp)
for account in self.accounts.values():
self.gateway.on_account(copy(account))
def contractDetails(self, reqId: int, contractDetails: ContractDetails): # pylint: disable=invalid-name
"""
Callback of contract data update.
"""
super().contractDetails(reqId, contractDetails)
# Generate symbol from ib contract details
ib_contract = contractDetails.contract
if not ib_contract.multiplier:
ib_contract.multiplier = 1
symbol = generate_symbol(ib_contract)
# Generate contract
contract = ContractData(
symbol=symbol,
exchange=EXCHANGE_IB2VT[ib_contract.exchange],
name=contractDetails.longName,
product=PRODUCT_IB2VT[ib_contract.secType],
size=ib_contract.multiplier,
pricetick=contractDetails.minTick,
net_position=True,
history_data=True,
stop_supported=True,
gateway_name=self.gateway_name,
)
if contract.vt_symbol not in self.contracts:
self.gateway.on_contract(contract)
self.contracts[contract.vt_symbol] = contract
self.save_contract_data()
def execDetails(
self, reqId: int, contract: Contract, execution: Execution
): # pylint: disable=invalid-name
"""
Callback of trade data update.
"""
super().execDetails(reqId, contract, execution)
# today_date = datetime.now().strftime("%Y%m%d")
trade = TradeData(
symbol=contract.conId,
exchange=EXCHANGE_IB2VT.get(contract.exchange, contract.exchange),
orderid=str(execution.orderId),
tradeid=str(execution.execId),
direction=DIRECTION_IB2VT[execution.side],
price=execution.price,
volume=execution.shares,
time=datetime.strptime(execution.time, "%Y%m%d %H:%M:%S"),
gateway_name=self.gateway_name,
)
self.gateway.on_trade(trade)
def managedAccounts(self, accountsList: str):
"""
Callback of all sub accountid.
"""
super().managedAccounts(accountsList)
if not self.account:
for account_code in accountsList.split(","):
self.account = account_code
self.gateway.write_log(f"当前使用的交易账号为{self.account}")
self.client.reqAccountUpdates(True, self.account)
def historicalData(self, reqId: int, ib_bar: IbBarData):
"""
Callback of history data update.
"""
dt = datetime.strptime(ib_bar.date, "%Y%m%d %H:%M:%S")
bar = BarData(
symbol=self.history_req.symbol,
exchange=self.history_req.exchange,
datetime=dt,
interval=self.history_req.interval,
volume=ib_bar.volume,
open_price=ib_bar.open,
high_price=ib_bar.high,
low_price=ib_bar.low,
close_price=ib_bar.close,
gateway_name=self.gateway_name
)
self.history_buf.append(bar)
def historicalDataEnd(self, reqId: int, start: str, end: str):
"""
Callback of history data finished.
"""
self.history_condition.acquire()
self.history_condition.notify()
self.history_condition.release()
def connect(self, host: str, port: int, clientid: int, account: str):
"""
Connect to TWS.
"""
if self.status:
return
self.clientid = clientid
self.account = account
self.client.connect(host, port, clientid)
self.thread.start()
self.client.reqCurrentTime()
def close(self):
"""
Disconnect to TWS.
"""
if not self.status:
return
self.status = False
self.client.disconnect()
def subscribe(self, req: SubscribeRequest):
"""
Subscribe tick data update.
"""
if not self.status:
return
if req.exchange not in EXCHANGE_VT2IB:
self.gateway.write_log(f"不支持的交易所{req.exchange}")
return
# Extract ib contract detail
ib_contract = generate_ib_contract(req.symbol, req.exchange)
if not ib_contract:
self.gateway.write_log("代码解析失败，请检查格式是否正确")
return
# Get contract data from TWS.
self.reqid += 1
self.client.reqContractDetails(self.reqid, ib_contract)
# Subscribe tick data and create tick object buffer.
self.reqid += 1
self.client.reqMktData(self.reqid, ib_contract, "", False, False, [])
tick = TickData(
symbol=req.symbol,
exchange=req.exchange,
datetime=datetime.now(),
gateway_name=self.gateway_name,
)
self.ticks[self.reqid] = tick
self.tick_exchange[self.reqid] = req.exchange
def send_order(self, req: OrderRequest):
"""
Send a new order.
"""
if not self.status:
return ""
if req.exchange not in EXCHANGE_VT2IB:
self.gateway.write_log(f"不支持的交易所：{req.exchange}")
return ""
if req.type not in ORDERTYPE_VT2IB:
self.gateway.write_log(f"不支持的价格类型：{req.type}")
return ""
self.orderid += 1
ib_contract = generate_ib_contract(req.symbol, req.exchange)
if not ib_contract:
return ""
ib_order = Order()
ib_order.orderId = self.orderid
ib_order.clientId = self.clientid
ib_order.action = DIRECTION_VT2IB[req.direction]
ib_order.orderType = ORDERTYPE_VT2IB[req.type]
ib_order.totalQuantity = req.volume
ib_order.account = self.account
if req.type == OrderType.LIMIT:
ib_order.lmtPrice = req.price
elif req.type == OrderType.STOP:
ib_order.auxPrice = req.price
self.client.placeOrder(self.orderid, ib_contract, ib_order)
self.client.reqIds(1)
order = req.create_order_data(str(self.orderid), self.gateway_name)
self.gateway.on_order(order)
return order.vt_orderid
def cancel_order(self, req: CancelRequest):
"""
Cancel an existing order.
"""
if not self.status:
return
self.client.cancelOrder(int(req.orderid))
def query_history(self, req: HistoryRequest):
""""""
self.history_req = req
self.reqid += 1
ib_contract = generate_ib_contract(req.symbol, req.exchange)
if req.end:
end = req.end
end_str = end.strftime("%Y%m%d %H:%M:%S")
else:
end = datetime.now()
end_str = ""
delta = end - req.start
days = min(delta.days, 180) # IB only provides 6-month data
duration = f"{days} D"
bar_size = INTERVAL_VT2IB[req.interval]
if req.exchange == Exchange.IDEALPRO:
bar_type = "MIDPOINT"
else:
bar_type = "TRADES"
self.client.reqHistoricalData(
self.reqid,
ib_contract,
end_str,
duration,
bar_size,
bar_type,
1,
1,
False,
[]
)
self.history_condition.acquire() # Wait for async data return
self.history_condition.wait()
self.history_condition.release()
history = self.history_buf
self.history_buf = [] # Create new buffer list
self.history_req = None
return history
def load_contract_data(self):
""""""
f = shelve.open(self.data_filepath)
self.contracts = f.get("contracts", {})
f.close()
for contract in self.contracts.values():
self.gateway.on_contract(contract)
self.gateway.write_log("本地缓存合约信息加载成功")
def save_contract_data(self):
""""""
f = shelve.open(self.data_filepath)
f["contracts"] = self.contracts
f.close()
