def __init__(self, num_chairs):
self.shopLock = Lock()
self.numChairs = num_chairs
self.nfreeBarbers = 0
self.nwaitingCustomers = 0
self.waitingCustomersCond = Condition(self.shopLock)
self.waitingBarbersCond= Condition(self.shopLock)
class _EventRef(object):
""" Helper class which acts as a threading.Event, but which can be used
to pass a reference when signaling the event.
"""
def __init__(self):
self._cond = Condition(Lock())
self._flag = False
self._ref = None
def isSet(self):
return self._flag
def set(self, ref):
with self._cond:
assert self._ref is None
self._ref = ref
self._flag = True
self._cond.notifyAll()
def get(self, timeout=None):
with self._cond:
if not self._flag:
self._cond.wait(timeout)
if not self._flag:
raise TimeoutExceeded('Could not get the reference.')
return self._ref
def clear(self, ref):
with self._cond:
self._ref = None
self._flag = False
class DirtyBit:
def __init__(self):
self._mon = RLock()
self._rc = Condition(self._mon)
self._dirty = 0
def clear(self):
self._mon.acquire()
self._dirty = 0
#self._rc.notify() only interested in knowing when we are dirty
self._mon.release()
def set(self):
self._mon.acquire()
self._dirty = 1
self._rc.notify()
self._mon.release()
def value(self):
return self._dirty
def wait(self):
self._mon.acquire()
self._rc.wait()
self._mon.release()
class TestBackend(Backend):
def __init__(self, queue):
self.lock = Condition()
queue._set_backend(self)
# Statistics
self.notifies = 0
def start(self):
pass
def stop(self):
pass
def start_feedback(self):
pass
def queue_lock(self):
return self.lock
def queue_notify(self):
self.notifies += 1
self.lock.notify_all()
def sleep(self):
pass
class SimpleBlockingQueue(object):
def __init__(self):
self.items = deque()
self.condition = Condition()
self.interrupted = False
def put(self, item):
with self.condition:
self.items.pushleft(item)
def take(self, block=False):
with self.condition:
if block:
while not self.items:
self.condition.wait()
if self.interrupted:
self.interrupted = False
return None
elif not self.items:
return None
return self.items.popright()
def interrupt(self):
with self.condition:
self.interrupted = True
self.condition.notifyAll()
class BinaryServer(Thread):
"""
Socket server forwarding request to internal server
"""
def __init__(self, internal_server, hostname, port):
Thread.__init__(self)
self.socket_server = None
self.hostname = hostname
self.port = port
self.iserver = internal_server
self._cond = Condition()
def start(self):
with self._cond:
Thread.start(self)
self._cond.wait()
def run(self):
logger.warning("Listening on %s:%s", self.hostname, self.port)
socketserver.TCPServer.allow_reuse_address = True # get rid of address already in used warning
self.socket_server = ThreadingTCPServer((self.hostname, self.port), UAHandler)
# self.socket_server.daemon_threads = True # this will force a shutdown of all threads, maybe too hard
self.socket_server.internal_server = self.iserver # allow handler to acces server properties
with self._cond:
self._cond.notify_all()
self.socket_server.serve_forever()
def stop(self):
logger.warning("server shutdown request")
self.socket_server.shutdown()
def __init__(self):
'''Init.'''
gobject.GObject.__init__(self)
# set condition lock.
self.__condition = Condition()
# set datebase operation lock.
self.__db_operation_lock = Condition()
# songs
self.__hiddens = set()
self.__song_types_capability = {}
self.__songs = {}
self.__song_types = []
self.__songs_by_type = {}
# playlist
self.__playlists = {}
self.__playlist_types = []
# init constant
self.__is_loaded = False
self.__force_check = False
self.__save_song_type = ["local", "cue", "unknown"]
self.__dirty = False
# Init queued signal.
self.__reset_queued_signal()
class Consumer(Thread):
"""Класс потока – обработчика данных"""
def __init__(self):
"""Конструктор класса"""
super().__init__()
self.condition = Condition()
self.received_data = None
def run(self):
"""Код, исполняемый в потоке"""
running_sum = 0
count = 0
while True:
with self.condition:
# Ожидание доступности данных
self.condition.wait()
# Получение данных
data = self.received_data
# Обработка данных
running_sum += data
count += 1
print('Got data: {}\nTotal count: {}\nAverage: {}\n'.format(
data, count, running_sum / count))
def send(self, data):
"""Метод отправки данных в обработчик"""
self.received_data = data
class SendBuffer(object):
def __init__(self, max_size):
self.size = 0
self.frontbuffer = ''
self.full = Condition()
self.backbuffer = StringIO()
self.max_size = max_size
def __len__(self):
return len(self.frontbuffer) + self.size
def write(self, data):
dlen = len(data)
with self.full:
while self.size + dlen > self.max_size and dlen < self.max_size:
# if a single write is bigger than the buffer, swallow hard.
# No amount of waitng will make it fit.
self.full.wait()
self.backbuffer.write(data)
self.size += dlen
def to_sock(self, sock):
if not self.frontbuffer:
with self.full:
buf, self.backbuffer = self.backbuffer, StringIO()
self.size = 0
self.full.notify_all()
self.frontbuffer = buf.getvalue()
written = sock.send(self.frontbuffer)
self.frontbuffer = self.frontbuffer[written:]
class Heartbeater(object):
interval = 5
def __init__(self, api, jobstep_id, interval=None):
self.api = api
self.jobstep_id = jobstep_id
self.cv = Condition()
self.finished = Event()
if interval is not None:
self.interval = interval
def wait(self):
with self.cv:
self.finished.clear()
while not self.finished.is_set():
data = self.api.get_jobstep(self.jobstep_id)
if data['status']['id'] == 'finished':
self.finished.set()
break
self.cv.wait(self.interval)
def close(self):
with self.cv:
self.finished.set()
self.cv.notifyAll()
class Future(object):
def __init__(self, func, *args, **kwargs):
self.__done = False
self.__result = None
self.__cond = Condition()
self.__func = func
self.__args = args
self.__kwargs = kwargs
self.__except = None
def __call__(self):
with self.__cond:
try:
self.__result = self.__func(*self.__args, **self.__kwargs)
except:
self.__result = None
self.__except = sys.exc_info()
self.__done = True
self.__cond.notify()
def result(self):
with self.__cond:
while not self.__done:
self.__cond.wait()
if self.__except:
exc = self.__except
reraise(exc[0], exc[1], exc[2])
result = self.__result
return copy.deepcopy(result)
class ResourcePool(object):
def __init__(self, initial_value):
super(ResourcePool, self).__init__()
self.condition = Condition()
self.value = initial_value
def acquire(self, amount):
with self.condition:
while amount > self.value:
self.condition.wait()
self.value -= amount
self.__validate()
def release(self, amount):
with self.condition:
self.value += amount
self.__validate()
self.condition.notifyAll()
def __validate(self):
assert 0 <= self.value
def __str__(self):
return str(self.value)
def __repr__(self):
return "ResourcePool(%i)" % self.value
@contextmanager
def acquisitionOf(self, amount):
self.acquire(amount)
try:
yield
finally:
self.release(amount)
class IOManager(Thread):
def __init__(self,so):
Thread.__init__(self)
self.mutexIO = Condition()
self.listIO = []
self.handlerIO = so.handlerIO
def process(self):
for irq in self.listIO:
print("Ejecuto IOManager")
self.handlerIO.handle(irq)
self.removeIOFromListIO()
def run(self):
while(True):
with self.mutexIO:
self.mutexIO.wait()
self.process()
def removeIOFromListIO(self):
for reqIO in self.listIO:
self.listIO.remove(reqIO)
def add(self,iOReq):
self.listIO.append(iOReq)
def __init__(self, node_id, data):
"""
Constructor.
@type node_id: Integer
@param node_id: the unique id of this node; between 0 and N-1
@type data: List of Integer
@param data: a list containing this node's data
"""
self.node_id = node_id
self.data = data
# temporary buffer for needed for scatter
self.copy = data[:]
self.lock_copy = Lock()
self.nodes = None
self.lock_data = Lock()
# list of threads (in this case 16 fo each node)
self.thread_list = []
# list with tasks that need to be computed
self.thread_pool = []
self.mutex = Lock()
# condition used for put and get
self.condition = Condition(self.mutex)
# condition needed for checking if there are
# still tasks that need o be solved
self.all_tasks_done = Condition(self.mutex)
# number of unfinished tasks
self.unfinished_tasks = 0
# start the 16 threads
for i in range(16):
th = Worker(self, i)
self.thread_list.append(th)
th.start()
def __init__(self, client):
self.prefix = "auau:"
self.mc = client
self.random = Random()
self.random.seed(os.urandom(128))
self.random.jumpahead(os.getpid())
# thread exit flag
self.exit_flag = [False]
from threading import Thread
from threading import Condition
# asynchronous deflation thread
self.def_cv = Condition()
self.def_que = []
self.def_thread = Thread(
target=lambda: self.async_work(self.def_cv, self.def_que, self.exit_flag, lambda x: self.deflate(x))
)
self.def_thread.setDaemon(True)
self.def_thread.start()
# asynchronous deletion thread
self.del_cv = Condition()
self.del_que = []
self.del_thread = Thread(
target=lambda: self.async_work(self.del_cv, self.del_que, self.exit_flag, lambda x: self.mc.delete(x))
)
self.del_thread.setDaemon(True)
self.del_thread.start()
def request(self, request_type, request, sequential=False):
"""
"""
cond = Condition()
# dict is used as reference type to allow result to be returned from
# seperate thread
response_cont = {}
def on_success(response_type, response):
response_cont.update({
"success": True,
"type": response_type,
"response": response,
})
with cond:
cond.notify()
def on_error(exception):
response_cont.update({
"success": False,
"exception": exception,
})
with cond:
cond.notify()
with cond:
self.request_async(request_type, request,
on_success, on_error,
sequential)
cond.wait()
if not response_cont["success"]:
raise response_cont["exception"]
return response_cont["type"], response_cont["response"]
class Presponse(PktHandler):
def __init__(self):
self.ping_response_wait = Condition()
'''
To wait for a ping response, do the following:{
start = time.time()
with Presponse.ping_response_wait:
Presponse.ping_response_wait.wait(5) # Standard timeout in PING latency is 5.
end = time.time()
}
(end - start) is the ping latency.
'''
def handle(self, packet_map):
with self.ping_response_wait:
self.ping_response_wait.notify_all()
return None
def getproperties(self):
return {
'DisplayName':'Ping-Response Notifier',
'CodeName':'PRESPONSE_NOTIFY',
'PacketType':'PRESPONSE',
'Version':0.01
}
def __init__(self, threads_num, task_queue_max=None):
self._threads_num = threads_num
self._tasks = deque()
self._threads = []
self._task_num = 0
self._task_lock = Condition(Lock())
self._thead_lock = Condition(Lock())
def __init__(self, numchairs):
self.barbers_ready = False
self.numchairs = numchairs
self.open_seats = numchairs
self.shop_lock = Lock()
self.barber_condition = Condition(self.shop_lock)
self.customer_condition = Condition(self.shop_lock)
def __init__(self):
self.nwasherWaiting = 0 # number of sims waiting for washing hands
self.nwasherUsing = 0 # number of sims washing hands right now
self.toiletBusy = 0 # is toilet busy right now ?
self.bathroomLock = Lock();
self.toiletLine = Condition(self.bathroomLock)
self.washerLine = Condition(self.bathroomLock)
class Broadcaster(object):
def __init__(self):
self._condition = Condition()
self._last_id = 0
self._queue = []
@acquire_then_notify
def send(self, item):
print "Sending message", item
self._last_id += 1
self._queue.append((self._last_id, item))
def _find_items(self, since_id=None):
if not since_id:
found = self._queue
else:
found = [(id, item) for (id,item) in self._queue if id > since_id]
return found
@acquire
def recv(self, since_id=None, timeout=10):
end_time = time() + timeout
while time() < end_time:
found = self._find_items(since_id)
if found:
break
print "Waiting"
self._condition.wait(timeout)
print "found %d" % len(found)
return found
class BackUpThread (threading.Thread):
def __init__(self):
threading.Thread.__init__ ( self )
self.srcfile = "D:\study\operating system prac\CS4410\mp3\mp3\src\mail.txt"
self.destfile = "D:\study\operating system prac\CS4410\mp3\mp3\src\\backup.txt"
self.mutex = Lock()
self.cv = Condition(self.mutex)
self.msgCount = 0
def run(self):
with self.mutex:
while True:
# TODO: BUG here.
while self.msgCount != 32:
self.cv.wait()
print "Backing up the mail file."
# TODO: copy only the new part.
# desthandle = open(self.destfile, "r")
# desthandle.seek(0, 2)
# offset = desthandle.tell()
shutil.copyfile(self.srcfile, self.destfile)
self.msgCount = 0
def newMsg(self):
with self.mutex:
self.msgCount += 1
if self.msgCount == 32:
self.cv.notifyAll()
class Worker(Thread):
def __init__(self):
Thread.__init__(self)
self.lock = Lock()
self.cond = Condition(self.lock)
self.stopped = False
self.queue = []
def run(self):
while True:
job = None
with self.cond:
if self.stopped:
return
if not self.queue:
self.cond.wait()
else:
job, params = self.queue.pop(0)
if not job:
continue
self.execute(job, params)
def execute(self, job, params):
try:
func, args = job(*params)
# The async function may decide to NOT update
# the UI:
if func:
gobject.idle_add(func, *args)
except Exception, e:
print "Warning:", e
def __init__(self, algorithm, interpreter, change_listener, debug=False,
progress_listener=None):
if not isinstance(algorithm, Algorithm):
raise TypeError('%r is not a matching algorithm object' %
algorithm)
self.current_location = None
self.previous_location = None
self._debug_enabled = debug
self._running = False
self._within_piece = False
self._startup_condition = Condition()
self._piece_control = Condition()
self._playing_piece = False
self._history = HistoryQueue()
self.intervals = None
self._incoming_notes = None
self.interpreter = interpreter
self.change_listener = change_listener
self.progress_listener = progress_listener
algorithm.assign_matcher(self)
self._algorithm = algorithm
self._thread = None
self._stopping = False
def __init__(self, number_threads): self.pool_lock = Lock() self.connection_available = Condition(self.pool_lock) self.request_available = Condition(self.pool_lock) self.connection_pool = [] self.number_connections = 0 self.max_connections = number_threads
def __init__(self):
# TODO
self.carsCrossingNorth = 0
self.carsCrossingSouth = 0
self.cvLock = Lock()
self.southQ = Condition(self.cvLock)
self.northQ = Condition(self.cvLock)
def __init__(self):
self.lock = Lock()
self.toWrite = Condition(self.lock)
self.toBackup = Condition(self.lock)
self.isWriting = False
self.isBackup = False
self.emailNum = 1
def __init__(self, num_photos=10, sources=[], pool_dir=cache_dir):
Thread.__init__(self)
self.num_photos = num_photos
self.sources = sources
self.dir = pool_dir
# Make sure cache dir exists
if not os.path.exists(self.dir):
os.mkdir(self.dir)
# Clean cache directory
self.clean_cache()
# Load cached photos
self.photos = os.listdir(self.dir)
# Delete queue
self.trash = []
# Condition when a new photo is added
self.added = Condition()
# Condition when a photo is removed
self.removed = Condition()
# Event for stopping the pool
self._stop = Event()
class Future:
def __init__(self, correlation_id, request_type=None):
self.correlation_id = correlation_id
self._result = None
self._condition = Condition()
self._request_type = request_type
def done(self):
return self._result is not None
def result(self, timeout=None):
with self._condition:
if self._result is None:
if not self._condition.wait(timeout):
message_type = validator_pb2.Message.MessageType.Name(
self._request_type) if self._request_type else None
raise FutureTimeoutError(
'Future timed out waiting for response to {}'.format(
message_type))
return self._result
def set_result(self, result):
with self._condition:
self._result = result
self._condition.notify()
def __init__(self, cache_dir, map_desc, db_schema, is_concurrency=True):
self.__map_desc = map_desc
self.__db_path = os.path.join(cache_dir, map_desc.map_id + ".mbtiles")
self.__conn = None
#configs
self.__db_schema = db_schema
self.__has_timestamp = True
self.__is_concurrency = is_concurrency
if is_concurrency:
self.__surrogate = None #the thread do All DB operations, due to sqlite3 requiring only the same thread.
self.__is_closed = False
#concurrency get/put
self.__sql_queue = []
self.__sql_queue_lock = Lock()
self.__sql_queue_cv = Condition(self.__sql_queue_lock)
self.__get_lock = Lock() #block the 'get' action
self.__get_respose = None #the pair (data, exception)
self.__get_respose_lock = Lock()
self.__get_respose_cv = Condition(self.__get_respose_lock)
class PoolBlockPut(PoolBase):
def __init__(self, needed_count):
super(PoolBlockPut, self).__init__()
self.condition = Condition() # 用于实现线程通信
self.needed_count = needed_count # 下一次读取所需数据数量
# 若池子中有足够数据，则阻塞当前线程，否则将数据放入池子
def put(self, data):
if not isinstance(data, np.ndarray):
raise TypeError("Input data must be ndarray!")
if data.ndim != 1:
raise ValueError("Input data must be 1D!")
self.condition.acquire()
if len(self.datas) >= self.needed_count:
self.condition.wait()
self.datas = np.concatenate((self.datas, data))
self.condition.release()
# 非循环读取池子中数据。若取后剩余数据小于所需数量，则释放被阻塞线程
def get(self, count):
if count < 0:
raise ValueError("Input count must >= 0!")
if not isinstance(count, int):
raise ValueError("Input count must be integer!")
if count > len(self.datas):
count = len(self.datas)
# logging.warning("IO is going to be close!")
self.condition.acquire()
re = self.datas[:count].copy()
self.datas = np.delete(self.datas, range(0, count))
self.needed_count = count
if len(self.datas
) < 4*self.needed_count: # 取后剩余数据不够下一次读取，释放被阻塞线程，让其继续生产数据
self.condition.notify_all()
self.condition.release()
return re
def release(self):
self.condition.acquire()
self.condition.notify_all()
self.condition.release()
class EventRPCClient(object):
def __init__(self,
event_engine,
service,
event_client=None,
event_server=None):
self.EVENT_CLIENT = event_client if event_client else "%s_CLIENT" % service.upper(
)
self.EVENT_SERVER = event_server if event_server else "%s_SERVER" % service.upper(
)
self.rid = 0
self._handlers = {}
self._handlers_lock = Lock()
self._event_engine = event_engine
self._event_engine.register(self.EVENT_SERVER, self._process_apiback)
self._pause_condition = Condition()
self._sync_ret = None
self._timeout = 0
self._timer_sleep = 1
self._sync_call_time_lock = Lock()
self._sync_call_time = datetime.now()
timer = Thread(target=self._run_timer)
timer.daemon = True
timer.start()
def _run_timer(self):
while True:
if not self._timeout == 0:
with self._sync_call_time_lock:
mtime = self._sync_call_time
delta = (datetime.now() - mtime).seconds
if delta >= self._timeout:
#print "timeout", self._timeout, delta
# 不可重入，保证self.rid就是超时的那个
with self._handlers_lock:
del self._handlers[self.rid]
log.debug("[RPCClient._runtimer] 处理超时, delete rid; %s" %
self.rid)
self._timeout = 0
self._notify_server_data()
time.sleep(self._timer_sleep)
def _process_apiback(self, event):
assert (event.route == self.EVENT_SERVER)
self._timeout = 0
rid = event.args['rid']
try:
with self._handlers_lock:
handler = self._handlers[rid]
except KeyError:
log.info('[RPCClient._process_apiback] 放弃超时任务的返回结果')
else:
try:
if handler:
# 异步
handler(event.args['ret'])
else:
# 同步
self._sync_ret = event.args['ret']
self._notify_server_data()
except Exception as e:
print e
log.debug("[RPCClient._process_apiback] 删除已经完成的任务 rid; %s" % rid)
with self._handlers_lock:
del self._handlers[rid]
def call(self, apiname, args, handler):
""" 给定参数args，异步调用RPCServer的apiname服务,
返回结果做为回调函数handler的参数。
Args:
apiname (str): 服务API名称。
args (dict): 给服务API的参数。
handler (function): 回调函数。
"""
if not isinstance(args, dict):
raise InvalidRPCClientArguments(argtype=type(args))
assert (not handler == None)
self.rid += 1
args['apiname'] = apiname
args['rid'] = self.rid
self._event_engine.emit(Event(self.EVENT_CLIENT, args))
with self._handlers_lock:
self._handlers[self.rid] = handler
def sync_call(self, apiname, args, timeout=10):
""" 给定参数args，同步调用RPCServer的apiname服务,
返回该服务的处理结果。如果超时，返回None。
Args:
apiname (str): 服务API名称。
args (dict): 给服务API的参数。
handler (function): 回调函数。
"""
log.debug('sync_call: %s' % apiname)
if not isinstance(args, dict):
self._timeout = 0
self._sync_ret = None
raise InvalidRPCClientArguments(argtype=type(args))
self.rid += 1
args['apiname'] = apiname
args['rid'] = self.rid
with self._sync_call_time_lock:
self._sync_call_time = datetime.now()
self._timeout = timeout
with self._handlers_lock:
self._handlers[self.rid] = None
self._event_engine.emit(Event(self.EVENT_CLIENT, args))
self._waiting_server_data()
ret = self._sync_ret
#self._sync_ret = None
return ret
def _waiting_server_data(self):
with self._pause_condition:
self._pause_condition.wait()
def _notify_server_data(self):
with self._pause_condition:
self._pause_condition.notify()
cmd = config['source'][len('pipe:'):]
(child_out,source) = os.popen2( cmd, 'b' )
else:
# File source
source = open(config['source'],"rb")
dscfg.set_video_ratelimit(tdef.get_bitrate())
restartstatefilename = config['name']+'.restart'
dscfg.set_video_source(source, restartstatefilename=restartstatefilename)
dscfg.set_max_uploads(config['nuploads'])
d = s.start_download(tdef,dscfg)
d.set_state_callback(state_callback,getpeerlist=False)
# condition variable would be prettier, but that don't listen to
# KeyboardInterrupt
#time.sleep(sys.maxint/2048)
#try:
# while True:
# x = sys.stdin.read()
#except:
# print_exc()
cond = Condition()
cond.acquire()
cond.wait()
s.shutdown()
time.sleep(3)
shutil.rmtree(statedir)
class ForkingBase(object):
'''Base class for classes which fork children and wait for them to exit.
Sub-classes must provide the following data attributes and methods:
log - an object of type getmailcore.logging.Logger()
'''
def _child_handler(self, sig, stackframe):
def notify():
self.__child_exited.acquire()
self.__child_exited.notify_all()
self.__child_exited.release()
self.log.trace('handler called for signal %s' % sig)
try:
pid, r = os.waitpid(self.child.childpid, 0)
except OSError as o:
# No children on SIGCHLD. Can't happen?
self.log.warning('handler called, but no children (%s)' % o)
notify()
return
signal.signal(signal.SIGCHLD, self.__orig_handler)
self.__child_pid = pid
self.__child_status = r
self.log.trace('handler reaped child %s with status %s' % (pid, r))
notify()
def _prepare_child(self):
self.log.trace('')
self.__child_exited = Condition()
self.__child_pid = 0
self.__child_status = None
self.__orig_handler = signal.signal(signal.SIGCHLD,
self._child_handler)
def _wait_for_child(self, childpid):
self.__child_exited.acquire()
if self.__child_exited.wait(60) == False: # Py2, <Py3.2: always None
raise getmailOperationError('waiting child pid %d timed out' %
childpid)
self.__child_exited.release()
if self.__child_pid != childpid:
#self.log.error('got child pid %d, not %d' % (pid, childpid))
raise getmailOperationError('got child pid %d, not %d' %
(self.__child_pid, childpid))
if os.WIFSTOPPED(self.__child_status):
raise getmailOperationError(
'child pid %d stopped by signal %d' %
(self.__child_pid, os.WSTOPSIG(self.__child_status)))
if os.WIFSIGNALED(self.__child_status):
raise getmailOperationError(
'child pid %d killed by signal %d' %
(self.__child_pid, os.WTERMSIG(self.__child_status)))
if not os.WIFEXITED(self.__child_status):
raise getmailOperationError('child pid %d failed to exit' %
self.__child_pid)
exitcode = os.WEXITSTATUS(self.__child_status)
return exitcode
def _pipemail(self, msg, delivered_to, received, unixfrom, stdout, stderr):
# Write out message
msgfile = TemporaryFile23()
msgfile.write(
msg.flatten(delivered_to, received, include_from=unixfrom))
msgfile.flush()
os.fsync(msgfile.fileno())
# Rewind
msgfile.seek(0)
# Set stdin to read from this file
os.dup2(msgfile.fileno(), 0)
# Set stdout and stderr to write to files
os.dup2(stdout.fileno(), 1)
os.dup2(stderr.fileno(), 2)
def child_replace_me(self,
msg,
delivered_to,
received,
unixfrom,
stdout,
stderr,
args,
nolog=False):
self._pipemail(msg, delivered_to, received, unixfrom, stdout, stderr)
nolog or self.log.debug('about to execl() with args %s\n' % str(args))
os.execl(*args)
def forkchild(self, childfun, with_out=True):
self.child = child = Namespace()
child.stdout = TemporaryFile23()
child.stderr = TemporaryFile23()
child.childpid = os.fork()
if child.childpid != 0: # here (in the parent)
self._prepare_child()
self.log.debug('spawned child %d\n' % child.childpid)
child.exitcode = self._wait_for_child(child.childpid)
child.stderr.seek(0)
child.err = child.stderr.read().strip().decode()
child.stdout.seek(0)
if with_out:
child.out = child.stdout.read().strip()
return child
else: #== 0 in the child
# calls child_replace_me to execl external command
childfun(child.stdout, child.stderr)
def get_msginfo(self, msg):
msginfo = {}
msginfo['sender'] = msg.sender.strip()
if msg.recipient != None:
rcpnt = msg.recipient.strip()
msginfo['recipient'] = rcpnt
msginfo['domain'] = rcpnt.lower().split('@')[-1]
msginfo['local'] = '@'.join(rcpnt.split('@')[:-1])
self.log.debug('msginfo "%s"\n' % msginfo)
return msginfo
class TaggerWorker(QObject):
"""This class is required to prevent locking up the main Qt thread."""
on_error = pyqtSignal(str)
on_review_ready = pyqtSignal(tagger.UpdatesResult)
on_updates_sent = pyqtSignal(int)
on_stopped = pyqtSignal()
on_mint_mfa = pyqtSignal()
on_progress = pyqtSignal(str, int, int)
stopping = False
mfa_condition = Condition()
@pyqtSlot()
def stop(self):
self.stopping = True
@pyqtSlot(str)
def mfa_code(self, code):
logger.info('Got code')
logger.info(code)
self.mfa_code = code
logger.info('Waking thread')
self.mfa_condition.notify()
@pyqtSlot(object)
def create_updates(self, args, parent):
try:
self.do_create_updates(args, parent)
except Exception as e:
msg = 'Internal error while creating updates: {}'.format(e)
self.on_error.emit(msg)
logger.exception(msg)
@pyqtSlot(list, object)
def send_updates(self, updates, args):
try:
self.do_send_updates(updates, args)
except Exception as e:
msg = 'Internal error while sending updates: {}'.format(e)
self.on_error.emit(msg)
logger.exception(msg)
def do_create_updates(self, args, parent):
def on_mint_mfa(prompt):
logger.info('Asking for Mint MFA')
self.on_mint_mfa.emit()
logger.info('Blocking')
self.mfa_condition.wait()
logger.info('got code!')
logger.info(self.mfa_code)
return self.mfa_code
# Factory that handles indeterminite, determinite, and counter style.
def progress_factory(msg, max=0):
return QtProgress(msg, max, self.on_progress.emit)
self.mint_client = MintClient(
email=args.mint_email,
password=args.mint_password,
session_path=args.session_path,
headless=args.headless,
mfa_method=args.mint_mfa_method,
wait_for_sync=args.mint_wait_for_sync,
mfa_input_callback=on_mint_mfa,
progress_factory=progress_factory)
results = tagger.create_updates(
args, self.mint_client,
on_critical=self.on_error.emit,
indeterminate_progress_factory=progress_factory,
determinate_progress_factory=progress_factory,
counter_progress_factory=progress_factory)
if results.success and not self.stopping:
self.on_review_ready.emit(results)
def do_send_updates(self, updates, args):
num_updates = self.mint_client.send_updates(
updates,
progress=QtProgress(
'Sending updates to Mint',
len(updates),
self.on_progress.emit),
ignore_category=args.no_tag_categories)
self.on_updates_sent.emit(num_updates)
self.mint_client.close()
print("{}:在".format(self.name))
self.cond.notify()
print("{}:我们来对古诗吧".format(self.name))
class TianMao(threading.Thread):
def __init__(self, cond):
super().__init__(name="tianmao")
self.cond = cond
def run(self) -> None:
with self.cond:
print("{}:小爱同学".format(self.name))
self.cond.notify()
self.cond.wait()
print("{}:好啊".format(self.name))
if __name__ == '__main__':
cond = Condition()
xiaoAi = XiaoAi(cond=cond)
tianMao = TianMao(cond=cond)
# 1. 启动顺序很重要
# 2. 在调用with cond之后才能调用wait或者notify方法，with自动实现了acquire和release，否则需要自己输入这两者
# 3. condition有两层锁，一把底层锁会在线程调用了wait方法的时候释放，上面的锁会在每次调用wait的时候分配一把并发
# 到cond的等待队列中，等待notify方法的唤醒。
xiaoAi.start() # 必须先启动xiaoai，不然notify唤醒不了xiaoai。
tianMao.start()
def __init__(self, needed_count):
super(PoolBlockPut, self).__init__()
self.condition = Condition() # 用于实现线程通信
self.needed_count = needed_count # 下一次读取所需数据数量
class BlockPublisher(BlockPublisherInterface):
"""DevMode consensus uses genesis utility to configure Min/MaxWaitTime
to determine when to claim a block.
Default MinWaitTime to zero and MaxWaitTime is 0 or unset,
ValidBlockPublishers default to None or an empty list.
DevMode Consensus (BlockPublisher) will read these settings
from the StateView when Constructed.
"""
def __init__(self, block_cache, state_view_factory, batch_publisher,
data_dir, config_dir, validator_id):
super().__init__(block_cache, state_view_factory, batch_publisher,
data_dir, config_dir, validator_id)
self._block_cache = block_cache
self._state_view_factory = state_view_factory
self._start_time = 0
self._wait_time = 0
# Set these to default values right now, when we asked to initialize
# a block, we will go ahead and check real configuration
self._min_wait_time = 0.01
self._max_wait_time = 0.06
self._valid_block_publishers = None # list of validator which can participate into consensus
self._consensus = None
self._condition = Condition()
self._is_finalize_complete = None
def set_consensus_name(self, name):
self._consensus = bytes(name, 'utf-8')
LOGGER.debug("PROXY:set_consensus_name=%s->%s", name, self._consensus)
def set_publisher(self, publisher):
self._publisher = publisher
LOGGER.debug("PROXY:set_publisher=%s", publisher)
def initialize_block(self, block_header):
"""Do initialization necessary for the consensus to claim a block,
this may include initiating voting activates, starting proof of work
hash generation, or create a PoET wait timer.
Args:
block_header (BlockHeader): the BlockHeader to initialize.
Returns:
True
"""
if not self._consensus:
LOGGER.debug(
"initialize_block: external consensus not regitered\n")
return False
# Using the current chain head, we need to create a state view so we
# can get our config values.
state_view = BlockWrapper.state_view_for_block(
self._block_cache.block_store.chain_head, self._state_view_factory)
settings_view = SettingsView(state_view)
self._min_wait_time = settings_view.get_setting(
"bgx.consensus.min_wait_time", self._min_wait_time, float)
self._max_wait_time = settings_view.get_setting(
"bgx.consensus.max_wait_time", self._max_wait_time, float)
self._valid_block_publishers = settings_view.get_setting(
"bgx.consensus.valid_block_publishers",
self._valid_block_publishers, list)
block_header.consensus = self._consensus # b"Devmode"
self._start_time = time.time()
self._wait_time = random.uniform(self._min_wait_time,
self._max_wait_time)
LOGGER.debug(
"PROXY:initialize_block min_wait_time=%s max_wait_time=%s",
self._min_wait_time, self._max_wait_time)
return True
def check_publish_block(self, block_header):
"""
Check if a candidate block is ready to be claimed.
For many peers we should control block's content .
If this peer is not owner of batch we must wait until all batches which were putted into block for peer owner of batch
will be putted into block for this peer too.
block_header (BlockHeader): the block_header to be checked if it
should be claimed
Returns:
Boolean: True if the candidate block_header should be claimed.
"""
if self._valid_block_publishers and block_header.signer_public_key not in self._valid_block_publishers:
return False
elif self._min_wait_time == 0:
return True
elif self._min_wait_time > 0 and self._max_wait_time <= 0:
if self._start_time + self._min_wait_time <= time.time():
return True
elif self._min_wait_time > 0 and self._max_wait_time > self._min_wait_time:
if self._start_time + self._wait_time <= time.time():
return True
else:
return False
def finalize_block_complete(self, consensus):
with self._condition:
self._is_finalize_complete = consensus
self._condition.notify()
def _finalize_complete(self):
return self._is_finalize_complete is not None
def finalize_block(self, block_header):
"""Finalize a block to be claimed. Provide any signatures and
data updates that need to be applied to the block before it is
signed and broadcast to the network.
Args:
block_header (BlockHeader): The candidate block that needs to be
finalized.
Returns:
True
"""
LOGGER.debug(
"PROXY:finalize_block inform external engine header=%s is_complete=%s",
block_header.block_num, self._is_finalize_complete)
self._publisher.on_finalize_block(block_header)
self._is_finalize_complete = None
"""
after that consensus engine should be informed that block could be finalized and engine can say finalize for this candidate
FIXME - for DAG we can say for all ready candidate that his block's could be finalized and only after that wait engine's reply
"""
LOGGER.debug(
"PROXY:finalize_block wait proxy reply via finalize_block_complete...\n"
)
with self._condition:
return self._condition.wait_for(self._finalize_complete)
return True
class ForkResolver(ForkResolverInterface):
"""Provides the fork resolution interface for the BlockValidator to use
when deciding between 2 forks.
"""
# pylint: disable=useless-super-delegation
def __init__(self, block_cache, state_view_factory, data_dir, config_dir,
validator_id):
super().__init__(block_cache, state_view_factory, data_dir, config_dir,
validator_id)
self._consensus = bytes(_CONSENSUS_NAME_, 'utf-8')
self._condition = Condition()
@staticmethod
def hash_signer_public_key(signer_public_key, header_signature):
m = hashlib.sha256()
m.update(signer_public_key.encode())
m.update(header_signature.encode())
digest = m.hexdigest()
number = int(digest, 16)
return number
def _compare_forks_complete(self):
return self._is_compare_forks is not None
def compare_forks_complete(self, result):
LOGGER.debug("PROXY:compare_forks_complete result=%s", result)
with self._condition:
self._is_compare_forks = result
self._condition.notify()
def compare_forks(self, cur_fork_head, new_fork_head):
"""The longest chain is selected. If they are equal, then the hash
value of the previous block id and publisher signature is computed.
The lowest result value is the winning block.
Args:
cur_fork_head: The current head of the block chain.
new_fork_head: The head of the fork that is being evaluated.
Returns:
bool: True if choosing the new chain head, False if choosing
the current chain head.
"""
LOGGER.debug(
"PROXY:compare_forks cur~new=%s~%s new fork consensus=%s~%s",
cur_fork_head.identifier[:8], new_fork_head.identifier[:8],
new_fork_head.consensus, self._consensus)
# If the new fork head is not DevMode consensus, bail out. This should
# never happen, but we need to protect against it.
if new_fork_head.consensus != self._consensus and new_fork_head.consensus != b"Genesis":
raise TypeError('New fork head {} is not a {} block'.format(
new_fork_head.identifier[:8], _CONSENSUS_NAME_))
self._is_compare_forks = None
_consensus_notifier.notify_block_valid(new_fork_head.identifier)
LOGGER.debug(
"PROXY:compare_forks waiting consensus reply for new head=%s\n",
new_fork_head.identifier[:8])
with self._condition:
if self._condition.wait_for(self._compare_forks_complete):
if self._is_compare_forks:
# send message to external consensus
_consensus_notifier.notify_block_commit(
new_fork_head.identifier)
return self._is_compare_forks
# If the current fork head is not DevMode consensus, check the new fork
# head to see if its immediate predecessor is the current fork head. If
# so that means that consensus mode is changing. If not, we are again
# in a situation that should never happen, but we need to guard
# against.
"""
class PoolBlockGet(PoolBase):
def __init__(self, needed_count):
super(PoolBlockGet, self).__init__()
self.condition = Condition() # 用于实现线程通信
self.needed_count = needed_count # 读取所需数据数量
# 将数据放入池子。若池子中有足够数据，则释放被阻塞线程
def put(self, data):
if not isinstance(data, np.ndarray):
raise TypeError("Input data must be ndarray!")
if data.ndim != 1:
raise ValueError("Input data must be 1D!")
self.condition.acquire()
self.datas = np.concatenate((self.datas, data))
if len(self.datas) >= self.needed_count:
self.condition.notify_all()
self.condition.release()
# 若要求数量大于库存，则阻塞当前线程。否则非循环读取池子中数据
def get(self, count):
if count < 0:
raise ValueError("Input count must >= 0!")
if not isinstance(count, int):
raise ValueError("Input count must be integer!")
self.condition.acquire()
if count > len(self.datas): # 所需帧数大于总量，则让调用该方法线程进入等待队列，直到存在足够多数据
self.needed_count = count
self.condition.wait()
re = self.datas[:count].copy()
self.datas = np.delete(self.datas, range(0, count))
self.condition.release()
return re
def release(self):
self.condition.acquire()
self.condition.notify_all()
self.condition.release()
class Rusher(object):
"""
A class for orchestrating the rush of some yielding callable.
This class creates self.thread_count worker threads which will be used to
rush a resource when they are ready.
During Rusher.rush() the workers are created and run until they yield.
Once all the workers have yielded they are then woken up at once so they
can rush.
"""
def __init__(self, worker, thread_count=2):
self.worker = worker
self.thread_count = thread_count
# the orchestrator waits for notification of this from the workers
self.ready_progress = Condition()
self._total_ready = 0
# this triggers the rush in the worker threads
self.trigger = Event()
self._return_list = []
self._threads = []
def _create_threads(self):
"""
Create and start the worker threads so they can get ready to rush.
"""
self._wait_for_threads()
self._threads = []
self._return_list = []
for i in range(self.thread_count):
thread = Thread(target=self._work, args=(i, ))
thread.start()
self._threads.append(thread)
def _wait_for_threads(self, end_time=None):
"""
Wait for all worker threads to finish.
Unfinished threads are not killed.
"""
for thread in self._threads:
if end_time is not None:
max_wait = end_time - time()
if max_wait < 0:
return
else:
max_wait = None
thread.join(max_wait)
# this is very likely to happen if the timeout tripped
if thread.is_alive():
return
# all workers returned before end_time
return
def rush(self, max_seconds=None):
"""
Create worker threads and trigger their rush once they are ready.
max_seconds is either None for no limiting, or a float.
Returns (duration, results).
"""
with self.ready_progress:
self._total_ready = 0
self._create_threads()
self.ready_progress.wait()
start = time()
wait_until = time() + max_seconds if max_seconds else None
self.trigger.set()
self._wait_for_threads(wait_until)
self.trigger.clear()
results = tuple(self._return_list)
end = time()
duration = end - start
return duration, results
def _work(self, index):
"""
Interface with the orchestration in the rush method, to run the worker.
self.worker is iterated once so it can preform any needed prepairation,
then again when this worker thread is awakened in self.rush() when
the final worker notifies it that it has completed.
Each worker is passed a unique index 0 ≤ index < self.thread_count.
The result of the second yield to self._return_list.
"""
worker_run = iter(self.worker(index, self.thread_count))
next(worker_run)
with self.ready_progress:
self._total_ready += 1
if self._total_ready == self.thread_count:
self.ready_progress.notify_all()
# Wait for the trigger to be fired
self.trigger.wait()
try:
result = next(worker_run)
except StopIteration:
result = None
self._return_list.append(result)
def rush_and_report(self, max_seconds=None, output=sys.stdout):
"""
Perform a rush and wite a summary of results to an output.
This requires the results of self.work be hashable.
Returns (duration, results) from the rush method.
"""
duration, results = self.rush(max_seconds)
# This avoids the use of collections.Counter to be 2.6+ compatible
counts = defaultdict(int)
for result in results:
counts[result] += 1
output.write("{} threads completed in {}, results:\n".format(
len(results),
str(timedelta(seconds=duration)).lstrip('0:'),
))
for result, count in counts.items():
output.write("\t{}: {}\n".format(result, count))
return (duration, results)
def __init__(self, block_cache, state_view_factory, data_dir, config_dir,
validator_id):
super().__init__(block_cache, state_view_factory, data_dir, config_dir,
validator_id)
self._consensus = bytes(_CONSENSUS_NAME_, 'utf-8')
self._condition = Condition()
def __init__(self):
# Number of tasks that are being executed
self._num_work_executing = 0
self._work_condition = Condition()
def __init__(self, queue, lock, condition):
super().__init__()
self._queue = queue
self._lock = lock
self._condition = condition
def run(self):
while True:
with self._condition:
if not self._queue.empty():
self._lock.acquire()
item = self._queue.get()
self._lock.release()
self._condition.notify()
self._condition.wait()
print('Consume ', item)
else:
print("Queue is empty， please wait!")
time.sleep(1.0e-4)
if __name__ == '__main__':
q = Queue(maxsize=20)
lock = Lock()
# 线程条件变量，实现线程调度
condition = Condition()
producer = Producer(q, lock, condition)
consumer = Consumer(q, lock, condition)
producer.start()
consumer.start()
class Job(object):
__slots__ = '__func', '__args', '__kwargs', '__completed', '__result', '__exception', '__lock', '__cond', '__creation_stack', '__id'
def __init__(self, func, *args, **kwargs):
if not callable(func):
raise ValueError(func)
self.__id = str(uuid.uuid4())
self.__lock = Lock()
self.__cond = Condition(self.__lock)
self.__func, self.__args, self.__kwargs = func, args, kwargs
self.__creation_stack = tuple(extract_stack())
self.__completed, self.__result, self.__exception = False, None, None
def abbrev_info_unlocked(self):
return 'Job %s: %s(*%s, **%s)' % (
self.__id, self.__func, repr(self.__args), repr(self.__kwargs))
def full_info_unlocked(self):
out = [self.abbrev_info_unlocked(), '\n']
out.extend(format_list(self.__creation_stack))
return ''.join(out)
def __str__(self):
with self.__lock:
return self.full_info_unlocked()
def __repr__(self):
with self.__lock:
return self.abbrev_info_unlocked()
@property
def stack(self):
with self.__lock:
return self.__creation_stack
def complete(self, result):
with self.__lock:
self.__completed = True
self.__result = result
self.__cond.notify_all()
LOGGER.debug('Job %s is complete' % self.abbrev_info_unlocked())
def complete_exception(self, exception):
with self.__lock:
self.__completed = True
self.__exception = exception
self.__cond.notify_all()
LOGGER.debug('Job %s is complete' % self.abbrev_info_unlocked())
def __call__(self):
func, args, kwargs = None, None, None
with self.__lock:
LOGGER.debug('Job %s is being executed' %
self.abbrev_info_unlocked())
func, args, kwargs = self.__func, self.__args, self.__kwargs
return func(*args, **kwargs)
@property
def result(self):
with self.__lock:
if not self.__completed:
raise RuntimeError('Job %s is not complete' % self)
elif self.__exception is not None:
raise self.__exception
else:
return self.__result
def wait(self, timeout=None):
with self.__lock:
if not self.__completed:
self.__cond.wait(timeout)
if not self.__completed:
raise RuntimeError('Job %s is not complete' % self)
elif self.__exception is not None:
raise self.__exception
else:
return self.__result
class SerialScheduler(Scheduler):
"""Serial scheduler which returns transactions in the natural order.
This scheduler will schedule one transaction at a time (only one may be
unapplied), in the exact order provided as batches were added to the
scheduler.
This scheduler is intended to be used for comparison to more complex
schedulers - for tests related to performance, correctness, etc.
"""
def __init__(self, squash_handler, first_state_hash, always_persist):
self._txn_queue = queue.Queue()
self._scheduled_transactions = []
self._batch_statuses = {}
self._txn_to_batch = {}
self._in_progress_transaction = None
self._final = False
self._complete = False
self._cancelled = False
self._previous_context_id = None
self._previous_valid_batch_c_id = None
self._squash = squash_handler
self._condition = Condition()
# contains all txn.signatures where txn is
# last in it's associated batch
self._last_in_batch = []
self._previous_state_hash = first_state_hash
# The state hashes here are the ones added in add_batch, and
# are the state hashes that correspond with block boundaries.
self._required_state_hashes = {}
self._already_calculated = False
self._always_persist = always_persist
def __del__(self):
self.cancel()
def __iter__(self):
return SchedulerIterator(self, self._condition)
def set_transaction_execution_result(self, txn_signature, is_valid,
context_id):
with self._condition:
if (self._in_progress_transaction is None
or self._in_progress_transaction != txn_signature):
raise ValueError(
"transaction not in progress: {}".format(txn_signature))
self._in_progress_transaction = None
if txn_signature not in self._txn_to_batch:
raise ValueError(
"transaction not in any batches: {}".format(txn_signature))
batch_signature = self._txn_to_batch[txn_signature]
if is_valid:
self._previous_context_id = context_id
else:
# txn is invalid, preemptively fail the batch
self._batch_statuses[batch_signature] = \
BatchExecutionResult(is_valid=False, state_hash=None)
if txn_signature in self._last_in_batch:
if batch_signature not in self._batch_statuses:
# because of the else clause above, txn is valid here
self._previous_valid_batch_c_id = self._previous_context_id
state_hash = self._calculate_state_root_if_required(
batch_id=batch_signature)
self._batch_statuses[batch_signature] = \
BatchExecutionResult(is_valid=True,
state_hash=state_hash)
else:
self._previous_context_id = self._previous_valid_batch_c_id
is_last_batch = \
len(self._batch_statuses) == len(self._last_in_batch)
if self._final and is_last_batch:
self._complete = True
self._condition.notify_all()
def add_batch(self, batch, state_hash=None):
with self._condition:
if self._final:
raise SchedulerError("Scheduler is finalized. Cannot take"
" new batches")
batch_signature = batch.header_signature
if state_hash is not None:
self._required_state_hashes[batch_signature] = state_hash
batch_length = len(batch.transactions)
for idx, txn in enumerate(batch.transactions):
if idx == batch_length - 1:
self._last_in_batch.append(txn.header_signature)
self._txn_to_batch[txn.header_signature] = batch_signature
self._txn_queue.put(txn)
self._condition.notify_all()
def get_batch_execution_result(self, batch_signature):
with self._condition:
return self._batch_statuses.get(batch_signature)
def count(self):
with self._condition:
return len(self._scheduled_transactions)
def get_transaction(self, index):
with self._condition:
return self._scheduled_transactions[index]
def next_transaction(self):
with self._condition:
if self._in_progress_transaction is not None:
return None
try:
txn = self._txn_queue.get(block=False)
except queue.Empty:
return None
self._in_progress_transaction = txn.header_signature
base_contexts = [] if self._previous_context_id is None \
else [self._previous_context_id]
txn_info = TxnInformation(txn=txn,
state_hash=self._previous_state_hash,
base_context_ids=base_contexts)
self._scheduled_transactions.append(txn_info)
return txn_info
def finalize(self):
with self._condition:
self._final = True
if len(self._batch_statuses) == len(self._last_in_batch):
self._complete = True
self._condition.notify_all()
def _compute_merkle_root(self, required_state_root):
"""Computes the merkle root of the state changes in the context
corresponding with _last_valid_batch_c_id as applied to
_previous_state_hash.
Args:
required_state_root (str): The merkle root that these txns
should equal.
Returns:
state_hash (str): The merkle root calculated from the previous
state hash and the state changes from the context_id
"""
state_hash = None
if self._previous_valid_batch_c_id is not None:
publishing_or_genesis = self._always_persist or \
required_state_root is None
state_hash = self._squash(
state_root=self._previous_state_hash,
context_ids=[self._previous_valid_batch_c_id],
persist=self._always_persist,
clean_up=publishing_or_genesis)
if self._always_persist is True:
return state_hash
if state_hash == required_state_root:
self._squash(state_root=self._previous_state_hash,
context_ids=[self._previous_valid_batch_c_id],
persist=True,
clean_up=True)
return state_hash
def _calculate_state_root_if_not_already_done(self):
if not self._already_calculated:
if not self._last_in_batch:
return
last_txn_signature = self._last_in_batch[-1]
batch_id = self._txn_to_batch[last_txn_signature]
required_state_hash = self._required_state_hashes.get(batch_id)
state_hash = self._compute_merkle_root(required_state_hash)
self._already_calculated = True
for t_id in self._last_in_batch[::-1]:
b_id = self._txn_to_batch[t_id]
if self._batch_statuses[b_id].is_valid:
self._batch_statuses[b_id].state_hash = state_hash
# found the last valid batch, so break out
break
def _calculate_state_root_if_required(self, batch_id):
required_state_hash = self._required_state_hashes.get(batch_id)
state_hash = None
if required_state_hash is not None:
state_hash = self._compute_merkle_root(required_state_hash)
self._already_calculated = True
return state_hash
def complete(self, block):
with self._condition:
if not self._final:
return False
if self._complete:
self._calculate_state_root_if_not_already_done()
return True
if block:
self._condition.wait_for(lambda: self._complete)
self._calculate_state_root_if_not_already_done()
return True
return False
def cancel(self):
with self._condition:
self._cancelled = True
self._condition.notify_all()
def is_cancelled(self):
with self._condition:
return self._cancelled
class DistributedPipelineRecord:
"""A class for storing a single mini-batch (consisting of multiple micro-batches) as input to
a single partition.
Args:
device: the local device that runs the partition.
rank: the rank of the partition in the pipeline.
chunks: number of micro-batches in a mini-batch
num_inputs: number of inputs to the partition.
consumers: list of consumers of outputs of the partition. Each consumer in the list is a tuple
(remote_partition_rref, input_idx, output_idx) where remote_partition_rref points to a
remote DistributedPipelineRecord for consumer partiton for this mini-batch. The output number
output_idx of this partition will be used as the input number input_idx of that partition.
"""
# Need to use Union due to https://github.com/python/mypy/issues/7866
DataConsumer = Union[DataConsumer[rpc.RRef]]
def __init__(
self,
device: torch.device,
rank: int,
chunks: int,
num_inputs: int,
num_outputs: Optional[int],
consumers: List[DataConsumer],
) -> None:
self.ready_cv = Condition()
# Each chunk consists of num_inputs tensors. self.tensors stores these individual tensors.
self.tensors: List[List[Optional[Tensor]]] = [[None] * num_inputs
for _ in range(chunks)]
# For each tensor in self.tensors, we record a cuda event in corrsponding tensorpipe stream in self.recv_events,
# and later the stream that processes that tensor will wait on that event.
self.recv_events = [[None] * num_inputs for _ in range(chunks)]
# Once all num_inputs tensors of a given chunk are recieved, they are assembled as a batch and stored in
# self.batches
self.batches: List[Optional[Batch]] = [None] * chunks
# For each tensor of each chunk, we fork a phony tensor, which will be used for injecting dependency between
# different chunks in backward path.
if num_outputs is None:
num_outputs = 1
self.forwarded_phony: List[List[List[rpc.RRef]]] = [
[[] for j in range(num_outputs)] for i in range(chunks)
]
self.consumers = consumers
self.rank = rank
self.device = device
def __getstate__(self) -> Dict:
# avoid pickling failure.
return {}
def feed(self, chunk: int, input_idx: int, input: Tensor) -> Tensor:
"""This function is called remotely to provide individual tensors of a given chunk."""
if input.device.type == "cpu":
input = input.to(self.device)
cuda_stream = torch.cuda.current_stream(
input.device) if input.device.type == "cuda" else None
with self.ready_cv:
assert self.tensors[chunk][input_idx] is None
input, phony = fork(input)
self.recv_events[chunk][input_idx] = (
cuda_stream.record_event()
if cuda_stream is not None else None # type: ignore
)
self.tensors[chunk][input_idx] = input
self.ready_cv.notify_all()
return phony
def wait_for(self, chunk: int) -> None:
"""Waits until all elements of given chunk is populated in self.tensors.
Then it constructs self.batches[chunk] if it is not constructed yet.
"""
with self.ready_cv:
while self.batches[chunk] is None and any(
b is None for b in self.tensors[chunk]):
self.ready_cv.wait()
if self.batches[chunk] is None:
tensors = cast(List[Tensor], self.tensors[chunk])
self.batches[chunk] = Batch(tuple(tensors), chunk)
def fence(self, chunk: int) -> None:
"""Prepares micro-batches for computation."""
# Ensure that batches[chunk-1] is executed after batches[chunk] in
# backpropagation by an explicit dependency.
# TODO: This dependency injection causes deadlock if this partition
# gets its input from model input. 1) Figure out why 2) If we need to live
# with this constraint, replace the condition 'self.rank > 0' below with
# a more accurate one.
if chunk != 0 and self.consumers and self.rank > 0:
batch = self.batches[chunk]
assert batch is not None
dependant_tensors = list(batch.tensors)
for remote_ph_list in self.forwarded_phony[chunk - 1]:
for remote_ph in remote_ph_list:
phony = remote_ph.to_here()
dependant_tensors[0] = join(dependant_tensors[0], phony)
self.batches[chunk] = Batch(tuple(dependant_tensors), chunk)
def sync_stream(self, chunk: int, stream: torch.cuda.Stream) -> None:
"""syncs the stream with cuda events associated with transmission of the chunck to the cuda device."""
for e in self.recv_events[chunk]:
if e is not None:
stream.wait_event(e)
def forward_results(self, chunk: int) -> None:
"""Forward outputs of processing the chunk in this parition for processing by next partition."""
for consumer in self.consumers:
v = self.get_batch(chunk).value[consumer.output_idx]
self.forwarded_phony[chunk][consumer.output_idx].append(
consumer.consumer.remote().feed(chunk,
consumer.consumer_input_idx,
v))
def get_batch(self, chunk: int) -> Batch:
batch = self.batches[chunk]
assert batch is not None
return batch
else:
sleep_time = None
if sleep_time: time.sleep(sleep_time)
# a new t_id produced
cv.acquire()
produced.append(t_id)
cv.notify()
cv.release()
# exit condition
if not sleep_time: break
with lock:
print(f'Thread {t_id}: sleep finished')
with lock:
print(f'Thread {t_id}: finished')
print('----- Condition Variable -----')
produced, threads = [0, 1, 2, 4], []
now = time.time()
cv = Condition()
for i in range(4):
t = Thread(target=calc_producer_consumer, args=(lock, cv))
t.start()
threads.append(t)
for t in threads:
t.join()
print(f'Total time: {round(time.time() - now, 2)} second(s)\n')
def __init__(self, name):
self.robot_name = name
# Mutual exclusion odometry
self.odometry_me = Condition()
# Create trajectory server
self.trajectory_server = SimpleActionServer(
'approach_server', ExecuteDroneApproachAction, self.goCallback,
False)
self.server_feedback = ExecuteDroneApproachFeedback()
self.server_result = ExecuteDroneApproachResult()
# Get client from hector_quadrotor_actions
self.move_client = SimpleActionClient("/{}/action/pose".format(name),
PoseAction)
self.move_client.wait_for_server()
# Subscribe to ground_truth to monitor the current pose of the robot
rospy.Subscriber("/{}/ground_truth/state".format(name), Odometry,
self.poseCallback)
# Subscribe to topic to receive the planned trajectory
rospy.Subscriber("/{}/move_group/display_planned_path".format(name),
DisplayTrajectory, self.planCallback)
#Auxiliary variables
self.trajectory = [] # Array with the trajectory to be executed
self.trajectory_received = False # Flag to signal trajectory received
self.odom_received = False # Flag to signal odom received
self.robot = RobotCommander(
robot_description="{}/robot_description".format(name),
ns="/{}".format(name))
self.display_trajectory_publisher = rospy.Publisher(
'/{}/move_group/display_planned_path'.format(name),
DisplayTrajectory,
queue_size=20)
# Variables for collision callback
self.validity_srv = rospy.ServiceProxy(
'/{}/check_state_validity'.format(name), GetStateValidity)
self.validity_srv.wait_for_service()
self.collision = False
# Set planning algorithm
# self.move = MoveGroupCommander(PLANNING_GROUP, robot_description="{}/robot_description".format(name), ns="/UAV_1") # Set group from srdf
# self.move_group.set_planner_id("RRTConnectkConfigDefault") # Set planner type (RRTConnectkConfigDefault)
# self.move_group.set_num_planning_attempts(10) # Set planning attempts
# self.move_group.set_workspace([XMIN,YMIN,ZMIN,XMAX,YMAX,ZMAX]) # Set the workspace size
#Start move_group
self.move_group = MoveGroup('earth', name)
self.move_group.set_planner()
#Start planningScenePublisher
self.scene_pub = PlanningScenePublisher(name)
# Get current robot position to define as start planning point
self.current_pose = self.robot.get_current_state()
# Start trajectory server
self.trajectory_server.start()
class SerialScheduler(Scheduler):
"""Serial scheduler which returns transactions in the natural order.
This scheduler will schedule one transaction at a time (only one may be
unapplied), in the exact order provided as batches were added to the
scheduler.
This scheduler is intended to be used for comparison to more complex
schedulers - for tests related to performance, correctness, etc.
"""
def __init__(self, squash_handler, first_state_hash):
self._txn_queue = queue.Queue()
self._scheduled_transactions = []
self._batch_statuses = {}
self._txn_to_batch = {}
self._in_progress_transaction = None
self._final = False
self._complete = False
self._squash = squash_handler
self._condition = Condition()
# contains all txn.signatures where txn is
# last in it's associated batch
self._last_in_batch = []
self._last_state_hash = first_state_hash
def __iter__(self):
return SchedulerIterator(self, self._condition)
def set_transaction_execution_result(
self, txn_signature, is_valid, context_id):
"""the control flow is that on every valid txn a new state root is
generated. If the txn is invalid the batch status is set,
if the txn is the last txn in the batch, is valid, and no
prior txn failed the batch, the
batch is valid
"""
with self._condition:
if (self._in_progress_transaction is None or
self._in_progress_transaction != txn_signature):
raise ValueError("transaction not in progress: {}",
txn_signature)
self._in_progress_transaction = None
if txn_signature not in self._txn_to_batch:
raise ValueError("transaction not in any batches: {}".format(
txn_signature))
if is_valid:
# txn is valid, get a new state hash
state_hash = self._squash(self._last_state_hash, [context_id])
self._last_state_hash = state_hash
else:
# txn is invalid, preemptively fail the batch
batch_signature = self._txn_to_batch[txn_signature]
self._batch_statuses[batch_signature] = \
BatchExecutionResult(is_valid=is_valid, state_hash=None)
if txn_signature in self._last_in_batch:
batch_signature = self._txn_to_batch[txn_signature]
if batch_signature not in self._batch_statuses:
# because of the else clause above, txn is valid here
self._batch_statuses[batch_signature] = \
BatchExecutionResult(
is_valid=is_valid,
state_hash=self._last_state_hash)
if self._final and self._txn_queue.empty():
self._complete = True
self._condition.notify_all()
def add_batch(self, batch, state_hash=None):
with self._condition:
if self._final:
raise SchedulerError("Scheduler is finalized. Cannnot take"
" new batches")
batch_signature = batch.header_signature
batch_length = len(batch.transactions)
for idx, txn in enumerate(batch.transactions):
if idx == batch_length - 1:
self._last_in_batch.append(txn.header_signature)
self._txn_to_batch[txn.header_signature] = batch_signature
self._txn_queue.put(txn)
self._condition.notify_all()
def get_batch_execution_result(self, batch_signature):
with self._condition:
return self._batch_statuses.get(batch_signature)
def count(self):
with self._condition:
return len(self._scheduled_transactions)
def get_transaction(self, index):
with self._condition:
return self._scheduled_transactions[index]
def next_transaction(self):
with self._condition:
if self._in_progress_transaction is not None:
return None
try:
txn = self._txn_queue.get(block=False)
except queue.Empty:
return None
self._in_progress_transaction = txn.header_signature
txn_info = TxnInformation(txn=txn,
state_hash=self._last_state_hash,
base_context_ids=[])
self._scheduled_transactions.append(txn_info)
return txn_info
def finalize(self):
with self._condition:
self._final = True
if self._txn_queue.empty():
self._complete = True
self._condition.notify_all()
def complete(self, block):
with self._condition:
if not self._final:
return False
if self._complete:
return True
if block:
self._condition.wait_for(lambda: self._complete)
return True
return False
class _ContextFuture:
"""Controls access to bytes set in the _result variable. The booleans
that are flipped in set_result, based on whether the value is being set
from the merkle tree or a direct set on the context manager are needed
to later determine whether the value was set in that context or was
looked up as a new address location from the merkle tree and then only
read from, not set.
In any context the lifecycle of a _ContextFuture can be several paths:
Input:
Address not in base:
F -----> get from merkle database ----> get from the context
Address in base:
|---> set (F)
F --->|
|---> get
Output:
Doesn't exist ----> set address in context (F)
Input + Output:
Address not in base:
|-> set
F |-> get from merkle -|
| |-> get
| |
| |-> noop
|--> set Can happen before the pre-fetch operation
|-> set (F) ---> get
|
|-> set (F) ----> set
|
Address in base: |-> set (F)
Doesn't exist -|
|-> get Future doesn't exit in context
|
|-> get ----> set (F)
"""
def __init__(self, address, result=None, wait_for_tree=False):
self.address = address
self._result = result
self._result_set_in_context = False
self._condition = Condition()
self._wait_for_tree = wait_for_tree
self._tree_has_set = False
self._read_only = False
self._deleted = False
def make_read_only(self):
with self._condition:
if self._wait_for_tree and not self._result_set_in_context:
self._condition.wait_for(
lambda: self._tree_has_set or self._result_set_in_context)
self._read_only = True
def set_in_context(self):
with self._condition:
return self._result_set_in_context
def deleted_in_context(self):
with self._condition:
return self._deleted
def result(self):
"""Return the value at an address, optionally waiting until it is
set from the context_manager, or set based on the pre-fetch mechanism.
Returns:
(bytes): The opaque value for an address.
"""
if self._read_only:
return self._result
with self._condition:
if self._wait_for_tree and not self._result_set_in_context:
self._condition.wait_for(
lambda: self._tree_has_set or self._result_set_in_context)
return self._result
def set_deleted(self):
self._result_set_in_context = False
self._deleted = True
def set_result(self, result, from_tree=False):
"""Set the addresses's value unless the future has been declared
read only.
Args:
result (bytes): The value at an address.
from_tree (bool): Whether the value is being set by a read from
the merkle tree.
Returns:
None
"""
if self._read_only:
if not from_tree:
LOGGER.warning(
"Tried to set address %s on a"
" read-only context.", self.address)
return
with self._condition:
if self._read_only:
if not from_tree:
LOGGER.warning(
"Tried to set address %s on a"
" read-only context.", self.address)
return
if from_tree:
# If the result has not been set in the context, overwrite the
# value with the value from the merkle tree. Otherwise, do
# nothing.
if not self._result_set_in_context:
self._result = result
self._tree_has_set = True
else:
self._result = result
self._result_set_in_context = True
self._deleted = False
self._condition.notify_all()
def __init__(self):
Thread.__init__(self)
self.loop = None
self._cond = Condition()
class SprintDatasetBase(Dataset):
"""
In Sprint, we use this object for multiple purposes:
- Multiple epoch handling via SprintInterface.getSegmentList().
For this, we get the segment list from Sprint and use the Dataset
shuffling method.
- Fill in data which we get via SprintInterface.feedInput*().
Note that each such input doesn't necessarily correspond to a single
segment. This depends which type of FeatureExtractor is used in Sprint.
If we use the BufferedFeatureExtractor in utterance mode, we will get
one call for every segment and we get also segmentName as parameter.
Otherwise, we will get batches of fixed size - in that case,
it doesn't correspond to the segments.
In any case, we use this data as-is as a new seq.
Because of that, we cannot really know the number of seqs in advance,
nor the total number of time frames, etc.
If you want to use this directly in RETURNN, see ExternSprintDataset.
"""
SprintCachedSeqsMax = 200
SprintCachedSeqsMin = 100
def __init__(self, target_maps=None, str_add_final_zero=False, input_stddev=1.,
orth_post_process=None, bpe=None, orth_vocab=None,
suppress_load_seqs_print=False,
**kwargs):
"""
:param dict[str,str|dict] target_maps: e.g. {"speaker": "speaker_map.txt"}
:param bool str_add_final_zero: adds e.g. "orth0" with '\0'-ending
:param float input_stddev: if != 1, will divide the input "data" by that
:param str|list[str]|((str)->str)|None orth_post_process: :func:`get_post_processor_function`, applied on orth
:param None|dict[str] bpe: if given, will be opts for :class:`BytePairEncoding`
:param None|dict[str] orth_vocab: if given, orth_vocab is applied to orth and orth_classes is an available target`
:param bool suppress_load_seqs_print: less verbose
"""
super(SprintDatasetBase, self).__init__(**kwargs)
self.suppress_load_seqs_print = suppress_load_seqs_print
if target_maps:
assert isinstance(target_maps, dict)
target_maps = target_maps.copy()
for key, tmap in list(target_maps.items()):
if isinstance(tmap, (str, unicode)):
tmap = {l: i for (i, l) in enumerate(open(tmap).read().splitlines())}
assert isinstance(tmap, dict) # dict[str,int]
target_maps[key] = tmap
self.target_maps = target_maps
self.str_add_final_zero = str_add_final_zero
self.input_stddev = input_stddev
# Note: "orth" is actually the raw bytes of the utf8 string,
# so it does not make quite sense to associate a single str to each byte.
# However, some other code might expect that the labels are all strings, not bytes,
# and the API requires the labels to be strings.
# The code in Dataset.serialize_data tries to decode this case as utf8 (if possible).
self.labels["orth"] = [chr(i) for i in range(255)]
self.orth_post_process = None # type: typing.Optional[typing.Callable[[str],str]]
if orth_post_process:
if callable(orth_post_process):
self.orth_post_process = orth_post_process
else:
from LmDataset import get_post_processor_function
self.orth_post_process = get_post_processor_function(orth_post_process)
self.bpe = None
if bpe:
from GeneratingDataset import BytePairEncoding
self.bpe = BytePairEncoding(**bpe)
self.labels["bpe"] = self.bpe.labels
self.orth_vocab = None
if orth_vocab:
assert not bpe, "bpe has its own vocab"
from GeneratingDataset import Vocabulary
self.orth_vocab = Vocabulary(**orth_vocab)
self.labels["orth_classes"] = self.orth_vocab.labels
self.cond = Condition(lock=self.lock)
self.add_data_thread_id = thread.get_ident() # This will be created in the Sprint thread.
self.ready_for_data = False
self.reached_final_seq = False
self.reached_final_seq_seen_all = False
self.multiple_epochs = False
self._complete_frac = None
self.sprintEpoch = None # in SprintInterface.getSegmentList()
self.crnnEpoch = None # in CRNN train thread, Engine.train(). set via init_seq_order
self.predefined_seq_list_order = None # via init_seq_order
self.sprintFinalized = False
self._target_black_list = [] # if we get non numpy arrays and cannot convert them
self._reset_cache()
assert self.shuffle_frames_of_nseqs == 0 # Currently broken. But just use Sprint itself to do this.
def use_multiple_epochs(self):
"""
Called via SprintInterface.getSegmentList().
"""
self.multiple_epochs = True
def set_dimensions(self, input_dim, output_dim):
"""
:type input_dim: int
:type output_dim: int
Called via python_train.
"""
assert input_dim > 0
self.num_inputs = input_dim
self.num_outputs = {"data": (input_dim * self.window, 2)}
if output_dim > 0:
self.num_outputs["classes"] = (output_dim, 1)
if self.bpe:
self.num_outputs["bpe"] = (self.bpe.num_labels, 1)
if self.orth_vocab:
self.num_outputs["orth_classes"] = (self.orth_vocab.num_labels, 1)
self.num_outputs["orth"] = (256, 1)
self._base_init()
# At this point, we are ready for data. In case we don't use the Sprint PythonSegmentOrdering
# (SprintInterface.getSegmentList()), we must call this at least once.
if not self.multiple_epochs:
self.init_sprint_epoch(None)
def _reset_cache(self):
self.expected_load_seq_start = 0
self.requested_load_seq_end = 0
self.next_seq_to_be_added = 0
self.reached_final_seq = False
self.reached_final_seq_seen_all = False
self._num_timesteps = 0
self.added_data = [] # type: typing.List[DatasetSeq]
self.ready_for_data = True
def init_sprint_epoch(self, epoch):
"""
:type epoch: int | None
Called by SprintInterface.getSegmentList() when we start a new epoch.
We must not call this via self.init_seq_order() because we will already have filled the cache by Sprint
before the CRNN train thread starts the epoch.
"""
with self.lock:
self.sprintEpoch = epoch
self.sprintFinalized = False
self._reset_cache()
self.cond.notify_all()
def finalize_sprint(self):
"""
Called when SprintInterface.getSegmentList() ends.
"""
with self.lock:
self.sprintFinalized = True
self.cond.notify_all()
def init_seq_order(self, epoch=None, seq_list=None):
"""
Called by CRNN train thread when we enter a new epoch.
"""
super(SprintDatasetBase, self).init_seq_order(epoch=epoch, seq_list=seq_list)
with self.lock:
self.crnnEpoch = epoch
self.predefined_seq_list_order = seq_list
self.cond.notify_all()
# No need to wait/check for Sprint thread here.
# SprintInterface.getSegmentList() will wait for us.
return True
def _cleanup_old_seq_cache(self, seq_end):
i = 0
while i < len(self.added_data):
if self.added_data[i].seq_idx >= seq_end:
break
i += 1
del self.added_data[:i]
def wait_for_returnn_epoch(self, epoch):
"""
Called by SprintInterface.
"""
with self.lock:
while epoch != self.crnnEpoch:
assert epoch > self.crnnEpoch
self.cond.wait()
def _wait_for_seq_can_pass_check(self, seq_start, seq_end):
"""
:param int seq_start:
:param int seq_end:
:return: True if _waitForSeq can pass/return. False means that we need to wait more (until next signal)
:rtype: bool
"""
if self.reached_final_seq:
return True
if self._have_seqs_added(seq_start, seq_end):
return True
return False
def _wait_for_seq(self, seq_start, seq_end=None):
"""
Called by RETURNN train thread.
Wait until we have seqs [seqStart,..,seqEnd-1] loaded,
or we now that they will not be loaded anymore because we reached the end.
:param int seq_start:
:param int|None seq_end:
"""
if seq_end is None:
seq_end = seq_start + 1
if seq_end > self.requested_load_seq_end:
self.requested_load_seq_end = seq_end
self.cond.notify_all()
if self._wait_for_seq_can_pass_check(seq_start=seq_start, seq_end=seq_end):
return
# We need to wait.
assert thread.get_ident() != self.add_data_thread_id
print("%s %s: wait for seqs (%i,%i) (last added: %s) (current time: %s)" % (
self, currentThread().name, seq_start, seq_end, self._latest_added_seq(), time.strftime("%H:%M:%S")), file=log.v5)
while not self._wait_for_seq_can_pass_check(seq_start=seq_start, seq_end=seq_end):
self.cond.wait()
def _latest_added_seq(self):
if not self.added_data:
return None
return self.added_data[-1].seq_idx
def _have_seqs_added(self, start, end=None):
if end is None:
end = start + 1
if start >= end:
return True
for data in self.added_data:
assert start >= data.seq_idx, "%s: We expect that we only ask about the cache of the upcoming seqs." % self
if data.seq_idx == start:
start += 1
if start >= end:
return True
return False
def _get_seq(self, seq_idx):
"""
:param int seq_idx:
:rtype: DatasetSeq
"""
for data in self.added_data:
if data.seq_idx == seq_idx:
return data
return None
def is_cached(self, start, end):
"""
:param int start:
:param int end:
:rtype: bool
"""
# Always False, to force that we call self._load_seqs().
# This is important for our buffer management.
return False
def load_seqs(self, start, end):
"""
Called by RETURNN train thread.
:param int start:
:param int end:
"""
if start == end:
return
if not self.suppress_load_seqs_print:
print("%s load_seqs in %s:" % (self, currentThread().name), start, end, end=' ', file=log.v5)
with self.lock:
super(SprintDatasetBase, self).load_seqs(start, end)
if not self.suppress_load_seqs_print:
print("first features shape:", self._get_seq(start).features["data"].shape, file=log.v5)
def _load_seqs(self, start, end):
"""
Called by RETURNN train thread.
We expect that start increase monotonic on each call
for not-yet-loaded data.
This will already be called with _load_seqs_superset indices.
:param int start:
:param int end:
"""
assert start >= self.expected_load_seq_start
if start > self.expected_load_seq_start:
# Cleanup old data.
self._cleanup_old_seq_cache(start)
self.expected_load_seq_start = start
self.cond.notify_all()
self._wait_for_seq(start, end)
def add_new_data(self, features, targets=None, segment_name=None):
"""
Adds a new seq.
This is called via the Sprint main thread.
:param numpy.ndarray features: format (input-feature,time) (via Sprint)
:param dict[str,numpy.ndarray|str] targets: format (time) (idx of output-feature)
:param str|None segment_name:
:returns the sorted seq index
:rtype: int
"""
# is in format (feature,time)
assert self.num_inputs == features.shape[0]
num_frames = features.shape[1]
# must be in format: (time,feature)
features = features.transpose()
assert features.shape == (num_frames, self.num_inputs)
if self.input_stddev != 1:
features /= self.input_stddev
if self.window > 1:
features = self.sliding_window(features)
assert features.shape == (num_frames, self.num_inputs * self.window)
if targets is None:
targets = {}
if not isinstance(targets, dict):
targets = {"classes": targets}
if "classes" in targets:
# 'classes' is always the alignment
assert targets["classes"].shape == (num_frames,), ( # is in format (time,)
"Number of targets %s does not equal to number of features %s" % (targets["classes"].shape, (num_frames,)))
if "orth" in targets:
targets["orth"] = targets["orth"].decode("utf8").strip()
if "orth" in targets and self.orth_post_process:
targets["orth"] = self.orth_post_process(targets["orth"])
if self.bpe:
assert "orth" in targets
orth = targets["orth"]
assert isinstance(orth, (str, unicode))
assert "bpe" not in targets
targets["bpe"] = numpy.array(self.bpe.get_seq(orth), dtype="int32")
if self.orth_vocab:
assert not self.orth_post_process
assert "orth" in targets
orth = targets["orth"]
assert isinstance(orth, (str, unicode))
assert "orth_classes" not in targets
targets["orth_classes"] = numpy.array(self.orth_vocab.get_seq(orth), dtype="int32")
# Maybe convert some targets.
if self.target_maps:
for key, target_map in self.target_maps.items():
assert key in targets
v = target_map[targets[key]]
v = numpy.asarray(v)
if v.ndim == 0:
v = numpy.zeros((num_frames,), dtype=v.dtype) + v # add time dimension
targets[key] = v
# Maybe remove some targets.
for key in self._target_black_list:
if key in targets:
del targets[key]
# Check if all targets are valid.
for key, v in sorted(list(targets.items())):
if isinstance(v, numpy.ndarray):
continue # ok
if isinstance(v, unicode):
v = v.encode("utf8")
if isinstance(v, (str, bytes)):
if PY3:
assert isinstance(v, bytes)
v = list(v)
else:
v = list(map(ord, v))
v = numpy.array(v, dtype="uint8")
targets[key] = v
if self.str_add_final_zero:
v = numpy.append(v, numpy.array([0], dtype=v.dtype))
assert key + "0" not in targets
targets[key + "0"] = v
continue
print("%s, we will ignore the target %r because it is not a numpy array: %r" % (self, key, v), file=log.v3)
self._target_black_list += [key]
del targets[key]
with self.lock:
# This gets called in the Sprint main thread.
# If this is used together with SprintInterface.getSegmentList(), we are always in a state where
# we just yielded a segment name, thus we are always in a Sprint epoch and thus ready for data.
assert self.ready_for_data
assert not self.reached_final_seq
assert not self.sprintFinalized
seq_idx = self.next_seq_to_be_added
if self.predefined_seq_list_order:
# Note: Only in ExternSprintDataset, we can reliably set the seq order for now.
assert seq_idx < len(self.predefined_seq_list_order), "seq_idx %i, expected predef num seqs %i" % (
seq_idx, len(self.predefined_seq_list_order))
expected_seq_name = self.predefined_seq_list_order[seq_idx]
if expected_seq_name != segment_name:
if segment_name in self.predefined_seq_list_order:
raise Exception("seq_idx %i expected to be tag %r but got tag %r; tag %r is at idx %i" % (
seq_idx, expected_seq_name, segment_name, segment_name,
self.predefined_seq_list_order.index(segment_name)))
raise Exception("seq_idx %i expected to be tag %r but got tag %r; tag %r not found" % (
seq_idx, expected_seq_name, segment_name, segment_name))
self.next_seq_to_be_added += 1
self._num_timesteps += num_frames
self.cond.notify_all()
if seq_idx > self.requested_load_seq_end - 1 + self.SprintCachedSeqsMax:
print("%s add_new_data: seq=%i, len=%i. Cache filled, waiting to get loaded..." % (
self, seq_idx, num_frames), file=log.v5)
while seq_idx > self.requested_load_seq_end - 1 + self.SprintCachedSeqsMin:
assert not self.reached_final_seq
assert seq_idx + 1 == self.next_seq_to_be_added
self.cond.wait()
self.added_data += [DatasetSeq(seq_idx, features, targets, seq_tag=segment_name)]
self.cond.notify_all()
return seq_idx
def finish_sprint_epoch(self, seen_all=True):
"""
Called by SprintInterface.getSegmentList().
This is in a state where Sprint asks for the next segment after we just finished an epoch.
Thus, any upcoming self.add_new_data() call will contain data from a segment in the new epoch.
Thus, we finish the current epoch in Sprint.
"""
with self.lock:
self.reached_final_seq = True
self.reached_final_seq_seen_all = seen_all
self.ready_for_data = False
self.cond.notify_all()
def _shuffle_frames_in_seqs(self, start, end):
assert False, "Shuffling in SprintDataset only via Sprint at the moment."
def get_num_timesteps(self):
"""
:rtype: int
"""
with self.lock:
assert self.reached_final_seq
return self._num_timesteps
@property
def num_seqs(self):
"""
:rtype: int
"""
with self.lock:
if self.predefined_seq_list_order:
return len(self.predefined_seq_list_order)
if not self.reached_final_seq:
raise NotImplementedError
return self.next_seq_to_be_added
def have_seqs(self):
"""
:rtype: bool
"""
with self.lock:
if self.next_seq_to_be_added > 0:
return True
self._wait_for_seq(0)
return self.next_seq_to_be_added > 0
def is_less_than_num_seqs(self, n):
"""
:param int n:
:rtype: bool
"""
with self.lock:
self._wait_for_seq(n)
return n < self.next_seq_to_be_added
def get_data_keys(self):
"""
:rtype: list[str]
"""
with self.lock:
if not self.added_data:
self._wait_for_seq(0)
assert self.added_data
return sorted(self.added_data[0].features.keys())
def get_target_list(self):
"""
:rtype: list[str]
"""
keys = list(self.get_data_keys())
if "data" in keys:
keys.remove("data")
return keys
def set_complete_frac(self, frac):
"""
:param float frac:
"""
self._complete_frac = frac
def get_complete_frac(self, seq_idx):
"""
:param int seq_idx:
:rtype: float
"""
with self.lock:
if self.predefined_seq_list_order:
return float(seq_idx + 1) / len(self.predefined_seq_list_order)
elif self._complete_frac is not None:
if not self.next_seq_to_be_added:
return self._complete_frac
else:
# We can do somewhat better. self._complete_frac is for self.next_seq_to_be_added.
return self._complete_frac * float(seq_idx + 1) / self.next_seq_to_be_added
else:
return super(SprintDatasetBase, self).get_complete_frac(seq_idx)
def get_seq_length(self, sorted_seq_idx):
"""
:param int sorted_seq_idx:
:rtype: Util.NumbersDict
"""
with self.lock:
self._wait_for_seq(sorted_seq_idx)
return self._get_seq(sorted_seq_idx).num_frames
def get_data(self, seq_idx, key):
"""
:param int seq_idx:
:param str key:
:rtype: numpy.ndarray
"""
with self.lock:
self._wait_for_seq(seq_idx)
return self._get_seq(seq_idx).features[key]
def get_input_data(self, sorted_seq_idx):
"""
:param int sorted_seq_idx:
:rtype: numpy.ndarray
"""
with self.lock:
self._wait_for_seq(sorted_seq_idx)
return self._get_seq(sorted_seq_idx).features["data"]
def get_targets(self, target, sorted_seq_idx):
"""
:param str target:
:param int sorted_seq_idx:
:rtype: numpy.ndarray
"""
with self.lock:
self._wait_for_seq(sorted_seq_idx)
return self._get_seq(sorted_seq_idx).features.get(target, None)
def get_ctc_targets(self, sorted_seq_idx):
"""
:param int sorted_seq_idx:
"""
assert False, "No CTC targets."
def get_tag(self, sorted_seq_idx):
"""
:param int sorted_seq_idx:
:rtype: str
"""
with self.lock:
self._wait_for_seq(sorted_seq_idx)
return self._get_seq(sorted_seq_idx).seq_tag
class Approach(object):
def __init__(self, name):
self.robot_name = name
# Mutual exclusion odometry
self.odometry_me = Condition()
# Create trajectory server
self.trajectory_server = SimpleActionServer(
'approach_server', ExecuteDroneApproachAction, self.goCallback,
False)
self.server_feedback = ExecuteDroneApproachFeedback()
self.server_result = ExecuteDroneApproachResult()
# Get client from hector_quadrotor_actions
self.move_client = SimpleActionClient("/{}/action/pose".format(name),
PoseAction)
self.move_client.wait_for_server()
# Subscribe to ground_truth to monitor the current pose of the robot
rospy.Subscriber("/{}/ground_truth/state".format(name), Odometry,
self.poseCallback)
# Subscribe to topic to receive the planned trajectory
rospy.Subscriber("/{}/move_group/display_planned_path".format(name),
DisplayTrajectory, self.planCallback)
#Auxiliary variables
self.trajectory = [] # Array with the trajectory to be executed
self.trajectory_received = False # Flag to signal trajectory received
self.odom_received = False # Flag to signal odom received
self.robot = RobotCommander(
robot_description="{}/robot_description".format(name),
ns="/{}".format(name))
self.display_trajectory_publisher = rospy.Publisher(
'/{}/move_group/display_planned_path'.format(name),
DisplayTrajectory,
queue_size=20)
# Variables for collision callback
self.validity_srv = rospy.ServiceProxy(
'/{}/check_state_validity'.format(name), GetStateValidity)
self.validity_srv.wait_for_service()
self.collision = False
# Set planning algorithm
# self.move = MoveGroupCommander(PLANNING_GROUP, robot_description="{}/robot_description".format(name), ns="/UAV_1") # Set group from srdf
# self.move_group.set_planner_id("RRTConnectkConfigDefault") # Set planner type (RRTConnectkConfigDefault)
# self.move_group.set_num_planning_attempts(10) # Set planning attempts
# self.move_group.set_workspace([XMIN,YMIN,ZMIN,XMAX,YMAX,ZMAX]) # Set the workspace size
#Start move_group
self.move_group = MoveGroup('earth', name)
self.move_group.set_planner()
#Start planningScenePublisher
self.scene_pub = PlanningScenePublisher(name)
# Get current robot position to define as start planning point
self.current_pose = self.robot.get_current_state()
# Start trajectory server
self.trajectory_server.start()
def goCallback(self, pose):
'''
Require a plan to go to the desired target and try to execute it 5 time or return erro
'''
self.target = pose.goal
rospy.loginfo("Try to start from [{},{},{}]".format(
self.odometry.position.x, self.odometry.position.y,
self.odometry.position.z))
rospy.loginfo("Try to go to [{},{},{}]".format(self.target.position.x,
self.target.position.y,
self.target.position.z))
trials = 0
while trials < 5:
rospy.logwarn("Attempt {}".format(trials + 1))
result = self.go(self.target)
if (result == 'replan') or (result == 'no_plan'):
trials += 1
else:
trials = 10
self.collision = False
if result == 'ok':
self.trajectory_server.set_succeeded()
elif (result == 'preempted'):
self.trajectory_server.set_preempted()
else:
self.trajectory_server.set_aborted()
self.trials = 0
def go(self, target_):
'''
Function to plan and execute the trajectory one time
'''
# Insert goal position on an array
target = []
target.append(target_.position.x)
target.append(target_.position.y)
target.append(target_.position.z)
target.append(target_.orientation.x)
target.append(target_.orientation.y)
target.append(target_.orientation.z)
target.append(target_.orientation.w)
#Define target for move_group
# self.move_group.set_joint_value_target(target)
self.move_group.set_target(target)
self.odometry_me.acquire()
self.current_pose.multi_dof_joint_state.transforms[
0].translation.x = self.odometry.position.x
self.current_pose.multi_dof_joint_state.transforms[
0].translation.y = self.odometry.position.y
self.current_pose.multi_dof_joint_state.transforms[
0].translation.z = self.odometry.position.z
self.current_pose.multi_dof_joint_state.transforms[
0].rotation.x = self.odometry.orientation.x
self.current_pose.multi_dof_joint_state.transforms[
0].rotation.x = self.odometry.orientation.y
self.current_pose.multi_dof_joint_state.transforms[
0].rotation.x = self.odometry.orientation.z
self.current_pose.multi_dof_joint_state.transforms[
0].rotation.x = self.odometry.orientation.w
self.odometry_me.release()
#Set start state
self.move_group.set_start_state(self.current_pose)
#Update PlanningSene
self.scene_pub.publishScene(self.current_pose)
#Insert start state on move_group
# self.move_group.set_start_state_to_current_state()
# Plan a trajectory till the desired target
plan = self.move_group.plan()
if plan.planned_trajectory.multi_dof_joint_trajectory.points: # Execute only if has points on the trajectory
# if plan.multi_dof_joint_trajectory.points: # Execute only if has points on the trajectory
while (not self.trajectory_received):
rospy.loginfo("Waiting for trajectory!")
rospy.sleep(0.2)
# rospy.loginfo("TRAJECTORY: {}".format(self.trajectory))
#Execute trajectory with action_pose
last_pose = self.trajectory[0]
for pose in self.trajectory:
# Verify preempt call
if self.trajectory_server.is_preempt_requested():
self.move_client.send_goal(last_pose)
self.trajectory_received = False
self.odom_received = False
return 'preempted'
#Send next pose to move
self.next_pose = pose.target_pose.pose
self.move_client.send_goal(pose,
feedback_cb=self.collisionCallback)
self.move_client.wait_for_result()
result = self.move_client.get_state()
# Abort if the drone can not reach the position
if result == GoalStatus.ABORTED:
self.move_client.send_goal(
last_pose) #Go back to the last pose
self.trajectory_received = False
self.odom_received = False
return 'aborted'
elif result == GoalStatus.PREEMPTED:
return 'replan'
last_pose = pose
self.server_feedback.current_pose = self.odometry
self.trajectory_server.publish_feedback(self.server_feedback)
# Reset control variables
self.trajectory_received = False
self.odom_received = False
rospy.loginfo("Trajectory is traversed!")
return 'ok'
else:
rospy.logerr("Trajectory is empty. Planning was unsuccessful.")
return 'no_plan'
def planCallback(self, msg):
'''
Receive planned trajectories and insert it into an array of waypoints
'''
if (not self.odom_received):
return
# Variable to calculate the distance difference between 2 consecutive points
last_pose = PoseGoal()
last_pose.target_pose.pose.position.x = self.odometry.position.x
last_pose.target_pose.pose.position.y = self.odometry.position.y
last_pose.target_pose.pose.position.z = self.odometry.position.z
last_pose.target_pose.pose.orientation.x = self.odometry.orientation.x
last_pose.target_pose.pose.orientation.y = self.odometry.orientation.y
last_pose.target_pose.pose.orientation.z = self.odometry.orientation.z
last_pose.target_pose.pose.orientation.w = self.odometry.orientation.w
self.trajectory = []
for t in msg.trajectory:
for point in t.multi_dof_joint_trajectory.points:
waypoint = PoseGoal()
waypoint.target_pose.header.frame_id = "{}/world".format(
self.robot_name)
waypoint.target_pose.pose.position.x = point.transforms[
0].translation.x
waypoint.target_pose.pose.position.y = point.transforms[
0].translation.y
waypoint.target_pose.pose.position.z = point.transforms[
0].translation.z
# Orientate the robot always to the motion direction
delta_x = point.transforms[
0].translation.x - last_pose.target_pose.pose.position.x
delta_y = point.transforms[
0].translation.y - last_pose.target_pose.pose.position.y
motion_theta = atan2(delta_y, delta_x)
# Make the robot orientation fit with the motion orientation if the movemente on xy is bigger than 0.2
if (abs(delta_x) > RESOLUTION) or (abs(delta_y) > RESOLUTION):
q = quaternion_from_euler(0, 0, motion_theta)
waypoint.target_pose.pose.orientation.x = q[0]
waypoint.target_pose.pose.orientation.y = q[1]
waypoint.target_pose.pose.orientation.z = q[2]
waypoint.target_pose.pose.orientation.w = q[3]
else:
waypoint.target_pose.pose.orientation.x = point.transforms[
0].rotation.x
waypoint.target_pose.pose.orientation.y = point.transforms[
0].rotation.y
waypoint.target_pose.pose.orientation.z = point.transforms[
0].rotation.z
waypoint.target_pose.pose.orientation.w = point.transforms[
0].rotation.w
last_pose = copy.copy(
waypoint) # Save pose to calc the naxt delta
self.trajectory.append(waypoint)
#Insert a last point to ensure that the robot end at the right position
waypoint = PoseGoal()
waypoint.target_pose.header.frame_id = "{}/world".format(
self.robot_name)
waypoint.target_pose.pose.position.x = point.transforms[
0].translation.x
waypoint.target_pose.pose.position.y = point.transforms[
0].translation.y
waypoint.target_pose.pose.position.z = point.transforms[
0].translation.z
waypoint.target_pose.pose.orientation.x = point.transforms[
0].rotation.x
waypoint.target_pose.pose.orientation.y = point.transforms[
0].rotation.y
waypoint.target_pose.pose.orientation.z = point.transforms[
0].rotation.z
waypoint.target_pose.pose.orientation.w = point.transforms[
0].rotation.w
self.trajectory.append(waypoint)
self.trajectory_received = True
def poseCallback(self, odometry):
'''
Monitor the current position of the robot
'''
self.odometry_me.acquire()
self.odometry = odometry.pose.pose
# print(self.odometry)
self.odometry_me.release()
self.odom_received = True
def collisionCallback(self, feedback):
'''
This callback runs on every feedback message received
'''
validity_msg = GetStateValidityRequest(
) # Build message to verify collision
validity_msg.group_name = PLANNING_GROUP
if self.next_pose and (not self.collision):
self.odometry_me.acquire()
x = self.odometry.position.x
y = self.odometry.position.y
z = self.odometry.position.z
# Distance between the robot and the next position
dist = sqrt((self.next_pose.position.x - x)**2 +
(self.next_pose.position.y - y)**2 +
(self.next_pose.position.z - z)**2)
# Pose to verify collision
pose = Transform()
pose.rotation.x = self.odometry.orientation.x
pose.rotation.y = self.odometry.orientation.y
pose.rotation.z = self.odometry.orientation.z
pose.rotation.w = self.odometry.orientation.w
self.odometry_me.release()
#Verify possible collisions on diferent points between the robot and the goal point
for d in arange(RESOLUTION, dist, RESOLUTION):
pose.translation.x = (self.next_pose.position.x -
x) * (d / dist) + x
pose.translation.y = (self.next_pose.position.y -
y) * (d / dist) + y
pose.translation.z = (self.next_pose.position.z -
z) * (d / dist) + z
self.current_pose.multi_dof_joint_state.transforms[
0] = pose # Insert the correct odometry value
validity_msg.robot_state = self.current_pose
#Update PlanningSene
self.scene_pub.publishScene(self.current_pose)
# Call service to verify collision
collision_res = self.validity_srv.call(validity_msg)
# print("\nCollision response:")
# print(collision_res)
# Check if robot is in collision
if not collision_res.valid:
# print(validity_msg)
rospy.logwarn('Collision in front [x:{} y:{} z:{}]'.format(
pose.translation.x, pose.translation.y,
pose.translation.z))
# print(collision_res)
self.move_client.cancel_goal()
self.collision = True
return
def __init__(self):
self.frame = None
self.buffer = io.BytesIO()
self.condition = Condition()
def __init__(self, target_maps=None, str_add_final_zero=False, input_stddev=1.,
orth_post_process=None, bpe=None, orth_vocab=None,
suppress_load_seqs_print=False,
**kwargs):
"""
:param dict[str,str|dict] target_maps: e.g. {"speaker": "speaker_map.txt"}
:param bool str_add_final_zero: adds e.g. "orth0" with '\0'-ending
:param float input_stddev: if != 1, will divide the input "data" by that
:param str|list[str]|((str)->str)|None orth_post_process: :func:`get_post_processor_function`, applied on orth
:param None|dict[str] bpe: if given, will be opts for :class:`BytePairEncoding`
:param None|dict[str] orth_vocab: if given, orth_vocab is applied to orth and orth_classes is an available target`
:param bool suppress_load_seqs_print: less verbose
"""
super(SprintDatasetBase, self).__init__(**kwargs)
self.suppress_load_seqs_print = suppress_load_seqs_print
if target_maps:
assert isinstance(target_maps, dict)
target_maps = target_maps.copy()
for key, tmap in list(target_maps.items()):
if isinstance(tmap, (str, unicode)):
tmap = {l: i for (i, l) in enumerate(open(tmap).read().splitlines())}
assert isinstance(tmap, dict) # dict[str,int]
target_maps[key] = tmap
self.target_maps = target_maps
self.str_add_final_zero = str_add_final_zero
self.input_stddev = input_stddev
# Note: "orth" is actually the raw bytes of the utf8 string,
# so it does not make quite sense to associate a single str to each byte.
# However, some other code might expect that the labels are all strings, not bytes,
# and the API requires the labels to be strings.
# The code in Dataset.serialize_data tries to decode this case as utf8 (if possible).
self.labels["orth"] = [chr(i) for i in range(255)]
self.orth_post_process = None # type: typing.Optional[typing.Callable[[str],str]]
if orth_post_process:
if callable(orth_post_process):
self.orth_post_process = orth_post_process
else:
from LmDataset import get_post_processor_function
self.orth_post_process = get_post_processor_function(orth_post_process)
self.bpe = None
if bpe:
from GeneratingDataset import BytePairEncoding
self.bpe = BytePairEncoding(**bpe)
self.labels["bpe"] = self.bpe.labels
self.orth_vocab = None
if orth_vocab:
assert not bpe, "bpe has its own vocab"
from GeneratingDataset import Vocabulary
self.orth_vocab = Vocabulary(**orth_vocab)
self.labels["orth_classes"] = self.orth_vocab.labels
self.cond = Condition(lock=self.lock)
self.add_data_thread_id = thread.get_ident() # This will be created in the Sprint thread.
self.ready_for_data = False
self.reached_final_seq = False
self.reached_final_seq_seen_all = False
self.multiple_epochs = False
self._complete_frac = None
self.sprintEpoch = None # in SprintInterface.getSegmentList()
self.crnnEpoch = None # in CRNN train thread, Engine.train(). set via init_seq_order
self.predefined_seq_list_order = None # via init_seq_order
self.sprintFinalized = False
self._target_black_list = [] # if we get non numpy arrays and cannot convert them
self._reset_cache()
assert self.shuffle_frames_of_nseqs == 0 # Currently broken. But just use Sprint itself to do this.
EngineSettingsTable,
ProcessStepTable,
ProcessSubscriptionTable,
ProcessTable,
SubscriptionTable,
WorkflowTable,
db,
)
from orchestrator.services.processes import shutdown_thread_pool
from orchestrator.settings import app_settings
from orchestrator.targets import Target
from orchestrator.workflow import ProcessStatus, done, init, step, workflow
from test.unit_tests.conftest import CUSTOMER_ID
from test.unit_tests.workflows import WorkflowInstanceForTests
test_condition = Condition()
@pytest.fixture
def long_running_workflow():
@step("Long Running Step")
def long_running_step():
with test_condition:
test_condition.wait()
return {"done": True}
@workflow("Long Running Workflow")
def long_running_workflow_py():
return init >> long_running_step >> long_running_step >> done
with WorkflowInstanceForTests(long_running_workflow_py,
class Sampler(object):
""" Sampler used to play, stop and mix multiple sounds.
.. warning:: A single sampler instance should be used at a time.
"""
def __init__(self, sr=22050, backend='sounddevice', timeout=1):
"""
:param int sr: samplerate used - all sounds added to the sampler will automatically be resampled if needed (- his can be a CPU consumming task, try to use sound with all identical sampling rate if possible.
:param str backend: backend used for playing sound. Can be either 'sounddevice' or 'dummy'.
"""
self.sr = sr
self.sounds = []
self.chunks = Queue(1)
self.chunk_available = Condition()
self.is_done = Event() # new event to prevent play to be called again before the sound is actually played
self.timeout = timeout # timeout value for graceful exit of the BackendStream
if backend == 'dummy':
from .dummy_stream import DummyStream
self.BackendStream = DummyStream
elif backend == 'sounddevice':
from sounddevice import OutputStream
self.BackendStream = OutputStream
else:
raise ValueError("Backend can either be 'sounddevice' or 'dummy'")
self.play_thread = None
def __enter__(self):
return self
def __exit__(self, *args):
self.play_thread.join()
def play(self, sound, wait=False):
""" Adds and plays a new Sound to the Sampler.
:param sound: sound to play
.. note:: If the sound is already playing, it will restart from the beginning.
"""
self.is_done.clear() # hold is_done until the sound is played
if self.sr != sound.sr:
raise ValueError('You can only play sound with a samplerate of {} (here {}). Use the Sound.resample method for instance.', self.sr, sound.sr)
if sound in self.sounds:
self.remove(sound)
with self.chunk_available:
self.sounds.append(sound)
sound.playing = True
self.chunk_available.notify()
if self.play_thread is None or not self.play_thread.is_alive():
self.play_thread = Thread(target=self.run)
self.play_thread.daemon = True
self.play_thread.start()
if wait:
self.is_done.wait() # wait for the sound to be entirely played
def remove(self, sound):
""" Remove a currently played sound. """
with self.chunk_available:
sound.playing = False
self.sounds.remove(sound)
# Play loop
def next_chunks(self):
""" Gets a new chunk from all played sound and mix them together. """
with self.chunk_available:
while True:
playing_sounds = [s for s in self.sounds if s.playing]
chunks = []
for s in playing_sounds:
try:
chunks.append(next(s.chunks))
except StopIteration:
s.playing = False
self.sounds.remove(s)
self.is_done.set() # sound was played, release is_done to end the wait in play
if chunks:
break
self.chunk_available.wait()
return numpy.mean(chunks, axis=0)
def run(self):
""" Play loop, i.e. send all sound chunk by chunk to the soundcard. """
self.running = True
def chunks_producer():
while self.running:
self.chunks.put(self.next_chunks())
t = Thread(target=chunks_producer)
t.start()
with self.BackendStream(samplerate=self.sr, channels=1) as stream:
while self.running:
try:
stream.write(self.chunks.get(timeout=self.timeout)) # timeout so stream.write() thread can exit
except Empty:
self.running = False # let play_thread exit
def main():
cv = Condition()
Thread(target=consumer, args=(cv, )).start()
Thread(target=producer, args=(cv, )).start()
while (wypelniona == 0
): #chroni przed przedwczesnym budzeniem, nie stosować if!!!
print("czekanie...")
zm_war.wait() #otwiera atomowo zamek
zm_war.release()
for i in range(0, 10):
print(i, ": tab = ", tab[i])
class Piszacz(Thread):
def run(self):
global tab, wypelniona
global zm_war
print("Pisarz wlasnie pisze")
for i in range(1, 11):
tab.append(1.0 / i / i)
print(i, "Piszetab = ", 1.0 / i / i)
zm_war.acquire()
wypelniona = 1
zm_war.notify()
zm_war.release()
if __name__ == "__main__":
zm_war = Condition()
czyt = Czytacz()
pisz = Piszacz()
czyt.start()
pisz.start()
