def __init__(self):
"""
@brief An abstract executable skill with a description (type,
label, params, conditions), a state and progress code
"""
# Description
self._id = SkillCore.gen_id.getId()
self._type = ""
self._label = ""
self._description = SkillDescription()
# Params
self._params = params.ParamHandler()
# Conditions
self._pre_conditions = []
self._hold_conditions = []
self._post_conditions = []
# Execution
self._state_change = Event()
self._state = State.Uninitialized
self._avg_time_keeper = TimeKeeper()
self._time_keeper = TimeKeeper()
self._progress_code = 0
self._progress_period = 0.0
self._progress_time = 0.0
self._progress_msg = ""
self._expand_on_start = False
def producer(queue):
for i in range(10):
event = Event()
queue.put((event, i))
print(f"[生产者]生产了产品{i}")
event.wait() # 等待消费者通知
print(f"生产者已经收到消费情况的反馈{i}")
class TaskResult(object):
def __init__(self):
self.__event = Event()
self.__result = None
def result(self):
# 阻塞等结果
self.__event.wait()
return self.__result
def set_result(self, value):
self.__result = value
self.__event.set() # 标记执行完毕
class BaseActor(object):
def __init__(self):
"""queue：Actor内部的邮箱队列"""
self.__mailbox = Queue()
def recv(self):
"""Actor接受消息"""
msg = self.__mailbox.get()
if msg is ActorExit:
# 抛出异常（模版方法会处理）
raise ActorExit
return msg
def send(self, msg):
"""Actor发送消息"""
self.__mailbox.put(msg)
def close(self):
"""发送结束标识"""
self.send(ActorExit)
def start(self):
self.__terminated_event = Event() # 为Join服务
t = threading.Thread(target=self.__templet)
t.setDaemon(True) # 设置为守护线程
t.start()
def __templet(self):
"""模版方法（run会被子类重写）"""
try:
self.run() # 执行Run代码
except ActorExit:
pass # 防止线程挂掉
finally:
# 设置Event标识
self.__terminated_event.set()
def join(self):
# Event在set之后便结束等待
self.__terminated_event.wait()
def run(self):
"""
由子类实现即可，eg：
while True:
msg = self.recv()
print(msg)
"""
pass
def createDemo(self):
usersData = []
event = Event()
pool = ThreadPool(multiprocessing.cpu_count() * 2)
pool = ThreadPool(5)
for i in range(0, 1000):
string = hashlib.sha224()
string.update('{}'.format(random.random()))
first = 'first{}'.format(string.hexdigest()[0:10])
string.update('{}'.format(random.random()))
last = 'last{}'.format(string.hexdigest()[0:10])
tel = '{}'.format(8005550000 + i)
email = 'email{}@localhost.email'.format(string.hexdigest()[0:10])
postData = {
'first': first,
'last': last,
'tel': tel,
'email': email,
'pass': '******',
'type': 'customer',
'event': event
}
usersData.append(postData)
results = pool.map(self.createUser, usersData)
pool.close()
pool.join()
def _draw_start(self):
if not self._somethingnew:
return True
# let's draw!
q = Queue.Queue()
evt = Event()
th = Process(target=self._draw_real, args=(q, evt), name='GTKDraw')
th.start()
gobject.timeout_add(500, self._draw_end, q, evt)
return False
def __init__(self):
# Description
self._id = SkillCore.gen_id.getId()
self._type = ""
self._label = ""
self._description = SkillDescription()
# Params
self._params = params.ParamHandler()
# Conditions
self._pre_conditions = []
self._hold_conditions = []
self._post_conditions = []
# Execution
self._state_change = Event()
self._state = State.Uninitialized
self._time_keeper = TimeKeeper()
self._progress_code = 0
self._progress_period = 0.0
self._progress_time = 0.0
self._progress_msg = ""
self._expand_on_start = False
def multi_thread(cam_nums, func, args):
event = Event()
try:
event.set()
pool = Pool(len(cam_nums))
for arg in args:
arg.event = event
results = pool.map(func, args)
except KeyboardInterrupt:
event.clear()
finally:
pool.close()
pool.join()
print('DONE')
from time import sleep
from multiprocessing.dummy import Pool as ThreadPool, Event
event = Event()
def click():
# event.clear() # 设置标准为假（默认是False）
print("用户在修改网页表单")
sleep(2)
print("点击了修改案例")
event.set() # 设置标准为真
def update():
print(f"事件状态：{event.is_set()}")
event.wait() # 等待到标志为真
print("修改成功")
print(f"事件状态：{event.is_set()}")
def main():
pool = ThreadPool()
pool.apply_async(click)
pool.apply_async(update)
pool.apply_async(click)
pool.close()
pool.join()
if __name__ == '__main__':
def start(self):
self.__terminated_event = Event() # 为Join服务
t = threading.Thread(target=self.__templet)
t.setDaemon(True) # 设置为守护线程
t.start()
class SkillCore(SkillDescription):
gen_id = IdGen()
def __init__(self):
"""
@brief An abstract executable skill with a description (type,
label, params, conditions), a state and progress code
"""
# Description
self._id = SkillCore.gen_id.getId()
self._type = ""
self._label = ""
self._description = SkillDescription()
# Params
self._params = params.ParamHandler()
# Conditions
self._pre_conditions = []
self._hold_conditions = []
self._post_conditions = []
# Execution
self._state_change = Event()
self._state = State.Uninitialized
self._avg_time_keeper = TimeKeeper()
self._time_keeper = TimeKeeper()
self._progress_code = 0
self._progress_period = 0.0
self._progress_time = 0.0
self._progress_msg = ""
self._expand_on_start = False
# --------Class functions--------
def expand(self, skill):
return
def hasChildren(self):
return False
def _setState(self, state):
self._state = state
self._state_change.set()
def _setProgress(self, msg, code=None):
if code is None:
code = self._progress_code + 1
self._progress_code = code
self._progress_period = self._avg_time_keeper.get_avg_time()
self._progress_time = self._time_keeper.time_from_start()
self._progress_msg = str(msg)
@property
def id(self):
return self._id
@property
def progress_code(self):
return self._progress_code
@property
def progress_period(self):
return self._progress_period
@property
def progress_time(self):
return self._progress_time
@property
def progress_msg(self):
return self._progress_msg
@property
def state(self):
return self._state
@property
def expand_on_start(self):
"""
@brief Default False. If true, the skill will expand every time it
is started. Used e.g. in a planner skill
"""
return self._expand_on_start
def _resetDescription(self, other=None):
if other:
self._params.reset(self._description._params.merge(other._params))
else:
self._params = deepcopy(self._description._params)
self._pre_conditions = deepcopy(self._description._pre_conditions)
self._hold_conditions = deepcopy(self._description._hold_conditions)
self._post_conditions = deepcopy(self._description._post_conditions)
def hasPreCond(self):
return bool(self._pre_conditions)
def checkPreCond(self, verbose=False):
"""
@brief Check pre-conditions.
@param verbose (bool) Print error message when check fail
@return A list of parameters that breaks the conditions, or an empty
list if all are satisfied
"""
to_ret = list()
err_msg = ""
for c in self._pre_conditions:
if not c.evaluate(self._params, self._wmi):
err_msg += "{} Check failed. \n".format(c.getDescription())
if verbose:
log.error(c.getDescription(), "ConditionCheck failed")
to_ret += c.getKeys()
self._setProgress(err_msg, -1)
return list(set(to_ret))
def checkHoldCond(self, verbose=False):
"""
@brief Check hold-conditions.
@param verbose (bool) Print error message when check fail
@return A list of parameters that breaks the conditions, or an empty
list if all are satisfied
"""
to_ret = list()
err_msg = ""
for c in self._hold_conditions:
if not c.evaluate(self._params, self._wmi):
err_msg += "{} Check failed. \n".format(c.getDescription())
if verbose:
log.error("HoldConditionCheck failed", c.getDescription())
to_ret += c.getKeys()
self._setProgress(err_msg, -2)
return list(set(to_ret))
def hasPostCond(self):
return bool(self._post_conditions)
def checkPostCond(self, verbose=False):
"""
@brief Check post-conditions.
@param verbose (bool) Print error message when check fail
@return A list of parameters that breaks the conditions, or an empty
list if all are satisfied
"""
to_ret = list()
for c in self._post_conditions:
if not c.evaluate(self._params, self._wmi):
if verbose:
log.error(c.getDescription(), "ConditionCheck failed")
to_ret += c.getKeys()
return list(set(to_ret))
# -------- Control functions--------
def preempt(self):
if self.hasState(State.Running):
self._setState(self.onPreempt())
if not self.onEnd():
self._setState(State.Failure)
return self._state
def getState(self):
return self._state
def hasState(self, state):
return self._state == state
def waitState(self, state, isset=True):
if isset: # Xor?
while self._state != state:
# print 'Waiting set.. {}'.format(self._state)
self._state_change.clear()
self._state_change.wait()
else:
while self._state == state:
# print 'Waiting not set.. {}'.format(self._state)
self._state_change.clear()
self._state_change.wait()
# print 'State changed {}'.format(self._state)
def reset(self):
self.onReset()
self._params.setDefault()
self._time_keeper.reset()
self._avg_time_keeper.reset()
self._setProgress("", 0)
self._setState(State.Idle)
return self._state
def start(self, params=None):
if params:
self.specifyParams(params, False)
self._time_keeper.reset()
if self.onStart():
self._setState(State.Running)
self._setProgress("Start", 0)
else:
self._setState(State.Failure)
return self._state
def printInfo(self, verbose=False):
s = "{}-{} ".format(self._type, self._label)
if verbose:
s += "["
s += self._params.printState() + ']\n'
s += self.printConditions()
else:
s += "\n"
return s
def printState(self, verbose=False):
s = "{}-{}({})".format(self.type[self.type.find(":") + 1:], self.label,
self.state)
if verbose:
s += "[{}]".format(self.params.printState())
return s
def printProgress(self):
return "[{}] {}".format(self._progress_code, self._progress_msg)
def specifyParamDefault(self, key, values):
"""
@brief Specify a value and set it as default value too
@param key (string) Parameter key
@param values Parameter value(s)
"""
if not self._params.hasParam(key):
log.error(
"specifyParamDefault", "No param '{}' found. Debug: {}".format(
key, self.printInfo(True)))
self._params.specifyDefault(key, values)
def specifyParam(self, key, values):
"""
@brief Specify a parameter and update the input cache
@param key (string) Parameter key
@param values Parameter value(s)
"""
if not self._params.hasParam(key):
log.error(
"specifyParam", "No param '{}' found. Debug: {}".format(
key, self.printInfo(True)))
self._params.specify(key, values)
def specifyParamsDefault(self, input_params):
"""
@brief Set the parameters and makes them default (they will no more
be overwritten by specifyParams, even with
keep_offline=False)
@param input_params (dict)
"""
self._params.specifyParamsDefault(input_params)
def specifyParams(self, input_params, keep_default=True):
"""
@brief Set the parameters
@param input_params (dict) Parameters to set
@param keep_default (bool) If True, params already specified are
preserved
"""
self._params.specifyParams(input_params, keep_default)
# -------- User's functions--------
def setDescription(self, description, label=""):
"""
@brief Description is a SkillDescription
"""
self._description = description
self._type = description._type
if label != "":
self._label = label
self._resetDescription()
def startError(self, msg, code):
"""
@brief signal an error during the starting routine
"""
assert type(msg) == str
assert type(code) == int
self.fail(msg, code)
return False
def step(self, msg=""):
"""
@brief Set a running breakpoint
"""
assert type(msg) == str
self._setProgress(msg)
return State.Running
def fail(self, msg, code):
"""
@brief Set a failure state
"""
assert type(msg) == str
assert type(code) == int
if code > 0:
code *= -1
self._setProgress(msg, code)
return State.Failure
def success(self, msg=""):
"""
@brief Set a success state
"""
assert type(msg) == str
self._setProgress(msg)
return State.Success
# -------- Virtual functions--------
def modifyDescription(self, skill):
"""
@brief Override to define additional parameters/condition over the skill
"""
pass
def onReset(self):
"""
@brief Called when resetting.
"""
pass
def onStart(self):
"""
@brief Called just before 1st execute
@return (Bool)
"""
return True
def onPreempt(self):
"""
@brief Called when skill is requested to stop.
@return (State)
"""
self._setProgress("Preempted", -1)
return State.Failure
def execute(self):
"""
@brief Main execution function
@return (State)
"""
raise NotImplementedError("Not implemented in abstract class")
def onEnd(self):
"""
@brief Called just after last execute or preemption
@return (Bool)
"""
return True
def run(self):
"""Start process all shots with a processbar."""
shots = self.get_shot_list()
shots_info = dict.fromkeys(self._all_shots, '本次未处理')
is_multi_threading = self.flags & MULTI_THREADING
thread_count = cpu_count() if is_multi_threading else 1
pool = Pool(thread_count)
proc_queue = Queue()
cancel_event = Event()
def _run(shot):
if cancel_event.is_set():
return '取消: {}'.format(shot)
output = os.path.join(
CONFIG['output_dir'], '{}_v0.nk'.format(shot))
input_dir = shot if os.path.isdir(
shot) else os.path.join(CONFIG['input_dir'], shot)
cmd = u'"{nuke}" -t -priority low "{script}" "{input_dir}" "{output}"'.format(
nuke=nuke.EXE_PATH,
script=__path__,
input_dir=input_dir,
output=output
)
try:
LOGGER.info('%s:开始', shot)
proc = Popen(cmd,
# shell=True,
stdout=PIPE,
stderr=PIPE)
proc_queue.put(proc)
stderr = u(proc.communicate()[1])
if START_MESSAGE in stderr:
stderr = stderr.partition(
START_MESSAGE)[2].strip()
if stderr:
shots_info[shot] = stderr
elif proc.returncode:
if cancel_event.is_set():
shots_info[shot] = '用户取消'
else:
shots_info[shot] = 'Nuke非正常退出: {}'.format(
proc.returncode)
else:
shots_info[shot] = '正常完成'
LOGGER.info('%s:结束', shot)
except:
shots_info[shot] = traceback.format_exc()
LOGGER.error('Unexpected exception during comp', exc_info=True)
raise RuntimeError
return '完成: {}'.format(shot)
if is_multi_threading:
_run = run_with_memory_require(8)(_run)
def _oncancel():
cancel_event.set()
while not proc_queue.empty():
proc = proc_queue.get()
if proc.poll() is None:
try:
proc.terminate()
except OSError:
pass
try:
for _ in progress(pool.imap_unordered(_run, shots),
name='批量合成',
total=len(shots),
start_message=(
'正在使用 {} 线程进行……'.format(thread_count)),
oncancel=_oncancel):
pass
except (CancelledError, RuntimeError):
pass
webbrowser.open(self.generate_report(shots_info))
webbrowser.open(CONFIG['output_dir'])
def __init__(self):
self.__event = Event()
self.__result = None
def do_GET(self):
# отправка видео по запросу из плейлиста
if re.match('.*p\d+\.ts', self.path):
print('REQUEST', self.path)
idx = int(re.match('.*p(\d).ts', self.path).groups()[0])
self.send_response(200, 'OK')
self.send_header('content-type', 'video/vnd.dlna.mpeg-tts')
self.end_headers()
parts = PartsHandler.parts
popen = PartsHandler.popen
if parts[idx]['hosting'] == 'google':
# через ffmpeg кусок скачивается и преобразуется в .ts
popen = subprocess.Popen(parts[idx]['cmd'],
stdout=subprocess.PIPE)
# выхлоп отправляется напрямую по запросу
while True:
line = popen.stdout.readline()
if line:
try:
self.wfile.write(line)
except Exception as e:
print('EXCEPTION: ', e)
popen.stdout.close()
popen.kill()
else:
popen.stdout.close()
popen.kill()
print('ffmpeg killed')
break
elif parts[idx]['hosting'] == 'big-sword':
# из-за особенностей сервера big-sword качать пришлось
# отдельно, pipe'ить в ffmpeg, а оттуда уже отправлять
# код с дополнительным процессом ниже необходим, т.к.
# баг в Popen под windows
ffmpeg = subprocess.Popen(parts[idx]['cmd'], \
stdin=subprocess.PIPE, stdout=subprocess.PIPE)
stop_event = Event()
gvp = Process(target=self.get_video,
args=(ffmpeg, parts[idx]['url'], stop_event))
gvp.start()
while True:
line = ffmpeg.stdout.readline()
if line:
try:
self.wfile.write(line)
except Exception as e:
print('Exception occured when sending ffmpeg ' \
'output:\n', e)
break
else:
break
ffmpeg.stdout.close()
ffmpeg.kill()
stop_event.set()
gvp.join()
# генерация и отправка плейлистов, при получении
# в запросе
elif re.search(r'playlist\.m3u8', self.path):
if re.search(r'http://lh3.googleusercontent', self.path):
hosting = 'google'
elif re.search(r'\.big-sword', self.path):
hosting = 'big-sword'
else:
return
print('PLAYLIST REQUEST:', self.path)
paths = uq(self.path)
self.urls = paths.lstrip('/playlist.m3u8?').split(' ')
PartsHandler.parts = get_video_info(self.urls, hosting)
playlist = build_playlist(PartsHandler.parts)
self.send_response(200, 'OK')
self.send_header('Content-Length', len(playlist))
self.send_header('Content-Type', 'application/vnd.apple.mpegurl')
self.end_headers()
self.wfile.write(playlist.encode())
from time import sleep
from random import random
from multiprocessing.dummy import Pool as ThreadPool, Event
global_list = []
event = Event()
stop_event = Event()
n = 0
def consumer(i):
print(f"消费者{i}等待ing")
while 1:
event.wait()
count = len(global_list)
# 防止List空的时候pop出错
if count > 0:
print(f"消费了产品{global_list.pop()}")
event.clear() # 重置状态（加这一句能减少很多次循环）
# 防止生产者结束了，但是消费者还没处理完成
elif len(global_list) == 0 and stop_event.is_set():
break
global n
n += 1
print(f"消费者{i}完成任务～总共循环{n}次")
def producer():
print("生产者正在生产商品")
for i in range(10):