def arg_genrc(args, params):
ap = ArgumentParser(description="Create a new run control file for TwitterBot", usage="%(prog)s genrc [options]")
ap.add_argument("path", help="The path to save the new run control file")
_args = ap.parse_args(args)
print("This wizard will help you setup a config file.")
print("Please answer the following questions.")
conf = Config(_args.path)
cp, _iter = conf.create()
for c in _iter():
section = c.getSection()
print("\nConfiguration for section: {0}".format(section))
for key, fn_fmt in c.getParams():
print("\n{0}".format(fn_fmt.__doc__))
while 1:
value = raw_input("{0}: ".format(key))
## Verify input was given
if len(value) == 0:
print("Please provide a value.")
continue
## Validate input using the format function
try:
val = fn_fmt(value, setup=True)
c.setParam(key, val, cp)
break;
except:
print("You input was invalid for this configuration parameter. Please try again")
raise
## Save the new config file
cp.save()
def test_loop_timeout(self):
asyncore.loop = Mock()
config = Config()
config.timeout = Mock(return_value=10)
registry = Registry(config)
registry.running = Mock(side_effect=[True, False])
self.assertIsNone(registry.loop())
self.assertEqual(2, registry.running.call_count)
asyncore.loop.assert_called_once_with(timeout=10)
def __init__(self, bot):
self.bot = bot
self.conf = Config.get_conf(self.__class__.__name__, 23894723987423987)
self.conf.register_global(is_ready=False)
self.conf.register_guild(is_server_enabled=False)
self.conf.register_channel(is_channel_enabled=False)
self.conf.register_role(is_role_enabled=False)
self.conf.register_member(is_member_enabled=False)
self.conf.register_user(is_user_enabled=False)
import core.Logs as Logs
import core.Parser as Parser
import core.Time_lib as Time_lib
import core.Config as Config
import core.Html_writer as Html
try:
output_data = {}
Logs.add_log(Config.SEPARATOR + "Program started.")
configs = Config.load_config()
output_data[configs[0]["prefix"]] = Parser.parse(configs[1], configs[0]) # Сохранение данных для каждого сайта
Html.write_html(output_data[configs[0]["prefix"]], configs[0])
Logs.add_log("Program finished. Time: " + Time_lib.time_from_start() + Config.SEPARATOR)
except:
Logs.add_log("Unknown error.")
elif command == "quit":
self.quit()
if __name__ == '__main__':
# Parse command-line options,
# use `Config` to load mind configuration
# command-line overrides config file
args = _parser(sys.argv[1:])
logger.debug("Arguments: {args}".format(args=args))
conf = Config(path=args.mind_dir, **vars(args))
#
# Further patching to ease transition..
#
# Configure Language
logger.debug("Configuring Module: Language")
conf.strings_file = os.path.join(conf.cache_dir, "sentences.corpus")
conf.dic_file = os.path.join(conf.cache_dir, 'dic')
conf.lang_file = os.path.join(conf.cache_dir, 'lm')
conf.fsg_file = None #os.path.join(conf.cache_dir, 'fsg')
# sphinx_jsgf2fsg < conf.jsgf_file > conf.fsg_file
l = LanguageUpdater(conf)
l.update_language()
def __init__(self, bot: Patbot):
self.bot = bot
self.config = Config.get_config(cog_instance=self)
self._register_defaults()
from dotenv import load_dotenv
from core import Config
from core import Authorization
from core import Database
import os
# Setup the .env file
env_file = os.path.join(os.path.dirname(__file__), '.env')
if os.path.isfile(
env_file
) == True: #TODO: ONLY try to load the .env file if the environment is dev and NOT production
load_dotenv(env_file)
# Setup the raven client for Sentry
client = Client(dsn=Config.getValue('SENTRY_DSN'),
environment=Config.getValue('ENVIRONMENT'))
db_file = os.path.join(os.path.dirname(__file__), 'system.db')
# Setup database if not exists
if Config.getValue('SQLITE') == 'yes':
if os.path.isfile(db_file) == False:
Database.createSQLite()
app = Flask(__name__)
app.debug = True
if Config.getValue('ENVIRONMENT') == 'local':
@app.after_request
def get_config(self, key=None):
classname = get_class_name(self)
return Config().get(classname, key)
class PreprocessingControllerThread(threading.Thread):
work_result_queue = Queue()
send_thread_over_queue = queue.Queue()
config = Config()
def __init__(self, ws, pending_queue, state_queue):
super(PreprocessingControllerThread, self).__init__()
self.__ws = ws
# 排队等待切图的队列
self.__pending_queue = pending_queue
# 向主进程发送自己状态的队列
self.__state_queue = state_queue
# 运行控制标记位
self.__running = threading.Event()
# 暂停控制标记位
self.__waiting = threading.Event()
# 存放子线程（工作进程）的列表
self.__sub_thread_array = []
# 消息回复线程
self.__result_thread = None
# 保存处理队列的文件位置
self.__save_file_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "pending_image_list.data")
# 初始化最大工作进程数
self.__work_thread_count = self.config.default_max_processing_thread_number
if not self.__work_thread_count:
self.__work_thread_count = int(multiprocessing.cpu_count() / self.config.default_concurrent_processes_number)
def run(self):
try:
logging.info("preprocessing controller thread start, name:%s" % self.name)
self.__running.set()
self.__waiting.set()
self.__result_thread = PreprocessingResultThread('preprocessing-result-thread', self.__ws, self.work_result_queue, self.send_thread_over_queue)
self.__result_thread.start() # 启动监听结果的线程
pending_image_list = []
lock = threading.Lock()
try:
if os.path.exists(self.__save_file_path):
with open(self.__save_file_path, 'rb+') as load_file:
pending_image_list = pickle.load(load_file)
load_file.truncate()
if pending_image_list:
for image in pending_image_list:
self.__pending_queue.put(image)
logging.info("%s:restore unprocessed pictures to the queue." % self.name)
except Exception as e:
logging.error("%s:failed to restore unprocessed pictures to the queue, error:"%(self.name, str(e)))
# 设置线程得独立loop
self.loop = asyncio.new_event_loop()
asyncio.set_event_loop(self.loop)
sub_thread_wati_count = 1
while self.__running.isSet():
if len(self.__sub_thread_array) == self.__work_thread_count:
logging.debug("%s:waiting for the work thread queue to be free..." % self.name)
time.sleep(sub_thread_wati_count)
if sub_thread_wati_count < self.config.default_max_sleep_time:
sub_thread_wati_count += 1
continue
else:
sub_thread_wati_count = 1
logging.debug("%s:check pause status." % self.name)
self.__waiting.wait() # 如果线程没有暂停就继续
'''
从队列里获取新得图片，如果队列为空，
则抛出异常捕获并休眠等待，
为的是不阻塞回收线程的处理。
'''
image = None
next_image_wati_count = 1
while True:
try:
image = self.__pending_queue.get_nowait()
break
except queue.Empty as e:
if self.__running.isSet():
time.sleep(next_image_wati_count)
if next_image_wati_count < self.config.default_max_sleep_time:
next_image_wati_count += 1
else:
break
if image:
thread_name = 'preprocessing-work-thread-' + image["id"]
work = PreprocessingWorkThread(thread_name, image, self.__sub_thread_array, self.work_result_queue, lock)
work.start()
self.__sub_thread_array.append(thread_name)
logging.debug("%s:start processing thread recycling..." % self.name)
while self.__sub_thread_array:
# 如果还有没有完成的线程，就等着
logging.debug("%s:wait for the work thread to end..." % self.name)
print(self.__sub_thread_array)
time.sleep(3)
# 连接都结束了，还有必要在等着发送结束吗？
logging.debug("%s:notify the send thread to end." % self.name)
if self.__result_thread.isAlive():
self.__result_thread.stop()
send_thread_over_result = self.send_thread_over_queue.get() # 阻塞等待消息发送线程结束
# 都结束了，开始处理未完成的数据
logging.debug("%s:start processing incomplete data." % self.name)
pending_image_list = []
while not self.__pending_queue.empty():
pending_image_list.append(self.__pending_queue.get())
# 将未处理的数据保存到本地文件，等待再启动的时候恢复回来
with open(self.__save_file_path, 'wb') as save_file:
pickle.dump(pending_image_list, save_file)
logging.debug("%s:incomplete data saved." % self.name)
# for task in asyncio.Task.all_tasks():
# #task.cancel()
# print(task)
# 保存完成，告诉父进程可以完了，我都弄好了
self.__state_queue.put((True, "save pending success."))
logging.info('%s:end of processing thread recycling.' % self.name)
finally:
if self.loop:
self.loop.close() # 线程结束关闭loop
def stop(self):
self.__running.clear()
def is_stoped(self):
return self.__running.isSet()
def pause(self):
self.__waiting.clear()
if self.__result_thread.isAlive():
self.__result_thread.pause()
def resume(self):
self.__waiting.set()
if self.__result_thread.isAlive():
self.__result_thread.resume()
def is_pause(self):
return self.__waiting.isSet()
if __name__ == '__main__':
from gi.repository import GObject
from core import Config, Assistant
# Parse command-line options,
# use `Config` to load mind configuration
# command-line overrides config file
args = _parser(sys.argv[1:])
if args.debug:
logging.root.setLevel(logging.DEBUG)
logger.debug("Arguments: {args}".format(args=args))
conf = Config(path=args.mind_dir, **vars(args))
# Database Prototyping
# from core.util.db import DB
# db = DB(os.path.join(conf.cache_dir, "db"))
# db.create_schema()
# for prompt, command in conf.commands.items():
# print("Adding {} -> {}".format(prompt, command))
# db.add_action(prompt, command)
#
# Pre-Configuration
#
# Configure Language
logger.debug("Configuring Module: Language")