def _get_work_book_and_sheet(self):
if not os.path.exists(self._file_path):
raise PlaywellServiceException(
"Could not found the excel file: %s" % self._file_path)
work_book = load_workbook(filename=self._file_path)
if self._sheet_name not in work_book:
raise PlaywellServiceException("找不到sheet %s 在workbook: %s" %
(self._sheet_name, self._file_path))
return work_book, work_book[self._sheet_name]
def _load_config(config_data: dict):
global log_config
log_config = config_data.get(ConfigItems.LOG, {})
global playwell_api
playwell_api = config_data[ConfigItems.PLAYWELL_API]
global input_message_bus_config
input_message_bus_config = config_data.get(ConfigItems.INPUT_MESSAGE_BUS, {})
global all_local_service_config
all_local_service_config = config_data.get(ConfigItems.ALL_LOCAL_SERVICE, {})
global web_server_config
web_server_config = config_data.get(ConfigItems.WEB_SERVER, {})
global runner_config
runner_config = config_data.get(ConfigItems.RUNNER, {})
if ConfigItems.CONFIG_HOOK in config_data:
print(config_data[ConfigItems.CONFIG_HOOK])
config_hook = pydoc.locate(config_data[ConfigItems.CONFIG_HOOK])
if not callable(config_hook):
raise PlaywellServiceException("The config hook must be callable!")
config_hook(config_data)
if ConfigItems.LAUNCH_HOOK in config_data:
global launch_hook_path
launch_hook_path = config_data[ConfigItems.LAUNCH_HOOK]
def get_driver(self, session_id):
"""Get web driver by session id
"""
if session_id not in self._drivers:
raise PlaywellServiceException(
"Could not found web driver session id: %s" % session_id)
return self._drivers[session_id]
def _post_messages(self, url, request_messages):
response = requests.post(
url, json=[msg.to_dict() for msg in request_messages])
if response.status_code != 200:
raise PlaywellServiceException(
"Deliver service request error, url: %s, status code: %d" %
(url, response.status_code))
def convert(target_type: str, data):
"""将数据转换为目标类型
"""
global _converts
if target_type not in _converts:
raise PlaywellServiceException("Unknown convert type: %s" %
target_type)
return _converts[target_type](data)
def register_local_service(self, service_meta: LocalServiceMeta):
"""register local service
"""
if not isinstance(service_meta, LocalServiceMeta):
raise ValueError("Only accept LocalServiceMeta object!")
service_meta.init()
self._all_local_service[service_meta.name] = service_meta
result = self.register_service_meta(service_meta)
if not result.is_ok():
raise PlaywellServiceException(
"Register local service error, the API return: %s" % result)
def _process(self, service, request_messages):
results = service.call(request_messages)
if len(results) != len(request_messages):
raise PlaywellServiceException(
("The service %s return results length "
"is not match with requests length") % service.name)
now = int(time.time() * 1000)
return [
ServiceResponseMessage.from_result(now, request_msg, result)
for request_msg, result in zip(request_messages, results)
if not request_msg.ignore_result
]
def refresh_all(self):
"""Refresh all service meta from API
"""
from playwell.service.config import playwell_api
result = Result.from_response(
requests.get(urljoin(playwell_api, "/v1/service_meta/all")))
if not result.is_ok():
raise PlaywellServiceException(
"Refresh service meta failure, the API return: %s" % result)
self._all_service_meta = {
service_meta["name"]: ServiceMeta(**service_meta)
for service_meta in result.data["services"]
}
def register_input_message_bus(self):
"""注册input message bus
"""
from playwell.service.config import input_message_bus_config
if not input_message_bus_config:
return
if self._input_message_bus is not None:
raise PlaywellServiceException(
"The input message bus has already been registered!")
from playwell.service.config import playwell_api
response = requests.post(urljoin(playwell_api,
"/v1/message_bus/register"),
json={"config": input_message_bus_config})
result = Result.from_response(response)
if not result.is_ok():
raise PlaywellServiceException(
"Register message bus error, the API return %s" % result)
response = requests.post(
urljoin(playwell_api, "/v1/message_bus/open"),
json={"name": input_message_bus_config["name"]})
result = Result.from_response(response)
if not result.is_ok() and result.error_code != "already_opened":
raise PlaywellServiceException(
"Open message bus error, the API return: %s" % result)
from playwell.service.message.http import HttpMessageBus
self._input_message_bus = HttpMessageBus(
name=input_message_bus_config["name"],
clazz=input_message_bus_config["class"],
alive=True,
opened=True,
available=True,
config=input_message_bus_config)
self._input_message_bus.init_web_server()
self._all_playwell_message_bus[
self._input_message_bus.name] = self._input_message_bus
def register_table(cls, activity_id: int, domain_id: str, name: str,
columns: Sequence[Column], meta: dict):
full_name = _full_table_name(activity_id, domain_id, name)
file_path, sheet_name, has_title = meta["path"], meta["sheet"], meta[
"has_title"]
if os.path.exists(file_path):
work_book = load_workbook(filename=file_path)
else:
work_book = Workbook()
ws1 = work_book.active # 移除新建Excel的默认sheet
work_book.remove(ws1)
try:
if sheet_name in work_book:
sheet = work_book[sheet_name]
if not columns:
if not has_title:
raise PlaywellServiceException(
"Excel table %s 没有定义columns，并且也无法从表格自动获取" % name)
columns = [
Column(name=cell.value) for cell in sheet[1]
if cell.value
]
else:
work_sheet = work_book.create_sheet(title=sheet_name)
if has_title:
work_sheet.append([col.title
for col in columns]) # 向新的sheet中写入标题
work_book.save(file_path)
finally:
work_book.close()
table = cls(name=name,
columns=columns,
file_path=file_path,
sheet_name=sheet_name,
has_title=has_title)
ExcelTable.ALL_EXCEL_TABLE_META[full_name] = table
return table
def from_config(cls):
from playwell.service.config import runner_config
executor_type = runner_config.get("executor", "thread_pool")
executor_workers = runner_config.get("executor_workers", 10)
if executor_type == "thread_pool":
executor = ThreadPoolServiceExecutor(max_workers=executor_workers)
else:
raise PlaywellServiceException(
"Unknown service executor type: %s" % executor_type)
sleep_time = runner_config.get("sleep_time", 1)
fetch_requests_num = runner_config.get("fetch_requests_num", 1000)
response_workers = runner_config.get("response_workers", 4)
send_response_timeout = runner_config.get("send_response_timeout", 10)
only_refresh = runner_config.get("only_refresh", False)
return ServiceRunner(service_executor=executor,
sleep_time=sleep_time,
fetch_requests_num=fetch_requests_num,
response_workers=response_workers,
send_response_timeout=send_response_timeout,
only_refresh=only_refresh)
def create_driver(self,
activity_id: int,
domain_id: str,
browser,
tmp: bool = True):
"""Create new web driver by browser and config
"""
if not hasattr(webdriver, browser):
raise PlaywellServiceException(
"Could not found the web driver of browser: %s" % browser)
config = self._all_driver_config.get(browser, {})
driver = getattr(webdriver, browser)(**config)
self._drivers[driver.session_id] = driver
if tmp:
resource_tracer.trace(activity_id=activity_id,
domain_id=domain_id,
type="browser",
name=driver.session_id)
logging.info("Created web driver session: %s" % driver.session_id)
return driver.session_id
def refresh_all(self):
from playwell.service.config import playwell_api
response = requests.get(urljoin(playwell_api, "/v1/message_bus/all"))
result = Result.from_response(response)
if not result.is_ok():
raise PlaywellServiceException(
"Refresh message bus failure, the API return %s" % result)
message_bus_data_seq = result.data.get("buses", [])
all_names = set()
for message_bus_data in message_bus_data_seq:
name = message_bus_data["name"]
clazz = message_bus_data["class"]
all_names.add(name)
if name in self._all_playwell_message_bus:
message_bus = self._all_playwell_message_bus[name]
if message_bus.opened != message_bus_data["opened"]:
message_bus.opened = message_bus_data["opened"]
if message_bus.alive != message_bus_data["alive"]:
message_bus.alive = message_bus_data["alive"]
if message_bus.available != message_bus_data["available"]:
message_bus.available = message_bus_data["available"]
else:
from playwell.service.message.http import HttpMessageBus
if clazz == HttpMessageBus.CLASS_NAME:
message_bus = HttpMessageBus(
name=name,
clazz=clazz,
alive=message_bus_data["alive"],
opened=message_bus_data["opened"],
available=message_bus_data["available"],
config=message_bus_data.get("config", {}))
self._all_playwell_message_bus[
message_bus.name] = message_bus
for message_bus in list(self._all_playwell_message_bus.values()):
if message_bus.name not in all_names:
del self._all_playwell_message_bus[message_bus.name]
def register_all_local_service(self):
from playwell.service.config import all_local_service_config
from playwell.service.message.bus import message_bus_manager
if not all_local_service_config:
return
for local_service_config in all_local_service_config:
service_path = local_service_config["service"]
service = pydoc.locate(service_path)
if not callable(service):
raise PlaywellServiceException(
"The service %s is not a callable component" %
service_path)
if isinstance(service, type):
service = service()
local_service_meta = LocalServiceMeta(
service, local_service_config["name"],
message_bus_manager.input_message_bus.name,
local_service_config.get("config", {}))
self.register_local_service(local_service_meta)
def launch(config_file_path: str):
"""初始化配置 & 开始运行
"""
load_config_from_yaml(config_file_path)
# 初始化日志
from playwell.service.log import init_logging
init_logging()
# 初始化web server
from playwell.service.resource.web import start_web_server
start_web_server()
# 初始化launch hook
from playwell.service.config import launch_hook_path
if launch_hook_path is not None:
launch_hook = pydoc.locate(launch_hook_path)
if not callable(launch_hook):
raise PlaywellServiceException(
"The launch hook %s must be callable" % launch_hook_path)
launch_hook()
# 注册input message bus
from playwell.service.message.bus import message_bus_manager
message_bus_manager.register_input_message_bus()
# 注册local service
from playwell.service.meta import service_meta_manager
service_meta_manager.register_all_local_service()
# 构建ServiceRunner
from playwell.service.runner import ServiceRunner
service_runner = ServiceRunner.from_config()
atexit.register(service_runner.close)
service_runner.run()
def __init__(self, rule):
if not callable(rule):
raise PlaywellServiceException(
"The ServiceRequestProxy rule must be callable!")
self._rule = rule
def _check_table_type(type: str):
if type not in _table_types:
raise PlaywellServiceException("未知的Table类型: %s" % type)
def drop_text(activity_id: int, domain_id: str, name: str):
full_name = MemoryText._full_text_name(activity_id, domain_id, name)
if full_name not in MemoryText.ALL_MEMORY_TEXT:
raise PlaywellServiceException("找不到名为%s的memory_text" % name)
del MemoryText.ALL_MEMORY_TEXT[full_name]
def clear(self):
raise PlaywellServiceException(
"Excel table not support this operation!")
def drop_table(activity_id: int, domain_id: str, name: str):
full_name = _full_table_name(activity_id, domain_id, name)
if full_name not in ExcelTable.ALL_EXCEL_TABLE_META:
raise PlaywellServiceException("找不到指定的Excel table: %s" % name)
del ExcelTable.ALL_EXCEL_TABLE_META[full_name]
def locate_table(activity_id: int, domain_id: str, name: str, _meta: dict):
full_name = _full_table_name(activity_id, domain_id, name)
if full_name not in ExcelTable.ALL_EXCEL_TABLE_META:
raise PlaywellServiceException("找不到指定的Excel table: %s" % name)
return ExcelTable.ALL_EXCEL_TABLE_META[full_name]
def drop_table(activity_id: int, domain_id: str, name: str):
full_name = _full_table_name(activity_id, domain_id, name)
if full_name not in MemoryTable.ALL_MEM_TABLES:
raise PlaywellServiceException("找不到指定的Memory table: %s" % name)
MemoryTable.ALL_MEM_TABLES[full_name].clear()
del MemoryTable.ALL_MEM_TABLES[full_name]
def locate_table(activity_id: int, domain_id: str, name: str, _meta: dict):
full_name = _full_table_name(activity_id, domain_id, name)
if full_name not in MemoryTable.ALL_MEM_TABLES:
raise PlaywellServiceException("找不到指定的Memory table: %s" % name)
return MemoryTable.ALL_MEM_TABLES[full_name]
def get_by_index(self, index: int):
if index >= len(self._elements):
raise PlaywellServiceException("Invalid row index: %d" % index)
return self._elements[index]
def locate_text(activity_id: int, domain_id: str, name: str, _meta):
full_name = FileText._full_text_name(activity_id, domain_id, name)
if full_name not in FileText.ALL_FILE_TEXT_META:
raise PlaywellServiceException("找不到名为%s的file_text" % name)
file_path = FileText.ALL_FILE_TEXT_META[full_name]["path"]
return FileText(name, file_path)
def locate_text(activity_id: int, domain_id: str, name: str, _meta):
full_name = MemoryText._full_text_name(activity_id, domain_id, name)
if full_name not in MemoryText.ALL_MEMORY_TEXT:
raise PlaywellServiceException("找不到名为%s的memory_text" % name)
return MemoryText.ALL_MEMORY_TEXT[name]
def drop_text(activity_id: int, domain_id: str, name: str):
full_name = FileText._full_text_name(activity_id, domain_id, name)
if full_name not in FileText.ALL_FILE_TEXT_META:
raise PlaywellServiceException("找不到名为%s的file_text" % name)
del FileText.ALL_FILE_TEXT_META[full_name]