def run(self, tabu_list: list) -> None:
#print(f"ic| Executing Tabu Search with distance {self.distance}")
if self.counter == 0:
self.output_file.write_line(
self.output_filename.replace('TEMP-', ''))
self.output_file.write_line(str(self.solution.optimum))
self.output_file.write_line(
f"{self.counter} {self.solution.value}")
mask_list = General.get_mask_list(self.solution.n,
self.distance,
climb=True)
(value_list,
weight_list) = General.parse_item_list_data(self.item_list)
teste = self.evaluate_neighborhood_tabu(self.solution, mask_list,
value_list, weight_list,
tabu_list)
if teste:
self.counter += 1
#self.solution.print_solution()
#ic(f"{self.counter} {self.solution.value}")
self.output_file.write_line(
f"{self.counter} {self.solution.value}")
def run_local_search_improved(self):
print(
f"ic| run_local_search_improved: Executing Local Search with distance {self.distance}"
)
counter = 0
output_file = FileWriter(file_name=self.output_filename)
output_file.write_line(self.output_filename.replace('TEMP-', ''))
output_file.write_line(str(self.solution.optimum))
output_file.write_line(f"{counter} {self.solution.value}")
mask_list = General.get_mask_list(self.solution.n,
self.distance,
climb=True)
(value_list,
weight_list) = General.parse_item_list_data(self.item_list)
while self.evaluate_neighborhood_improved(self.solution, mask_list,
value_list, weight_list):
counter += 1
#self.solution.print_solution()
ic(f"{counter} {self.solution.value}")
output_file.write_line(f"{counter} {self.solution.value}")
# prevent looping indefinitely
if counter >= 100:
return
def run_vns(self, solution: Solution, item_list: list, max_iterations: int,
neighborhood_size: int, output_filename: str):
print(f"ic| run_vns: Executing Variable Neighbourhood Search")
counter = 0
output_file = FileWriter(file_name=output_filename)
output_file.write_line(output_filename.replace('TEMP-', ''))
output_file.write_line(str(solution.optimum))
output_file.write_line(f"{counter} {solution.value}")
(value_list,
weight_list) = General.parse_item_list_data(self.item_list)
for i in range(max_iterations):
initial_solution = solution
k = 1
while k <= neighborhood_size:
mask_list = General.get_mask_list(solution.n, k, climb=False)
initial_solution = self.random_neighbor(
solution, k, value_list, weight_list)
if not initial_solution:
continue
best_neighbor = self.evaluate_neighborhood(
initial_solution, mask_list, value_list, weight_list)
if best_neighbor and best_neighbor.value > solution.value:
counter += 1
solution = deepcopy(best_neighbor)
#solution.print_solution()
#ic(f"{counter} {solution.value}")
output_file.write_line(f"{counter} {solution.value}")
else:
k += 1
def random_neighbor(self, distance: int):
mask_list1 = General.get_mask_list(self.solution.n, distance, climb=False)
(value_list1, weight_list1) = General.parse_item_list_data(self.item_list)
solution_binary = "".join([str(item) for item in self.solution.item_list])
solution_number = int(solution_binary, 2)
mask_list = deepcopy(mask_list1)
random.shuffle(mask_list)
for mask in mask_list:
masked_number = solution_number ^ int(mask, 2)
masked_binary = bin(masked_number)[2:].zfill(self.solution.n)
neighbor = [int(digit) for digit in masked_binary]
neighbor_weight_list = [a*b for a,b in zip(neighbor, weight_list1)]
if sum(neighbor_weight_list) <= self.solution.capacity:
neighbor_value_list = [a*b for a,b in zip(neighbor, value_list1)]
self.solution.value = sum(neighbor_value_list)
self.solution.weight =sum(neighbor_weight_list)
self.solution.item_list = deepcopy(neighbor)
return True
return False
def get_test_data_path(self):
self.general = General()
self.config_data = ConfigData()
path = str(self.general.get_directory_path())
# full_path = path + self.config_data.TEST_DATA_PATH
full_path = path + self.get_os_based_excel_path()
return full_path
def __init__(self, task_list):
self._task_list = task_list
self._cfg = Config()
self._dbmgr = DBMgr()
self._general = General()
# 將task_list中每一筆子任務的細節抓出來
self.task_list_detail = self._get_task_list_detail()
self.factor_list = self._get_factor_list()
def __init__(self):
self._cfg = Config()
self._path = self._cfg.get_value('path', 'path_to_portfolio_performance')
self._api_server_IP = self._cfg.get_value('IP', 'api_server_IP')
self._start_equity = int(self._cfg.get_value('parameter', 'start_equity'))
self._start_date = self._cfg.get_value('parameter', 'start_date')
self._end_date = self._cfg.get_value('parameter', 'end_date')
self._general = General()
self._cal = Calendar('TW')
def __init__(self, request):
self._request = request
self._task_list = []
self._task_id = -1
self._factor_analysis_handler = FactorAnalysisHandler()
self._node_handler = NodeHandler()
self._dbmgr = DBMgr()
self._general = General()
self._cfg = Config()
self._api_server_IP = self._cfg.get_value('IP', 'api_server_IP')
def __init__(self):
self._cfg = Config()
self._dbmgr = DBMgr()
self._general = General()
# 基本參數 (伺服器設定值)
self._processes = os.cpu_count() # 伺服器CPU核心數
self._server_name = getpass.getuser() # 伺服器使用者名稱
self._IP = socket.gethostbyname(socket.gethostname())
self._host_IP = self._cfg.get_value('IP', 'host_IP')
self.client_ID = self._server_name + "_" + self._IP
self._mqtt_account = self._cfg.get_value('MQTT', 'account')
self._mqtt_password = self._cfg.get_value('MQTT', 'password')
class FindIMDB(BasePage):
"""Locators of the Find page"""
lblSearchResult = (By.XPATH, "//span[@class= 'findSearchTerm']")
lnkSearchResults = (
By.XPATH,
"//div[@class='findSection']//table[@class='findList']//td[@class='result_text']//a[contains(@href, 'title')]"
)
"""constructor of the FindIMDB class"""
def __init__(self, driver):
super().__init__(driver)
self.general = General()
self.get_title("Find - IMDb")
# Get Search result label value
# params: None
def get_search_result(self):
search_result_label = self.get_element_text(self.lblSearchResult)
return search_result_label
# Get Search result label value
# params: expected search result
def verify_search_result_label(self, expected_search_result_label):
actual_search_result_label = self.get_search_result()
actual_search_result_label = self.general.strip_string_value(
actual_search_result_label, '"')
assert actual_search_result_label == expected_search_result_label, "Actual search label and expected label is not same as expected"
# get Search result links
# params: None
def get_search_result_links(self):
search_links = self.get_value_from_grid(self.lnkSearchResults)
search_result_list = []
for element in search_links:
search_result_list.append(element.text)
return search_result_list
# verify text in Search result links
# params: search text, list
def verify_search_result_links_text(self, expected_text,
search_result_list):
self.general.verify_list_values(search_result_list, expected_text)
# verify navigate back page
# params: expected title of page
def verify_navigate_back(self, expected_title):
self.driver.back()
time.sleep(10)
actual_title = self.driver.title
assert actual_title == expected_title, "Actual and expected title values does not match"
def __init__(self, strategy_config, cal, fac):
self._strategy_config = strategy_config
self._cal = cal
self._fac = fac
self._cfg = Config()
self._path = self._cfg.get_value('path',
'path_to_portfolio_performance')
self._general = General()
self._factor_str = self._general.factor_to_string(
self._strategy_config['factor'])
self._show_task_detail()
self._portfolio = Portfolio(self._strategy_config, self._cal,
self._fac)
self._write_portfolio_performance()
def run(self, counter):
#print(f"ic| Executing Iterated Local Search with distance {self.distance}")
if counter == 0:
self.output_file.write_line(
self.output_filename.replace('TEMP-', ''))
self.output_file.write_line(str(self.solution.optimum))
self.output_file.write_line(f"{counter} {self.solution.value}")
mask_list = General.get_mask_list(self.solution.n,
self.distance,
climb=True)
(value_list,
weight_list) = General.parse_item_list_data(self.item_list)
while self.evaluate_neighborhood(self.solution, mask_list, value_list,
weight_list):
#self.solution.print_solution()
#ic(f"{self.counter} {self.solution.value}")
pass
def run(self, counter):
#print(f"ic| Executing Multi Start Local Search with distance {self.distance}")
if counter == 0:
self.output_file.write_line(
self.output_filename.replace('TEMP-', ''))
self.output_file.write_line(str(self.solution.optimum))
self.output_file.write_line(f"{counter} {self.solution.value}")
mask_list = General.get_mask_list(self.solution.n,
self.distance,
climb=True)
(value_list,
weight_list) = General.parse_item_list_data(self.item_list)
counter_stop = 0
while self.evaluate_neighborhood(self.solution, mask_list, value_list,
weight_list):
counter_stop += 1
# prevent from looping indefinitely
if counter_stop >= 100:
return
class FactorAnalysisTask:
def __init__(self, task_list):
self._task_list = task_list
self._cfg = Config()
self._dbmgr = DBMgr()
self._general = General()
# 將task_list中每一筆子任務的細節抓出來
self.task_list_detail = self._get_task_list_detail()
self.factor_list = self._get_factor_list()
def _get_task_list_detail(self):
sql = " SELECT * \
FROM `task_status` \
WHERE "
args = {}
for task_status in self._task_list:
label = 'task_status_id_' + str(task_status)
if task_status == self._task_list[-1]:
sql = sql + "`task_status_id`=%({})s".format(label)
else:
sql = sql + "`task_status_id`=%({})s OR ".format(label)
args[label] = str(task_status)
status, row, data = self._dbmgr.query(sql, args, fetch='all')
return data if data else {}
def _get_factor_list(self):
factor_list = []
# 找出不重複的因子
for i, task_detail in enumerate(self.task_list_detail):
self.task_list_detail[i]['factor'] = self._general.string_to_list(
task_detail['factor'])
for factor in task_detail['factor']:
if factor not in factor_list:
factor_list.append(factor)
return factor_list
def get(self):
output = []
try:
factor_list_str = request.args.get('factor_list')
strategy_list = list(
map(int, request.args.getlist('strategy_list')))
window_list = list(map(int, request.args.getlist('window_list')))
method_list = list(map(int, request.args.getlist('method_list')))
group_list = list(map(int, request.args.getlist('group_list')))
position_list = list(
map(int, request.args.getlist('position_list')))
factor_list = General().factor_string_to_list(factor_list_str)
data = (factor_list, strategy_list, window_list, method_list,
group_list, position_list)
for combi in product(*data):
# strategy: B&H
if combi[1] == 0:
# B&H 排除0之外的窗格 排除等權重之外的方法
if combi[2] != 0 or combi[3] != 0:
continue
# if combi[2] == 1 or combi[2] == 2:
# if len(combi[0]) != 1 and combi[0][0] != "EV_EBITDA":
# continue
output.append(combi)
return jsonify({"result": output})
except Exception as e:
print("[Error]: {}".format(e))
return jsonify({"Error": e})
pass
request = {
"factor_list": factor_list,
"strategy_list": [1],
"window_list": [0],
"method_list": [1],
# "group_list": [1, 2, 3],
# "position_list": [6, 15, 60, 150, 300],
"group_list": [1],
"position_list": [6, 15, 60],
}
# 使用 multiprocessing 必須加上
if __name__ == "__main__":
freeze_support()
general = General()
cfg = Config()
cal = Calendar('TW')
fac = Factor(general.get_distinct_factor_list(factor_list))
api_server_IP = cfg.get_value('IP', 'api_server_IP')
request["factor_list"] = general.factor_list_to_string(
request["factor_list"])
response = requests.get(
"http://{}/task/get_task_combi".format(api_server_IP), params=request)
combination = json.loads(response.text)['result']
print(combination)
try:
class NodeMsgHandler:
def __init__(self):
self._cfg = Config()
self._dbmgr = DBMgr()
self._general = General()
# 基本參數 (伺服器設定值)
self._processes = os.cpu_count() # 伺服器CPU核心數
self._server_name = getpass.getuser() # 伺服器使用者名稱
self._IP = socket.gethostbyname(socket.gethostname())
self._host_IP = self._cfg.get_value('IP', 'host_IP')
self.client_ID = self._server_name + "_" + self._IP
self._mqtt_account = self._cfg.get_value('MQTT', 'account')
self._mqtt_password = self._cfg.get_value('MQTT', 'password')
def active_mqtt(self):
mqtt_client = self.client_ID + " ProcessTask" # 設定節點名稱
client = mqtt.Client(client_id=mqtt_client)
client.on_connect = self._on_connect
client.on_message = self._on_message
client.username_pw_set(self._mqtt_account, self._mqtt_password)
client.connect(self._host_IP, 1883)
# 開始連線 執行設定的動作和處理重新連線問題
client.loop_forever()
def _on_connect(self, client, userdata, flag, rc):
print("Connected with result code {}".format(str(rc)))
# 0: 連接成功
# 1: 協議版本錯誤
# 2: 無效的客戶端標示
# 3: 伺服器無法使用
# 4: 使用者帳號或密碼錯誤
# 5: 未經授權
# 將訂閱主題寫在 on_connect 中，當重新連線時將會重新訂閱
client.subscribe("Analysis/FactorAnalysisTask", qos=2)
client.subscribe("Analysis/StatusCheck", qos=2)
client.subscribe("Analysis/HealthCheck", qos=2)
def _on_message(self, client, userdata, msg):
if msg.topic == "Analysis/FactorAnalysisTask":
self._handle_factor_analysis_task(client, userdata, msg)
elif msg.topic == "Analysis/StatusCheck":
self._status_receive(client, userdata, msg)
elif msg.topic == "Analysis/HealthCheck":
self._health_receive(client, userdata, msg)
# (CallBack) 處理狀態確認訊息
# input: client : 發送訊息的節點ID
# userdata : 資料型態
# msg : 訊息內容
def _status_receive(self, client, userdata, msg):
print("Receive Status Check and Respond...")
self._publish_status_respond()
# (Publish) 回傳節點系統狀態
def _publish_status_respond(self):
mqtt_client = self.client_ID + " StatusRespond" # 設定節點名稱
# 轉換Json格式
payload = {
'node_name': self.client_ID,
'node_core_number': self._processes,
'node_cpu_status': psutil.cpu_percent()
}
payload = json.dumps(payload)
# 送出訊息
publish.single(qos=2,
keepalive=60,
payload=payload,
topic="Analysis/StatusRespond",
client_id=mqtt_client,
hostname=self._host_IP,
auth={'username': self._mqtt_account,
'password': self._mqtt_password}
)
print('%s publish status respond' % self.client_ID)
# (CallBack) 處理健康狀態確認訊息
# input: client : 發送訊息的節點ID
# userdata : 資料型態
# msg : 訊息內容
def _health_receive(self, client, userdata, msg):
print("Receive Health Check and Respond...")
self._publish_health_respond()
# (Publish) 回傳節點系統狀態
def _publish_health_respond(self):
mqtt_client = self.client_ID + " HealthResponse" # 設定節點名稱
# 轉換Json格式
payload = {
'node_name': self.client_ID,
'node_core_number': self._processes,
'node_cpu_status': psutil.cpu_percent(),
}
payload = json.dumps(payload)
# 送出訊息
publish.single(qos=2,
keepalive=60,
payload=payload,
topic="Analysis/HealthResponse",
client_id=mqtt_client,
hostname=self._host_IP,
auth={'username': self._mqtt_account,
'password': self._mqtt_password}
)
print('%s publish health request' % self.client_ID)
# (CallBack) 處理接收到之任務
# input: client : 發送訊息的節點ID
# userdata : 資料型態
# msg : 訊息內容
def _handle_factor_analysis_task(self, client, userdata, msg):
# MQTT CallBack參數取出
payload = self._phase_mqtt_msg(msg)
print("task_message = ", payload)
# 收到指派之任務
print('task_message["owner"] = ', payload["owner"])
if payload["owner"] == self.client_ID:
print("Processing...", str(payload['task_list']))
factor_analysis_task = FactorAnalysisTask(payload['task_list'])
task_list_detail = factor_analysis_task.task_list_detail
factor_list = factor_analysis_task.factor_list
self._run_factor_analysis(task_list_detail, factor_list)
time.sleep(10)
# 全部工作完成 更新節點狀態
self._change_node_status('0')
def _run_factor_analysis(self, task_list_detail, factor_list):
# 預載交易日&因子資料
cal = Calendar('TW')
get_factor_start = time.time()
fac = Factor(factor_list)
get_factor_end = time.time()
print("Get factor time: %f second" % (get_factor_end - get_factor_start))
for task_detail in task_list_detail:
try:
start = time.time()
strategy_config = {
'factor': task_detail['factor'],
'strategy': task_detail['strategy'],
'window': task_detail['window'],
'method': task_detail['method'],
'group': task_detail['group'],
'position': task_detail['position'],
}
factor_str = self._general.factor_to_string(task_detail['factor'])
path = self._cfg.get_value('path', 'path_to_portfolio_performance') + factor_str
file_name = "{}_{}_{}_{}_{}_{}".format(
factor_str,
task_detail['strategy'],
task_detail['window'],
task_detail['method'],
task_detail['group'],
task_detail['position']
)
file = pathlib.Path("{}/{}.csv".format(path, file_name))
if file.exists():
self._publish_factor_analysis_task_finish(task_detail['task_status_id'], 1)
else:
my_stra = MyAsset(strategy_config, cal, fac)
end = time.time()
print("Execution time: %f second" % (end - start))
# status: 0 - undo, 1 - success, 2 - error
self._publish_factor_analysis_task_finish(task_detail['task_status_id'], 1)
except Exception as e:
# status: 0 - undo, 1 - success, 2 - error
self._publish_factor_analysis_task_finish(task_detail['task_status_id'], 2)
print(e)
# (Publish) 回傳節點完成任務
# status
# 0 - undo
# 1 - success
# 2 - error
def _publish_factor_analysis_task_finish(self, task_status_id, status):
mqtt_client = self.client_ID + " StatusRespond" # 設定節點名稱
payload = {
'node_name': self.client_ID,
'task_status_id': task_status_id,
'status': status
}
payload = json.dumps(payload)
publish.single(
qos=2,
keepalive=60,
payload=payload,
topic="Analysis/FinishTask",
client_id=mqtt_client,
hostname=self._host_IP,
auth={
'username': self._mqtt_account,
'password': self._mqtt_password
}
)
def _change_node_status(self, health):
current_time = str(datetime.now())
sql = " UPDATE `node` \
SET `cpu_status`=%(cpu_status)s, `core_num`=%(core_num)s, \
`health`=%(health)s, `health_time`=%(health_time)s \
WHERE `name`=%(name)s"
args = {
"name": str(self.client_ID),
"cpu_status": str(psutil.cpu_percent()),
"core_num": str(self._processes),
"health": str(health),
"health_time": current_time
}
status, row, result = self._dbmgr.update(sql, args)
def _phase_mqtt_msg(self, msg):
node_msg = str(msg.payload, encoding="utf-8")
node_msg = json.loads(node_msg)
return node_msg
class FactorAnalysisHandler:
def __init__(self):
self._dbmgr = DBMgr()
self._general = General()
def add_task_to_db(self, request):
current_time = str(datetime.now())
sql = "INSERT INTO `task` \
VALUES ('0', %(factor_list)s, %(strategy_list)s, %(window_list)s, %(method_list)s, \
%(group_list)s, %(position_list)s, %(begin_time)s)"
args = {
"factor_list":
self._general.factor_list_to_string(request['factor_list']),
"strategy_list":
self._general.list_to_string(request['strategy_list']),
"window_list":
self._general.list_to_string(request['window_list']),
"method_list":
self._general.list_to_string(request['method_list']),
"group_list":
self._general.list_to_string(request['group_list']),
"position_list":
self._general.list_to_string(request['position_list']),
"begin_time":
str(current_time),
}
status, row, result = self._dbmgr.insert(sql, args)
task_id = self.get_last_task_id(current_time)
return task_id
def add_task_detail_to_db(self, task_id, combination):
task_result = self.get_task_by_id(task_id)
args = []
for task_list in combination:
factor = task_list[0]
strategy = task_list[1]
window = task_list[2]
method = task_list[3]
group = task_list[4]
position = task_list[5]
args.append({
"task_id": str(task_id),
"factor": self._general.list_to_string(factor),
"strategy": str(strategy),
"window": str(window),
"method": str(method),
"group": str(group),
"position": str(position),
"finish_time": str(''),
"owner": str(''),
"status": str('0'),
})
sql = " INSERT INTO `task_status` \
VALUES ('0', %(task_id)s, %(factor)s, %(strategy)s, %(window)s, %(method)s, \
%(group)s, %(position)s, %(finish_time)s, %(owner)s, %(status)s)"
status, row, result = self._dbmgr.insert(sql, args, multiple=True)
def check_unfinished_task(self):
sql = "SELECT `task_status_id`, `task_id` FROM `task_status` WHERE status=%(status)s"
args = {"status": str('0')}
status, row, result = self._dbmgr.query(sql, args)
task_list = []
# 表示無未完成任務
if len(result) == 0:
status = False
task_id = -1
# 有未完成任務
else:
status = True
task_id = result[0]['task_id']
for sub_task in result:
task_list.append(sub_task['task_status_id'])
return status, task_id, task_list
def check_exist_task(self, request):
sql = " SELECT `task_id` \
FROM `task` \
WHERE `factor_list` = %(factor_list)s AND \
`strategy_list` = %(strategy_list)s AND \
`window_list` = %(window_list)s AND \
`method_list` = %(method_list)s AND \
`group_list` = %(group_list)s AND \
`position_list` = %(position_list)s"
args = {
"factor_list":
self._general.factor_list_to_string(request['factor_list']),
"strategy_list":
self._general.list_to_string(request['strategy_list']),
"window_list":
self._general.list_to_string(request['window_list']),
"method_list":
self._general.list_to_string(request['method_list']),
"group_list":
self._general.list_to_string(request['group_list']),
"position_list":
self._general.list_to_string(request['position_list']),
}
status, row, data = self._dbmgr.query(sql, args, fetch='one')
if status and data and len(data) == 1:
return True, data['task_id']
else:
return False, -1
def get_request(self, task_id):
task_info = {}
plateau_info = {}
# 取回任務清單
sql = "SELECT * FROM `task` WHERE `task_id`=%(task_id)s"
args = {"task_id": str(task_id)}
status, row, task_info = self._dbmgr.query(sql, args, fetch='one')
# 該任務不存在
if row == 0:
print('[FactorAnalysisHandler] 該任務編號 {} 不存在！'.format(task_id))
return {
'task_id': task_id,
'factor_list': [],
'strategy_list': [],
'window_list': [],
'method_list': [],
'group_list': [],
'position_list': [],
}
else:
return {
'task_id':
task_id,
'factor_list':
self._general.factor_string_to_list(task_info['factor_list']),
'strategy_list':
self._general.string_to_list(task_info['strategy_list']),
'window_list':
self._general.string_to_list(task_info['window_list']),
'method_list':
self._general.string_to_list(task_info['method_list']),
'group_list':
self._general.string_to_list(task_info['group_list']),
'position_list':
self._general.string_to_list(task_info['position_list']),
}
def get_task_by_id(self, task_id):
sql = "SELECT * FROM `task` WHERE task_id = %(task_id)s"
args = {"task_id": str(task_id)}
status, row, result = self._dbmgr.query(sql, args, fetch='one')
return result
def get_last_task_id(self, curr_time):
sql = "SELECT * FROM `task` WHERE begin_time = %(begin_time)s"
args = {"begin_time": str(curr_time)}
status, row, result = self._dbmgr.query(sql, args, fetch='one')
return result['task_id']
def get_all_task_list_by_status(self, task_id, status=0):
sql = "SELECT `task_status_id` FROM `task_status` WHERE status=%(status)s and task_id=%(task_id)s"
args = {"status": str(status), "task_id": str(task_id)}
sql_status, row, result = self._dbmgr.query(sql, args)
task_list = []
for sub_task in result:
task_list.append(sub_task['task_status_id'])
return task_list
def get_all_task_list(self, task_id):
sql = "SELECT `task_status_id` FROM `task_status` WHERE task_id=%(task_id)s"
args = {"task_id": str(task_id)}
status, row, result = self._dbmgr.query(sql, args)
task_list = []
for sub_task in result:
task_list.append(sub_task['task_status_id'])
return task_list
def check_node_have_task(self, node):
# 檢查這個 node 目前是否有未完成的任務
sql = " SELECT * \
FROM `task_status` \
WHERE `owner` = %(node)s AND `status` = 0"
args = {'node': node}
status, row, result = self._dbmgr.query(sql, args)
if status and row == 0:
return False
else:
return True
def update_task_owner(self, owner, task_list):
for task in task_list:
sql = " UPDATE `task_status` \
SET `owner`=%(owner)s \
WHERE `task_status_id` = %(task_status_id)s"
args = {"owner": str(owner), "task_status_id": str(task)}
status, row, result = self._dbmgr.update(sql, args)
class MyAsset:
def __init__(self, strategy_config, cal, fac):
self._strategy_config = strategy_config
self._cal = cal
self._fac = fac
self._cfg = Config()
self._path = self._cfg.get_value('path',
'path_to_portfolio_performance')
self._general = General()
self._factor_str = self._general.factor_to_string(
self._strategy_config['factor'])
self._show_task_detail()
self._portfolio = Portfolio(self._strategy_config, self._cal,
self._fac)
self._write_portfolio_performance()
def _show_task_detail(self):
columns_list = [
'factor', 'strategy', 'window', 'method', 'group', 'position',
'n_season', 'start_equity', 'period'
]
df = pd.DataFrame().append(
{
'factor':
self._factor_str,
'strategy':
self._strategy_config['strategy'],
'window':
self._strategy_config['window'],
'method':
self._strategy_config['method'],
'group':
self._strategy_config['group'],
'position':
self._strategy_config['position'],
'n_season':
int(self._cfg.get_value('parameter', 'n_season')),
'start_equity':
int(self._cfg.get_value('parameter', 'start_equity')),
'period':
"from {} to {}".format(
self._cfg.get_value('parameter', 'start_date'),
self._cfg.get_value('parameter', 'end_date'))
},
ignore_index=True)[columns_list]
print(df.set_index('factor').T)
def _write_portfolio_performance(self):
portfolio_performance, portfolio_equity = self._portfolio.get_performance_data(
)
path = self._path + self._factor_str
# e.g. MOM_0_0_1_5 or MOM&GVI_1_0_1_5
file_name = "{}_{}_{}_{}_{}_{}".format(
self._factor_str, str(self._strategy_config['strategy']),
str(self._strategy_config['window']),
str(self._strategy_config['method']),
str(self._strategy_config['group']),
str(self._strategy_config['position']))
# 如果沒有這個因子的資料夾就新增一個
if not os.path.exists(path):
os.makedirs(path)
path = "{}/{}".format(path, file_name)
portfolio_performance.to_csv(path + '.csv', header=True)
portfolio_equity['date'] = portfolio_equity['date'].dt.strftime(
'%Y-%m-%d')
with open(path + '.json', 'w') as file:
json.dump(portfolio_equity.to_dict(), file)
print('[MyAsset]: completed writing portfolio performance files')
def __init__(self):
self._dbmgr = DBMgr()
self._general = General()
class FactorAnalysisTaskHandler:
def __init__(self, request):
self._request = request
self._task_list = []
self._task_id = -1
self._factor_analysis_handler = FactorAnalysisHandler()
self._node_handler = NodeHandler()
self._dbmgr = DBMgr()
self._general = General()
self._cfg = Config()
self._api_server_IP = self._cfg.get_value('IP', 'api_server_IP')
# 自動分配排程工作
def schedule_task(self):
# 抓出未完成未完成的任務組合
status, task_id, task_list = self._factor_analysis_handler.check_unfinished_task(
)
# 檢查此 request 是否與過去的任務相同
exist_status, exist_task_id = self._factor_analysis_handler.check_exist_task(
self._request)
# 有未完成的任務
if status:
self._task_id = task_id
self._task_list = task_list
print(
"[FactorAnalysisTaskHandler] 執行尚未完成任務，任務編號：{}；任務清單：{}".format(
task_id, task_list))
self._request = self._factor_analysis_handler.get_request(task_id)
# 如沒有未完成的任務，再檢查是否有已經完成之相同任務
elif exist_status:
self._task_id = exist_task_id
print("[FactorAnalysisTaskHandler] 資料庫內已經有完成之任務 編號：{}".format(
self._task_id))
# 需要創建不同組合工作
else:
payloads = {
'factor_list':
self._general.factor_list_to_string(
self._request['factor_list']),
'strategy_list':
self._request['strategy_list'],
'window_list':
self._request['window_list'],
'method_list':
self._request['method_list'],
'group_list':
self._request['group_list'],
'position_list':
self._request['position_list'],
}
response = requests.get("http://{}/task/get_task_combi".format(
self._api_server_IP),
params=payloads)
combination = json.loads(response.text)['result']
# 將 request 轉換成任務新增至 DB ，並取得 task_id
self._task_id = self._factor_analysis_handler.add_task_to_db(
self._request)
# 將所有任務的組合新增至 DB
self._factor_analysis_handler.add_task_detail_to_db(
self._task_id, combination)
self._task_list = self._factor_analysis_handler.get_all_task_list_by_status(
self._task_id)
task_id = self._task_id
task_list = self._task_list
if not exist_status or status:
# 直到全部任務完成才出去
while True:
self._distribute_to_node(task_id, task_list)
# 檢查是否仍有未完成任務
status, task_id, task_list = self._factor_analysis_handler.check_unfinished_task(
)
# 確認所有任務完成
if not status:
break
print("[FactorAnalysisTaskHandler] 仍有未完成任務 編號:{} ;任務細節編號:{}".
format(task_id, task_list))
self._request["isFinish"] = 1
self._request['task_id'] = self._task_id
self._task_list = self._factor_analysis_handler.get_all_task_list(
self._task_id)
self._request['task_list'] = self._task_list
return self._request
def _distribute_to_node(self, task_id, task_list):
self._node_handler.clean_all_node_data()
# 發送狀態確認並更新狀態
self._node_handler.check_node_status()
# 檢查所有節點狀態 是否死亡
if not self._node_handler.check_node_health():
self._node_handler.clean_all_node_data()
self._node_handler.check_node_status()
# 取得所有運算節點
nodes = self._node_handler.get_node_status()
if len(nodes) == 0:
print("[FactorAnalysisTaskHandler] 目前沒有存活的節點，但任務尚未發送完畢")
else:
batch_task_slice_list = self._node_handler.distribute_batch_task(
len(task_list), len(nodes))
for i, node in enumerate(nodes):
# 判斷節點是否已經有工作 避免重複發送
if self._factor_analysis_handler.check_node_have_task(
node['name']):
continue
# 判斷節點是否為正常無工作 且健康狀態發送新任務
elif node['health'] == 0:
batch_task_list = task_list[batch_task_slice_list[i][0]:
batch_task_slice_list[i][1]]
# MQTT 發送任務
print("node name: ", node['name'])
HostMsgHandler().publish_processing_task(
node['name'], task_id, batch_task_list)
# 更新任務持有者
self._factor_analysis_handler.update_task_owner(
node['name'], batch_task_list)
# 更新節點狀態為「執行中」
self._node_handler.update_node_status(node['name'], 1)
else:
continue
time.sleep(30)
# 0：無任務
# 1：執行中
# 2：節點已死亡
# 檢查節點是否還有在運算 (超過10分鐘未回應即判斷死亡)
while self._node_handler.check_node_health():
print("[FactorAnalysisTaskHandler] 檢查各運算節點")
self._node_handler.publish_node_health_check()
# 判斷所有節點是否已經完成（即為全部為 0 或 2）
if self._node_handler.check_all_node_finish():
print("[FactorAnalysisTaskHandler] 全部節點已經運算完成或死亡")
break
# 若仍有節點在運算中 間隔300秒(5分鐘)檢查一次
time.sleep(300)
print("[FactorAnalysisTaskHandler] 仍有節點在運算 持續檢查健康度")
def __init__(self, driver):
super().__init__(driver)
self.general = General()
self.get_title("Sign in with IMDb - IMDb")
def __init__(self, driver):
super().__init__(driver)
self.general = General()
self.get_title("Find - IMDb")
class Excel():
# Get path of test data
def get_test_data_path(self):
self.general = General()
self.config_data = ConfigData()
path = str(self.general.get_directory_path())
# full_path = path + self.config_data.TEST_DATA_PATH
full_path = path + self.get_os_based_excel_path()
return full_path
# Get sheet name
def get_sheet_name(self, sheet_name):
path = self.get_test_data_path()
workbook = xlrd.open_workbook(path)
sheet = workbook.sheet_by_name(sheet_name)
return sheet
# get number of rows
def get_row_count(self, sheet_name):
row_count = sheet_name.nrows
return row_count
# get number of column
def get_col_count(self, sheet_name):
col_count = sheet_name.ncols
return col_count
# get excel path based on operating system
def get_os_based_excel_path(self):
os_name = self.general.get_operating_system_name()
if os_name == "Windows":
excel_path = self.config_data.TEST_DATA_PATH_WINDOWS
elif os_name == "Darwin":
excel_path = self.config_data.TEST_DATA_PATH_MAC
elif os_name == "Linux":
excel_path = self.config_data.TEST_DATA_PATH_LINUX
else:
print("Support provided for MAc, Linux and Windows ")
return excel_path
# read test data from excel as a list of dictionaries
def read_test_data(self, sheet_name):
sheet = self.get_sheet_name(sheet_name)
row_count = self.get_row_count(sheet)
col_count = self.get_col_count(sheet)
first_row = [] # The row where we stock the name of the column
for col in range(col_count):
first_row.append(sheet.cell_value(0, col))
# transform the workbook to a list of dictionary
data = []
for row in range(1, row_count):
elm = {}
for col in range(col_count):
elm[first_row[col]] = sheet.cell_value(row, col)
data.append(elm)
# Splitting the values from keys
tc1 = []
for i in range(len(data)):
tc2 = []
data_dict = data[i]
for x in data_dict.values():
tc2.append(x)
tc1.append(tc2)
return tc1
def __init__(self, driver):
super().__init__(driver)
self.general = General()
self.get_title("IMDb Sign-In")
class TopRatedMoviesPage(BasePage):
"""Locator details"""
grdMovieTitle = (By.XPATH, "//tbody[@Class='lister-list']//td[@Class='titleColumn']")
cmbSortBy = (By.ID, "lister-sort-by-options")
txtSearch = (By.ID, "suggestion-search")
btnSearch = (By.XPATH, "//*[@id='suggestion-search-button']")
grdMovieRating = (By.XPATH, "// tbody[ @ Class = 'lister-list'] // td[ @ Class = 'ratingColumn imdbRating'] // strong")
btnDescendingOrder = (By.XPATH, "//span[contains(@Class, 'global-sprite lister-sort-reverse')]")
rbnAddWishList = (By.XPATH, "//tbody[@Class='lister-list']//td[@Class='watchlistColumn']//div[@class='wlb_ribbon']")
divUserProfile = (By.XPATH, "//*[@id='imdbHeader']//div[@class='ipc-button__text']//span")
rbnAddWishListDetail = (By.XPATH, "//tbody[@Class='lister-list']//td[@Class='watchlistColumn']//div[@class='wlb_ribbon']//div")
"""constructor of the TopRatedMoviesPage class"""
"""launch IMDB top rated movies page """
def __init__(self, driver):
super().__init__(driver)
self.driver.get(ConfigData.BASE_URL)
self.general = General()
"""Actions for TopRatedMoviesPage class"""
# Get top rated movie list
# params: None
def get_movie_list(self):
list_movie_details = []
top_rated_movie_chart = self.get_value_from_grid(self.grdMovieTitle)
for ele in top_rated_movie_chart:
list_movie_details.append(ele.text)
return list_movie_details
# Verify top rated movie list count
# params: movie list, expected count
def verify_movie_count(self, movie_list, expected_movie_list_count):
actual_movie_list_count = len(movie_list)
assert int(actual_movie_list_count) == int(expected_movie_list_count)
# Select value in drop down
# params: drop down value
def select_value_in_sort_by(self, dropdown_value):
self.do_send_keys(self.cmbSortBy, dropdown_value)
# Input value in Search text box
# params: search text value
def input_search_imbd_text(self, search_text):
self.do_send_keys(self.txtSearch, str(search_text))
# Input value in Search text box
# params: None
def click_search_button(self):
self.do_click(self.btnSearch)
# Get top rated movie rating list
# params: None
def get_movie_rating_list(self):
list_rating_details = []
top_rated_movie_ratings = self.get_value_from_grid(self.grdMovieRating)
for ele in top_rated_movie_ratings:
list_rating_details.append(ele.text)
return list_rating_details
# Verify top rated movie rating list
# params: rating list
def verify_movie_rating_list(self, rating_list, order_details):
expected_list = self.general.sort_list(rating_list, order_details)
if expected_list == rating_list:
assert True
else:
assert False
# Click on descending order icon
# params: None
def click_descending_order_icon(self):
self.do_click(self.btnDescendingOrder)
# Click on add to wish list icon
# params: None
def click_wish_list_icon(self):
self.do_click(self.rbnAddWishList)
# Verify user profile details
# params: Profile name
def verify_profile_name(self, profile_name):
self.wait_for_page_load(self.divUserProfile)
user_profile_detail = self.get_value_from_grid(self.divUserProfile)
profile_detail = user_profile_detail[1].text
self.general.verify_values(str(profile_detail), str(profile_name))
# Verify user watch list details
# params: Profile name
def verify_watch_list(self, watch_list_count):
self.wait_for_page_load(self.divUserProfile)
user_profile_detail = self.get_value_from_grid(self.divUserProfile)
watchlist_detail = user_profile_detail[0].text
self.general.verify_values(int(watchlist_detail), int(watch_list_count))
# Verify user watch list added in movie chart
# params : Watch list details list
def verify_watch_list_added_to_movie(self, actual_wishlist_title):
self.wait_for_page_load(self.rbnAddWishListDetail)
watchlist_title = self.get_attribute_value(self.rbnAddWishListDetail, "title")
self.general.verify_values(str(watchlist_title), str(actual_wishlist_title))
def __init__(self, driver):
super().__init__(driver)
self.driver.get(ConfigData.BASE_URL)
self.general = General()
from analysis.analysis import Analysis
from utils.general import General
general = General()
analysis = Analysis()
factor_list = [
# ['FCF_P'],
['EV_EBITDA'],
# ['P_B'],
# ['P_S'],
# ['MOM'],
# ['EPS'],
# ['ROIC'],
# ['FCF_OI'],
# ['EV_EBITDA', 'ROIC'],
# ['P_B', 'EPS'],
# ['FCF_P', 'MOM'],
# ['FCF_OI', 'P_S'],
]
# 權益曲線折線圖
def plot_equity_curve():
request = {
'factor_list': general.factor_list_to_string(factor_list),
'strategy_list': [1],
'window_list': [0],
'method_list': [5015, 5010, 5005, 5020, 5025],
# 'group_list': [1, 2, 3, 4, 5],
'group_list': [1],
class Analysis:
def __init__(self):
self._cfg = Config()
self._path = self._cfg.get_value('path', 'path_to_portfolio_performance')
self._api_server_IP = self._cfg.get_value('IP', 'api_server_IP')
self._start_equity = int(self._cfg.get_value('parameter', 'start_equity'))
self._start_date = self._cfg.get_value('parameter', 'start_date')
self._end_date = self._cfg.get_value('parameter', 'end_date')
self._general = General()
self._cal = Calendar('TW')
def plot_equity_curve(self, request):
perf_ind_inf_str = ""
equity_df, performance_df = self._read_request_files(request)
perf_ind, equity_max = self._compute_performance_ind(equity_df)
# 計算CAGR MDD MAR並製作顯示字串
for elm in perf_ind:
if elm != perf_ind[-1]:
perf_ind_inf_str = perf_ind_inf_str + "[{}] CAGR: {}, MDD: {}, MAR: {}\n".format(
elm[0], elm[1], elm[2], elm[3]
)
else:
perf_ind_inf_str = perf_ind_inf_str + "[{}] CAGR: {}, MDD: {}, MAR: {}".format(
elm[0], elm[1], elm[2], elm[3]
)
# 日期需設定為index
ax = equity_df.plot.line(figsize=(12, 8))
# ax = equity_df.plot.line(figsize=(12, 8), color=COLOR[0])
ax.set_xlabel('Date')
ax.set_ylabel('Equity')
# 加上千位數逗點
ax.get_yaxis().set_major_formatter(
FuncFormatter(lambda x, p: format(int(x), ','))
)
# 強調Y軸上某一個點
ax.axhline(10000000, color='black', linestyle='--')
# 顯示文字註解
plt.text(
# equity_df.index[200], equity_max/10*9,
equity_df.index[500], 8000000,
perf_ind_inf_str,
size=14, ha='left', va='top',
bbox=dict(
facecolor='gray', alpha=1.5, edgecolor='gray'
)
)
plt.show()
def plot_equity_curve_and_cap_util_rate(self, request):
perf_ind_inf_str = ""
equity_df, performance_df = self._read_request_files(request)
df = equity_df[[equity_df.columns[0]]].rename({equity_df.columns[0]: 0}, axis=1)
df.loc[:, :] = np.nan
position = performance_df['flow'].max()+1
for i in range(1, position):
df[i] = df[0]
for i, row in performance_df.iterrows():
df.loc[row['start']: row['end'], row['flow']] = 1
df['cap_util'] = df.iloc[:, :].sum(axis=1)
df['cap_util_rate'] = df['cap_util'].apply(lambda x: x/position*100)
df['cap_util_rate'].iloc[-1] = np.nan
equity_df = equity_df.join(df['cap_util_rate'])
perf_ind, equity_max = self._compute_performance_ind(equity_df.iloc[:, :-1])
# 計算CAGR MDD MAR並製作顯示字串
for elm in perf_ind:
if elm != perf_ind[-1]:
perf_ind_inf_str = perf_ind_inf_str + "[{}] CAGR: {}%, MDD: {}%\n".format(
elm[0], elm[1], elm[2]
)
else:
perf_ind_inf_str = perf_ind_inf_str + "[{}] CAGR: {}%, MDD: {}%".format(
elm[0], elm[1], elm[2]
)
perf_ind_inf_str = perf_ind_inf_str + "\nFull Use of Equity Rate: {}%".format(
100 - round(df.loc[df['cap_util'] != position].shape[0] / df.shape[0] * 100, 2)
)
# ax = equity_df.iloc[:, :-1].plot.line(figsize=(12, 8))
ax = equity_df.iloc[:, :-1].plot.line(figsize=(12, 8), color=[COLOR[1], COLOR[2]])
# 顯示文字註解
plt.text(
# equity_df.index[1], equity_df[equity_df.columns[0]].min(),
equity_df.index[0], 3580000,
perf_ind_inf_str,
size=13, ha='left', va='top',
bbox=dict(
facecolor='gray', alpha=1.5, edgecolor='gray'
)
)
ax2 = equity_df['cap_util_rate'].plot.line(color='grey', secondary_y=True)
# 強調Y軸上某一個點
ax.axhline(10000000, color='black', linestyle='--')
# 加上千位數逗點
ax.get_yaxis().set_major_formatter(
FuncFormatter(lambda x, p: format(int(x), ','))
)
ax.set_xlabel("Date")
ax.set_ylabel("Equity")
ax2.set_ylabel("Capital Utilization Rate[%]")
plt.ylim(-10, 110)
plt.show()
def plot_performance_heatmap(self, request):
heatmap_df = pd.DataFrame(columns=[6, 15, 60, 150, 300])
df = self.read_performance_file()
df = df.loc[
(df['factor'] == request['factor']) &
(df['strategy'] == request['strategy']) &
(df['window'] == request['window']) &
(df['method'] == request['method'])
]
for i in [1, 2, 3, 4, 5]:
cagr_list = []
for col in heatmap_df.columns:
cagr_list.append(
df[request['perf_ind']].loc[
(df['group']==i) & (df['position']==col)
].values[0]
)
heatmap_df.loc[i] = cagr_list
print(heatmap_df)
ax = sns.heatmap(heatmap_df, cmap="YlGnBu")
ax.set_xlabel("Position")
ax.set_ylabel("Group")
# ax.set_title("[{}] {}_{}_{}".format(
# request['perf_ind'], request['factor'],
# request['window'], request['method'])
# )
print("[{}] {}_{}_{}_{}".format(
request['perf_ind'], request['factor'],
request['strategy'], request['window'],
request['method']
)
)
plt.show()
def plot_profit_bar_chart(self, request):
equity_df, trade_df = self._read_request_files(request)
file_name = equity_df.columns[0]
trade_df['win'] = trade_df['return'].apply(lambda x: 1 if x > 0 else 0)
win_rate = round(trade_df['win'].sum() / trade_df.shape[0] * 100, 2)
lose_rate = round(100 - win_rate, 2)
trade_df['+0-20'] = trade_df['return'].apply(lambda x: 1 if x > 0 and x <= 20 else 0)
trade_df['+20-100'] = trade_df['return'].apply(lambda x: 1 if x > 20 and x <= 100 else 0)
trade_df['+100<'] = trade_df['return'].apply(lambda x: 1 if x > 100 else 0)
trade_df['-0-20'] = trade_df['return'].apply(lambda x: 1 if x <= 0 and x >= -20 else 0)
trade_df['-20>'] = trade_df['return'].apply(lambda x: 1 if x < -20 else 0)
positive_df = trade_df.copy()
positive_df['return'] = positive_df['return'].apply(lambda x: 0 if x < 0 else x)
negative_df = trade_df.copy()
negative_df['return'] = negative_df['return'].apply(lambda x: 0 if x >= 0 else x)
text_str = "{}\n--------------------------------\n100 <: {}\n20 ~ 100: {}\n0 ~ 20: {}\n--------------------------------\n0 ~ -20: {}\n-20 >: {}\n--------------------------------\nTotal Trades Number: {}\nWin Rate: {}\nLose Rate: {}".format(
file_name,
trade_df['+100<'].sum(), trade_df['+20-100'].sum(), trade_df['+0-20'].sum(),
trade_df['-0-20'].sum(), trade_df['-20>'].sum(),
len(trade_df.index), win_rate, lose_rate
)
fig = plt.figure(figsize=(80, 8))
sns.set_theme(style="whitegrid")
ax = sns.barplot(x="index", y="return", data=positive_df.reset_index(), color="navy")
ax = sns.barplot(x="index", y="return", data=negative_df.reset_index(), color="gray")
plt.legend()
plt.xlabel("Trade Series Number")
plt.ylabel("Return[%]")
plt.xticks(fontsize=2, rotation=90)
plt.yticks(fontsize=12)
# 顯示文字註解
plt.text(
trade_df.index[200], trade_df['return'].max(),
# trade_df.index[180], 90,
text_str,
size=14, ha='left', va='top',
bbox=dict(
facecolor='gray', alpha=1.5, edgecolor='gray'
)
)
plt.show()
def plot_window_profit_bar_chart(self, request):
equity_df, _ = self._read_request_files(request)
file_name = equity_df.columns[0]
report_date_list = self._cal.get_report_date_list(self._start_date, self._end_date)
window_return = []
peroid_list = []
# 窗格數量會比期間內財報公布日數量多一個
for i in range(len(report_date_list)+1):
# 第一個窗格T2: 回測期間第一個交易日~第一個財報公布日
if i == 0:
date = self._cal.get_trade_date(self._start_date, 0, 'd')
peroid_list.append(date)
strat_equity = equity_df.loc[date, file_name]
final_equity = equity_df.loc[report_date_list[i], file_name]
final_equity = final_equity.iloc[0]
# 最後一個窗格T2: 最後一個財報公布日~回測期間最後一個交易日
elif i == len(report_date_list):
date = self._cal.get_trade_date(self._end_date, -1, 'd')
peroid_list.append(report_date_list[i-1])
strat_equity = equity_df.loc[report_date_list[i-1], file_name]
final_equity = equity_df.loc[date, file_name]
strat_equity = strat_equity.iloc[0]
# 其他窗格T2: 兩公布日之間
else:
peroid_list.append(report_date_list[i-1])
strat_equity = equity_df.loc[report_date_list[i-1], file_name]
final_equity = equity_df.loc[report_date_list[i], file_name]
strat_equity = strat_equity.iloc[0]
final_equity = final_equity.iloc[0]
window_return.append(round((final_equity-strat_equity)/strat_equity*100, 2))
window_return_df = pd.DataFrame(window_return, columns=['return'])
window_return_df['window_peroid'] = peroid_list
total_return = (equity_df.iloc[-1, 0]-10000000) / 10000000 * 100
positive_df = window_return_df.copy()
positive_df['return'] = positive_df['return'].apply(lambda x: 0 if x < 0 else x)
negative_df = window_return_df.copy()
negative_df['return'] = negative_df['return'].apply(lambda x: 0 if x >= 0 else x)
result, _ = self._compute_performance_ind(equity_df)
text_str = "{}\nCAGR: {}%, MDD: {}%".format(
file_name, result[0][1], result[0][2]
)
fig = plt.figure(figsize=(12, 7))
sns.set_theme(style="whitegrid")
ax = sns.barplot(x="window_peroid", y="return", data=positive_df, color="cornflowerblue")
ax = sns.barplot(x="window_peroid", y="return", data=negative_df, color="gray")
plt.legend()
plt.xlabel("Start Date of Window", fontsize=8)
plt.ylabel("Return[%]")
plt.xticks(fontsize=8, rotation=35)
plt.yticks(fontsize=12)
# 顯示文字註解
plt.text(
17, window_return_df['return'].max()-2,
text_str,
size=13, ha='left', va='top',
bbox=dict(
facecolor='gray', alpha=1.5, edgecolor='gray'
)
)
plt.show()
def plot_linear_regression(self, request):
p_df = self.read_performance_file()
lr_df = self.read_regression_file()
factor_str_list = []
query_index = "{}_{}_{}_{}".format(
request['strategy'], request['window'],
request['method'], request['position'],
)
print(lr_df.loc[
(lr_df['perf_ind'] == request['perf_ind']) &
(lr_df['index'] == query_index)
])
lr_df = lr_df.loc[
(lr_df['perf_ind'] == request['perf_ind']) &
(lr_df['index'] == query_index)
].set_index('index').T.drop('perf_ind')
print(lr_df.sort_values(by=[query_index]))
path = self._cfg.get_value('path', 'path_to_performance_analysis')
lr_df.sort_values(by=[query_index]).to_csv(path+'lr_df.csv')
fig = plt.figure(figsize=(8, 7))
for i, factor in enumerate(request['factor_list']):
factor_str = self._general.factor_to_string(factor)
factor_str_list.append(factor_str)
temp_df = p_df.loc[
(p_df['factor'] == factor_str) &
(p_df['strategy'] == request['strategy']) &
(p_df['window'] == request['window']) &
(p_df['method'] == request['method']) &
(p_df['position'] == request['position'])
]
ax = sns.regplot(x=temp_df['group'], y=temp_df[request['perf_ind']], color=COLOR[i])
ax.set(xticks=range(1, len(temp_df['group'])+1, 1))
plt.legend(labels=factor_str_list)
plt.xlabel('Group', fontsize=12)
plt.ylabel(request['perf_ind'], fontsize=12)
plt.xticks(fontsize=12)
plt.yticks(fontsize=12)
plt.show()
def output_performance_file(self, request):
performance_df = pd.DataFrame(
columns=[
'factor', 'strategy', 'window', 'method', 'group',
'position', 'CAGR[%]', 'MDD[%]', 'MAR'
]
)
response = requests.get("http://{}/task/get_task_combi".format(self._api_server_IP), params=request)
combination = json.loads(response.text)['result']
for combi in combination:
equity_df, _ = self._read_file(combi)
ind_list, _ = self._compute_performance_ind(equity_df)
performance_df = performance_df.append({
'factor': self._general.factor_to_string(combi[0]),
'strategy': combi[1],
'window': combi[2],
'method': combi[3],
'group': combi[4],
'position': combi[5],
'CAGR[%]': ind_list[0][1],
'MDD[%]': ind_list[0][2],
'MAR': ind_list[0][3],
}, ignore_index=True)
print(performance_df)
path = self._cfg.get_value('path', 'path_to_performance_analysis')
performance_df.to_csv(path+'performance_indicator.csv')
# performance_df.to_csv(path+'performance_indicator_a&na.csv')
def output_linear_regression_file(self, request):
df = self.read_performance_file()
combination = []
lr_df = pd.DataFrame()
column_order = ['index', 'perf_ind']
data = (
request['strategy_list'], request['window_list'],
request['method_list'], request['position_list']
)
for combi in product(*data):
# strategy: B&H
if combi[0] == 0:
# B&H 排除0之外的窗格 排除等權重之外的方法
if combi[1] != 0 or combi[2] != 0:
continue
combination.append(combi)
for perf_ind in request['perf_ind_list']:
for c in combination:
factor_b1_val_dict = {}
factor_b1_val_dict["index"] = "{}_{}_{}_{}".format(
c[0], c[1], c[2], c[3]
)
for i, factor in enumerate(request['factor_list']):
factor_str = self._general.factor_to_string(factor)
# if len(column_order) != 14:
if len(column_order) != 3:
column_order.append(factor_str)
temp_df = df.loc[
(df['factor'] == factor_str) &
(df['strategy'] == c[0]) &
(df['window'] == c[1]) &
(df['method'] == c[2]) &
(df['position'] == c[3])
]
x = self._compute_simple_linear_regression(temp_df['group'], temp_df[perf_ind])
factor_b1_val_dict[factor_str] = round(x[1], 3)
factor_b1_val_dict['perf_ind'] = perf_ind
lr_df = lr_df.append(factor_b1_val_dict, ignore_index=True)
lr_df = lr_df[column_order]
print(lr_df)
path = self._cfg.get_value('path', 'path_to_performance_analysis')
lr_df.to_csv(path+'regression_b1.csv')
# lr_df.to_csv(path+'regression_b1_a&na.csv')
def read_performance_file(self):
path = self._cfg.get_value('path', 'path_to_performance_analysis')
df = pd.read_csv(path+'performance_indicator.csv').drop('Unnamed: 0', axis=1)
# df = pd.read_csv(path+'performance_indicator_a&na.csv').drop('Unnamed: 0', axis=1)
return df
def read_regression_file(self):
path = self._cfg.get_value('path', 'path_to_performance_analysis')
df = pd.read_csv(path+'regression_b1.csv').drop('Unnamed: 0', axis=1)
# df = pd.read_csv(path+'regression_b1_a&na.csv').drop('Unnamed: 0', axis=1)
return df
def _compute_simple_linear_regression(self, raw_x, raw_y):
n = np.size(raw_x)
x = np.array(raw_x)
y = np.array(raw_y)
x_mean = np.mean(x)
y_mean = np.mean(y)
num1 = np.sum(y*x) - n*y_mean*x_mean
num2 = np.sum(x*x) - n*x_mean*x_mean
b_1 = num1 / num2
b_0 = y_mean - b_1 * x_mean
return (b_0, b_1)
def _compute_performance_ind(self, df):
result = []
equity_max = 0
for col in df.columns:
equity_df = df[col]
# [CAGR]
start_date_list = self._start_date.split("-")
end_date_list = self._end_date.split("-")
final_equity = equity_df.iloc[-1]
if end_date_list[1] == "1" and end_date_list[2] == "1":
n = int(end_date_list[0]) - int(start_date_list[0])
else:
n = int(end_date_list[0]) - int(start_date_list[0]) + 1
cagr = ((final_equity / self._start_equity) ** (1/n) - 1) * 100
# [MDD]
highest_equity = equity_df.max()
MDD = 0
for i, elm in enumerate(equity_df.values.tolist()):
drawdown = elm - equity_df.iloc[:i].max()
if drawdown < MDD:
MDD = drawdown
MDD = MDD / highest_equity *100
MAR = cagr / abs(MDD)
result.append([col, round(cagr, 2), round(MDD, 2), round(MAR, 3)])
if equity_df.max() > equity_max:
equity_max = equity_df.max()
return result, equity_max
def _read_file(self, param):
# 將[fac1, fac2] 轉成 fac1&fac2
factor_str = self._general.factor_to_string(param[0])
path = "{}{}/".format(self._path, factor_str)
file_name = "{}_{}_{}_{}_{}_{}".format(
factor_str,
param[1], # strategy
param[2], # window
param[3], # method
param[4], # group
param[5], # position
)
performance_df = pd.read_csv("{}{}.csv".format(path, file_name)).drop('Unnamed: 0', axis=1)
with open(path+file_name+'.json', 'r') as file:
json_data = json.load(file)
equity_df = pd.DataFrame.from_dict(json_data)
equity_df['date']= pd.to_datetime(equity_df['date'])
equity_df = equity_df.set_index('date')
equity_df.columns = [file_name]
print("read {}".format(file_name))
return equity_df, performance_df
def _read_request_files(self, request):
equity_df = pd.DataFrame()
response = requests.get("http://{}/task/get_task_combi".format(self._api_server_IP), params=request)
combination = json.loads(response.text)['result']
# combination = [
# [['EV_EBITDA'], 0, 0, 0, 1, 6],
# [['EV_EBITDA'], 1, 0, 0, 1, 6],
# [['EV_EBITDA'], 1, 1, 0, 1, 6],
# [['EV_EBITDA'], 1, 2, 0, 1, 6],
# ]
# 讀檔並串接 equity
for param in combination:
temp_equity_df, performance_df = self._read_file(param)
if equity_df.empty:
equity_df = temp_equity_df
else:
equity_df = equity_df.join(temp_equity_df)
return equity_df, performance_df