def get_db_stat(day_ws_time): try: num_signal_query = 'SELECT COUNT(*) FROM signals WHERE self_corr > 0 OR prod_corr > 0' num_combo_query = 'SELECT COUNT(*) FROM combo WHERE self_corr > 0 OR prod_corr > 0' db = mysql.connect(**config.config_db) cursor = db.cursor() cursor.execute(num_signal_query) num_signal = cursor.fetchall()[0][0] cursor.execute(num_combo_query) num_combo = cursor.fetchall()[0][0] diff_signal_query = 'SELECT COUNT(*) FROM signals WHERE created_at = \'{}\''.format( day_ws_time) cursor.execute(diff_signal_query) diff_signal = cursor.fetchall()[0][0] diff_combo_query = 'SELECT COUNT(*) FROM combo WHERE created_at = \'{}\''.format( day_ws_time) cursor.execute(diff_combo_query) diff_combo = cursor.fetchall()[0][0] db.close() return num_signal, num_combo, diff_signal, diff_combo except Exception as ex: trace_msg = traceback.format_exception(etype=type(ex), value=ex, tb=ex.__traceback__) utils.db_insert_log("STAT: ", str(trace_msg), "")
def re_check(sess, num=15): start = 0 try: db = mysql.connect(**config.config_db) cursor = db.cursor() cursor.execute(select_query) records = cursor.fetchall() for alpha_id in records: if start == num: break result, selfcorr, prodcorr = utils.check_submission( alpha_id[0], sess) if result == True and selfcorr > 0: print("RESULT: Pass : "******" : " + str(prodcorr)) cursor.execute( update_query.format(selfcorr, prodcorr, alpha_id[0])) db.commit() start = start + 1 print("Number of qualified alpha: {}/{}".format(start, num)) elif result == False: print("RESULT: Fail : " + str(selfcorr) + " : " + str(prodcorr)) cursor.execute(update_combo_query.format(alpha_id[0])) db.commit() utils.change_name(alpha_id[0], sess, name='FAILED') else: print("Timed-out or Exception") db.close() except Exception as ex: trace_msg = traceback.format_exception(etype=type(ex), value=ex, tb=ex.__traceback__) utils.db_insert_log("re_check", str(trace_msg), "")
def submit_alpha(alpha_id, sess): # Submit alpha, using alpha_id as input. # Return ... tried_time = 1 submit_url = 'https://api.worldquantvrc.com/alphas/{}/submit'.format( alpha_id) print("SUBMITTING Alpha ID: {}".format(alpha_id)) try: sess.post(submit_url) while tried_time < max_tried_time: response = sess.get(submit_url) # POST lan 1, cac lan sau GET. if utils.ERRORS(sess, response.text, "submit_alpha"): time.sleep(1) elif 'checks' in response.text: if 'FAIL' in response.text: return False, -1, -1, tried_time else: res_info_submitted = json.loads(response.content) checks = res_info_submitted["is"]["checks"] for check in checks: if 'SELF_CORRELATION' in str(check): selfcorr = check["value"] if 'PROD_CORRELATION' in str(check): prodcorr = check["value"] return True, selfcorr, prodcorr, tried_time time.sleep(0.5) tried_time = tried_time + 1 return False, -1, -1, tried_time except Exception as ex: trace_msg = traceback.format_exception(etype=type(ex), value=ex, tb=ex.__traceback__) utils.db_insert_log("submit", str(trace_msg), response.text) return False, -2, -2, tried_time
def update_actual_use_signals(alpha_id, alpha_code, settings): # Update actual use from existed submitted alpha try: update_signal_query = "UPDATE signals SET actual_use = actual_use + 1 WHERE alpha_id != \'\' and alpha_code = \'{}\' and region = \'{}\' and universe = \'{}\'" db = mysql.connect(**config.config_db) cursor = db.cursor() alpha_settings = ast.literal_eval(settings) region = alpha_settings["region"] universe = alpha_settings["universe"] signals = alpha_code.split(";") signals.pop(-1) signals.pop(-1) for signal in signals: signal_code = signal[4:] if signal_code[0] == "=": signal_code = signal_code[1:] print(signal_code) cursor.execute( update_signal_query.format(signal_code, region, universe)) db.commit() db.close() except Exception as ex: trace_msg = traceback.format_exception(etype=type(ex), value=ex, tb=ex.__traceback__) utils.db_insert_log("update_actual_use", str(trace_msg), "")
def update_actual_use_signals_submitted(): # Update actual use from existed submitted alpha try: select_query = "SELECT alpha_id, alpha_code, settings FROM submitted" update_signal_query = "UPDATE signals SET actual_use = actual_use + 1 WHERE alpha_id != \'\' and alpha_code = \'{}\' and region = \'{}\' and universe = \'{}\'" #select_signals_query = "SELECT alpha_id FROM signals WHERE alpha_code = \'{}\'" db = mysql.connect(**config.config_db) cursor = db.cursor() cursor.execute(select_query) records = cursor.fetchall() for alpha in records: alpha_code = alpha[1] settings = ast.literal_eval(alpha[2]) region = settings["region"] universe = settings["universe"] signals = alpha_code.split(";") signals.pop(-1) signals.pop(-1) for signal in signals: signal_code = signal[4:] if signal_code[0] == "=": signal_code = signal_code[1:] print(signal_code) cursor.execute(update_signal_query.format(signal_code, region, universe)) db.commit() # query = select_signals_query.format(signal_code) # print(query) # cursor.execute(query) # result = cursor.fetchall() # if len(result) != 0: # print(result[0][0]) db.close() except Exception as ex: trace_msg = traceback.format_exception(etype=type(ex), value=ex, tb=ex.__traceback__) utils.db_insert_log("update_actual_use", str(trace_msg),"")
def get_combo_data(num,data): try: rndData = [] for count in range(0, num): num_ope = int(random.choice(config.num_max_ope)) rndOpe = random.sample(operators.operators(), num_ope) if num_ope > 1: filled_data = "add(" for ope in rndOpe: if ope[2] == 0: filled_data = filled_data + ope[0].format(*random.sample(data,ope[1])) + ", " else: rndGroup = random.choice(group) filled_data = filled_data + ope[0].format(*random.sample(data,ope[1]), rndGroup) + ", " filter = random.choice(['true', 'false']) filled_data = filled_data + "filter = " + filter + ")" rndData.append(filled_data) else: if rndOpe[0][2] == 0: rndData.append(rndOpe[0][0].format(*random.sample(data,rndOpe[0][1]))) else: rndGroup = random.choice(group) rndData.append(rndOpe[0][0].format(*random.sample(data,rndOpe[0][1]), rndGroup)) return rndData except Exception as ex: trace_msg = traceback.format_exception(etype=type(ex), value=ex, tb=ex.__traceback__) utils.db_insert_log("get_combo_data", str(trace_msg), str(rndData)+str(rndOpe))
def re_simulate_signals(): try: websim_time = datetime.now(pytz.timezone('EST5EDT')) day_ws_time = str(websim_time).split(" ")[0] num_days = int(day_ws_time.split("-")[-1]) select_query = "SELECT alpha_id, alpha_code, universe, region FROM signals WHERE datediff(\'{}\',updated_at) > 30".format( num_days) db = mysql.connect(**config.config_db) cursor = db.cursor() cursor.execute( "SELECT count(*) FROM signals WHERE WHERE datediff(\'{}\',updated_at) > 30" .format(num_days)) num_alpha = cursor.fetchall()[0][0] cursor.execute(select_query) results = cursor.fetchall() count = 0 for result in results: count = count + 1 print("ALPHA: {}/{}".format(count, num_alpha)) alpha_id = simulator.simulate_alpha(sess, result[1], result[2], result[3], 1) if alpha_id != None: alpha_info = utils.get_alpha_info(alpha_id, sess) db_update_signals(result[0], alpha_info) except Exception as ex: trace_msg = traceback.format_exception(etype=type(ex), value=ex, tb=ex.__traceback__) utils.db_insert_log("re-simulate", str(trace_msg), "")
def signal_simulate(thread_num): while True: try: #alpha_codes = signal_generator.get_alphas(data) #alpha_ids = simulator.multi_simulate(sess, alpha_codes, top, region, thread_num) # alpha_ids = simulator.simulate_alpha(sess, alpha_codes[0], top, region, thread_num) # if alpha_ids != None: # print("ALPHA_IDS :" + str(alpha_ids)) # for alpha_id in alpha_ids: # results = utils.get_alpha_info(alpha_id, sess) # if results["weight_test"] == 'FAIL': # print("Thread {}: Alpha {}: Not enough performance".format(thread_num, alpha_id)) # break # elif results["sharpe"] >= config.min_signal[0] and results["fitness"] >= config.min_signal[1]: # selfcorr = float(utils.check_selfcorr(alpha_id, sess)) # if selfcorr <= config.min_signal[2]: # prodcorr = float(utils.check_prodcorr(alpha_id, sess)) # if prodcorr <= config.min_signal[2]: # results["self_corr"]=selfcorr # results["prod_corr"]=prodcorr # results["theme"] = theme # utils.db_insert_signals(results) # utils.change_name(alpha_id, sess, "signal") # else: # print("Thread {}: Alpha {}: Not enough performance".format(thread_num, alpha_id)) alpha_code = signal_generator.get_alphas(data) alpha_id = simulator.simulate_alpha(sess, alpha_code, top, region, thread_num) if alpha_id != None: results = utils.get_alpha_info(alpha_id, sess) if results["weight_test"] == 'FAIL': print("Thread {}: Alpha {}: Not enough performance".format( thread_num, alpha_id)) elif results["sharpe"] >= config.min_signal[0] and results[ "fitness"] >= config.min_signal[1]: selfcorr = float(utils.check_selfcorr(alpha_id, sess)) if selfcorr <= config.min_signal[2]: prodcorr = float(utils.check_prodcorr(alpha_id, sess)) if prodcorr <= config.min_signal[2]: results["self_corr"] = selfcorr results["prod_corr"] = prodcorr results["theme"] = theme utils.db_insert_signals(results) utils.change_name(alpha_id, sess, "signal") else: print("Thread {}: Alpha {}: Not enough performance".format( thread_num, alpha_id)) except Exception as ex: trace_msg = traceback.format_exception(etype=type(ex), value=ex, tb=ex.__traceback__) if 'alpha_ids' in locals() or 'alpha_ids' in globals(): #utils.db_insert_log("signal_simulate", str(trace_msg), str(alpha_ids)+" - "+ str(type(alpha_ids)) + str(datetime.now())) utils.db_insert_log( "signal_simulate", str(trace_msg), str(alpha_id) + " - " + str(type(alpha_id)) + str(datetime.now())) else: utils.db_insert_log("signal_simulate", str(trace_msg), "")
def get_ann(sess): try: response = sess.get(ann_url) messages = json.loads(response.content)["results"] return messages except Exception as ex: trace_msg = traceback.format_exception(etype=type(ex), value=ex, tb=ex.__traceback__) utils.db_insert_log("get_ann", str(trace_msg), "")
def update_count_used(alpha_id): # Update count value for each signal in alpha_ids. try: db = mysql.connect(**config.config_db) cursor = db.cursor() last_used = str(datetime.now()).split(" ")[0] query = "UPDATE signals SET count_used = count_used + 1, last_used = \'{}\' WHERE alpha_id = \'{}\'".format(last_used, alpha_id) cursor.execute(query) db.commit() db.close() except Exception as ex: trace_msg = traceback.format_exception(etype=type(ex), value=ex, tb=ex.__traceback__) utils.db_insert_log("update_count_used", str(trace_msg), str(query))
def get_summary(sess): try: while True: response = sess.get(summary_url) if "is" in response.text: res_summary = json.loads(response.content) is_sum = res_summary["is"] os_sum = res_summary["os"] prod_sum = res_summary["prod"] return is_sum, os_sum, prod_sum except Exception as ex: trace_msg = traceback.format_exception(etype=type(ex), value=ex, tb=ex.__traceback__) utils.db_insert_log("get_summary", str(trace_msg), "")
def get_set_signals(top, region): # Return a dictionary contains alpha_id and alpha_code of signals. # For prepare combination step. try: db = mysql.connect(**config.config_db) cursor = db.cursor() query = "SELECT alpha_id, alpha_code FROM signals WHERE count_used <= {} AND universe = \'{}\'".format(config.max_signal_count, top) cursor.execute(query) records = cursor.fetchall() diction = dict((x,y) for x,y in records) db.close() return diction except Exception as ex: trace_msg = traceback.format_exception(etype=type(ex), value=ex, tb=ex.__traceback__) utils.db_insert_log("get_set_signals", str(trace_msg), str(query))
def num_alpha_submitted(day, sess): try: os_url = 'https://api.worldquantvrc.com/users/self/alphas?limit=15&offset=0&stage=OS%1fPROD&order=-dateSubmitted&hidden=false' response = sess.get(os_url) res_os = json.loads(response.content)["results"] num_alpha_submitted = 0 for alpha in res_os: if day in alpha["dateSubmitted"]: num_alpha_submitted = num_alpha_submitted + 1 return num_alpha_submitted except Exception as ex: trace_msg = traceback.format_exception(etype=type(ex), value=ex, tb=ex.__traceback__) utils.db_insert_log("num_alpha_submitted", str(trace_msg), response.text)
def get_failed_status(sess): try: select_failed_combo_query = "SELECT COUNT(*) FROM combo WHERE self_corr < 0 or prod_corr < 0 and flag > 0" select_failed_signal_query = "SELECT COUNT(*) FROM signals WHERE self_corr < 0 or prod_corr < 0" db = mysql.connect(**config.config_db) cursor = db.cursor() cursor.execute(select_failed_combo_query) num_failed_combo = cursor.fetchall()[0][0] cursor.execute(select_failed_signal_query) num_failed_signal = cursor.fetchall()[0][0] db.close() return num_failed_combo, num_failed_signal except Exception as ex: trace_msg = traceback.format_exception(etype=type(ex), value=ex, tb=ex.__traceback__) utils.db_insert_log("FAILED ALPHAS: ", str(trace_msg), "")
def generate_combo(signals, num_signal, top, region): try: rnd_signals = random.sample(signals.items(), num_signal) # Get a list contains alpha_id and alpha_code of (num_signal) signal. combo_signal= "" combo = {"alpha_code":"", "region":region, "top":top} # Get a dictionary contains alpha_code, region and top. index = 0 list_alpha_ids = [] # Get list of signal's alpha_id to increase count if combo is qualified. combo_code = "" for signal in rnd_signals: combo_signal = combo_signal + "sn" + str(index) + "=" + signal[1] + "; " combo_code = combo_code + "sn" + str(index) + ", " list_alpha_ids.append(signal[0]) index = index + 1 rndCombo = random.choice(config.combo_template).format(combo_signal, combo_code) combo["alpha_code"]=rndCombo return combo, list_alpha_ids except Exception as ex: trace_msg = traceback.format_exception(etype=type(ex), value=ex, tb=ex.__traceback__) utils.db_insert_log("generate_combo", str(trace_msg), combo_code)
def remove_space_alpha(table): # Remove space in signals, combo, and submitted table try: select_query = "SELECT alpha_id, alpha_code FROM {}".format(table) update_query = "UPDATE " + str(table) + " SET alpha_code = \'{}\' WHERE alpha_id = \'{}\'" db = mysql.connect(**config.config_db) cursor = db.cursor() cursor.execute(select_query) records = cursor.fetchall() for record in records: alpha_id = record[0] alpha_code = record[1].replace(" ","") print("ALPHA_CODE: {} | {}".format(alpha_id, alpha_code)) cursor.execute(update_query.format(alpha_code, alpha_id)) db.commit() db.close() except Exception as ex: trace_msg = traceback.format_exception(etype=type(ex), value=ex, tb=ex.__traceback__) utils.db_insert_log("remove_space", str(trace_msg),"")
def get_system_info(day): try: db = mysql.connect(**config.config_db) cursor = db.cursor() cursor.execute("SELECT COUNT(*) FROM log") total_log_count = cursor.fetchall()[0][0] cursor.execute( "SELECT COUNT(*) FROM log WHERE logged_time LIKE \'%{}%\'".format( day)) today_log_count = cursor.fetchall()[0][0] cursor.execute( "SELECT COUNT(*) FROM login_log WHERE logged_time LIKE \'%{}%\'". format(day)) today_login_count = cursor.fetchall()[0][0] return total_log_count, today_log_count, today_login_count except Exception as ex: trace_msg = traceback.format_exception(etype=type(ex), value=ex, tb=ex.__traceback__) utils.db_insert_log("SYS STAT: ", str(trace_msg), "")
def get_payout(sess): try: while True: response = sess.get( 'https://api.worldquantvrc.com/users/self/activities/base-payment' ) payout_res = json.loads(response.content) if "yesterday" in response.text: yesterday = payout_res["yesterday"]["value"] this_month = payout_res["current"]["value"] total = payout_res["total"]["value"] return yesterday, this_month, total except Exception as ex: trace_msg = traceback.format_exception(etype=type(ex), value=ex, tb=ex.__traceback__) if response.text: utils.db_insert_log("get_payout", str(trace_msg), response.text) else: utils.db_insert_log("get_payout", str(trace_msg), "")
def get_alphas(data): try: #alphas = [] rndTemplate = random.choice(config.signal_template) rndData = get_combo_data(rndTemplate[1], data) # Number of data rndGroup = random.choice(neutral_group) if rndTemplate[2] == 0: # for rndGroupNeutralize in group: # alphas.append(rndTemplate[0].format(*rndData, rndGroupNeutralize)) alpha = rndTemplate[0].format(*rndData, rndGroup) elif rndTemplate[2] == 1: # for rndGroupNeutralize in group: # alphas.append(rndTemplate[0].format(*rndData, rndGroup, rndGroupNeutralize)) rndGroup_2 = random.choice(group) alpha = rndTemplate[0].format(*rndData, rndGroup_2, rndGroup) #return alphas return alpha except Exception as ex: trace_msg = traceback.format_exception(etype=type(ex), value=ex, tb=ex.__traceback__) #utils.db_insert_log("get_alphas", str(trace_msg), str(alphas)) utils.db_insert_log("get_alphas", str(trace_msg), str(alpha))
def combo_simulate(thread_num): while True: try: list_signal = combo_generator.get_set_signals(top, region) combo_alpha, list_alpha_ids = combo_generator.generate_combo( list_signal, num_signals, top, region) alpha_id = simulator.simulate_alpha(sess, combo_alpha["alpha_code"], top, region, thread_num) utils.change_name(alpha_id, sess, name="potential") if alpha_id != None: alpha_info = utils.get_alpha_info(alpha_id, sess) if alpha_info["sharpe"] >= config.min_combo[0] and alpha_info[ "fitness"] >= config.min_combo[1]: result, selfcorr, prodcorr = utils.check_submission( alpha_id, sess) #print(result) if result == True and max(selfcorr, prodcorr) <= config.min_combo[2]: alpha_info["self_corr"] = selfcorr alpha_info["prod_corr"] = prodcorr alpha_info["theme"] = utils.set_theme( alpha_info["alpha_code"], alpha_info["region"], data_theme) utils.change_name(alpha_id, sess, "can_submit") utils.db_insert_combo(alpha_info) for signal_id in list_alpha_ids: combo_generator.update_count_used(signal_id) else: print("Thread {}: Alpha {}: Not enough performance".format( thread_num, alpha_id)) else: print("Time-out") except Exception as ex: trace_msg = traceback.format_exception(etype=type(ex), value=ex, tb=ex.__traceback__) utils.db_insert_log("combo_simulate", str(trace_msg), str(combo_alpha["alpha_code"]))
def get_payout_all(sess): try: update_payout_query = "UPDATE submitted SET payout = {} WHERE submitted_at = \'{}\' AND alpha_id != \'\'" response = sess.get( 'https://api.worldquantvrc.com/users/self/activities/base-payment') payout_res = json.loads(response.content) records = payout_res["records"]["records"] db = mysql.connect(**config.config_db) cursor = db.cursor() for record in records: print("UPDATE: DAY: {} - PAYOUT: {}".format(record[0], record[1])) cursor.execute(update_payout_query.format(record[1], record[0])) db.commit() db.close() except Exception as ex: trace_msg = traceback.format_exception(etype=type(ex), value=ex, tb=ex.__traceback__) if response.text: utils.db_insert_log("get_payout", str(trace_msg), response.text) else: utils.db_insert_log("get_payout", str(trace_msg), "")
def theme_check(alpha_id, alpha_code, settings, data, query): # Checking if any combo is satisfied current multi-theme try: # if 'ASI' in settings: # region = 'ASI' # elif 'EUR' in settings: # region = 'EUR' # else: # region = 'USA' multiplier = utils.set_theme(alpha_code, "ASI", data) if multiplier > 0: print(str(alpha_id) + ": THEME x{}".format(multiplier)) else: print(str(alpha_id) + ": NOT THEME") db = mysql.connect(**config.config_db) cursor = db.cursor() cursor.execute(query.format(multiplier, alpha_id)) #cursor.execute(update_query.format(multiplier, alpha_id)) #cursor.execute(update_sub_query.format(multiplier, alpha_id)) db.commit() db.close() except Exception as ex: trace_msg = traceback.format_exception(etype=type(ex), value=ex, tb=ex.__traceback__) utils.db_insert_log("theme_check", str(trace_msg),"")
def auto_submit(mode, num_today, sess): try: max_num_alpha = 5 num_alpha = num_today select_query = 'SELECT alpha_id, alpha_code, settings FROM combo WHERE self_corr > 0 AND prod_corr > 0 ORDER BY {} DESC LIMIT {}' if mode == "1": while num_alpha < max_num_alpha: alpha_id = str(input("Alpha ID: ")) result, selfcorr, prodcorr, tried_time = submit_alpha( alpha_id, sess) if result == True and tried_time < max_tried_time: num_alpha = num_alpha + 1 db_move_combo(alpha_id, selfcorr, prodcorr, sess) utils.change_name(alpha_id, sess, name='submitted') alpha_info = utils.get_alpha_info(alpha_id, sess) alpha_code = alpha_info["alpha_code"] alpha_settings = alpha_info["settings"] update_actual_use_signals(alpha_id, alpha_code, alpha_settings) print("Alpha {} submitted successfully. ({}/5)".format( alpha_id, num_alpha)) elif result == False and tried_time < max_tried_time: #db_delete_combo(alpha_id) db_update_combo(alpha_id) print("Can not submit alpha {}.".format(alpha_id)) elif result == False and tried_time == max_tried_time: print("Time-out") else: print("There was an exception!!!") elif mode == "2": db = mysql.connect(**config.config_db) cursor = db.cursor() cursor.execute(select_query.format(config.combo_criteria, 50)) records = cursor.fetchall() db.close() for alpha_id in records: if num_alpha == 5: break result, selfcorr, prodcorr, tried_time = submit_alpha( alpha_id[0], sess) if result == True and tried_time < max_tried_time: num_alpha = num_alpha + 1 db_move_combo(alpha_id[0], selfcorr, prodcorr, sess) utils.change_name(alpha_id[0], sess, name='submitted') update_actual_use_signals(alpha_id[0], alpha_id[1], alpha_id[2]) print("Alpha {} submitted successfully. ({}/5)".format( alpha_id[0], num_alpha)) elif result == False and tried_time < max_tried_time: db_update_combo(alpha_id[0]) print("Can not submit alpha {}.".format(alpha_id[0])) elif result == False and tried_time == max_tried_time: print("Time-out") else: print("There was an exception!!!") elif mode == "3": print("WARNING: Pause the combo simulation!") # Recheck 15 alphas >> Submit 1 >> Increase index >> Loop. # If an alpha get stucks, cannot submit and show "Time-out" forever, the function will be stucked. But I'm too lazy for it, so well :v. while num_alpha < 5: stuff.re_check(sess, 10) db = mysql.connect(**config.config_db) cursor = db.cursor() cursor.execute(select_query.format( config.combo_criteria, 1)) # Choose one alpha to submit alpha_id = cursor.fetchall() db.close() result, selfcorr, prodcorr, tried_time = submit_alpha( alpha_id[0][0], sess) if result == True and tried_time < max_tried_time: num_alpha = num_alpha + 1 db_move_combo(alpha_id[0][0], selfcorr, prodcorr, sess) utils.change_name(alpha_id[0][0], sess, name='submitted') update_actual_use_signals(alpha_id[0][0], alpha_id[0][1], alpha_id[0][2]) print("Alpha {} submitted successfully. ({}/5)".format( alpha_id[0][0], num_alpha)) elif result == False and tried_time < max_tried_time: db_update_combo(alpha_id[0][0]) print("Can not submit alpha {}.".format(alpha_id[0][0])) elif result == False and tried_time == max_tried_time: print("Time-out") else: print("There was an exception!!!") except Exception as ex: trace_msg = traceback.format_exception(etype=type(ex), value=ex, tb=ex.__traceback__) utils.db_insert_log("auto_submit", str(trace_msg), "")