def pull_data_and_count_kpi(start_date=None, start_code=None, retry=False, retryDict=None): shp = SharePuller() df = shp.query_from_mysql() now = datetime.datetime.now() end = now.strftime('%Y-%m-%d') dict = {} # 拉取数据&保存到DB kp = Kpuller() # 获取基本数据,按列表计算 for index, row in df.iterrows(): code = row['code'] start = str(row['timeToMarket']) if (start_code and code < start_code): continue if (retry and code not in retryDict['pd']): continue if (start_date != None and start < start_date): start = start_date dict[code] = kp.pull(code, start, end) time.sleep(1.5) KManager.to_csv(dict, 'pd')
def count_kpi(): shp = SharePuller(); df = shp.query_from_mysql(); for index, row in df.iterrows(): code = row['code']; nma=NetProfitMA(); nma.run(code,to_mysql=True,type=1); nms = NetProfitSort(); nms8= NetProfitSort8(); for dt in['2017-09-30','2017-12-31','2018-03-31','2018-06-30','2018-09-30']: nms.run(dt,to_mysql=True,type=1) nms8.run(dt,to_mysql=True,type=1)
def pull_data_hk(start_date=None, start_code=None, retry=False, retryDict=None): shp = SharePuller() df = shp.query_from_mysql() dict = {} hkp = HkHoldPuller() for dt in DateUtil.getDateSeq(start_date): if (retry and dt not in retryDict['ph']): continue dict[dt] = hkp.pull(dt) time.sleep(3) KManager.to_csv(dict, 'ph')
def count_kpi(start_date, s=True, m=True, start_code=None, retry=False, retryDict=None): shp = SharePuller() df = shp.query_from_mysql() end = datetime.datetime.now().strftime('%Y-%m-%d') # ================================ ## 单指标计算 # ================================ if (s): dict = {} for index, row in df.iterrows(): code = row['code'] start = str(row['timeToMarket']) if (start_code and code < start_code): continue if (retry and code not in retryDict['ks']): continue if (start_date != None and start < start_date): start = start_date KManager.kpi_s(code, start_date, pm, dict) KManager.to_csv(dict, 'ks') # ================================ ## 多指标计算 # ================================ if (m): dict = {} if (start_date == None): start_date = end dict[start_date] = KManager.kpi_m(start_date, pm) elif (start_date <= end): for dt in DateUtil.getDateSeq(start_date): if (retry and code not in retryDict['km']): continue dict[dt] = KManager.kpi_m(dt, pm) #更新数据 pm.execute( "update share_data_day set valid=1 where trade_date='%s'" % (dt)) KManager.to_csv(dict, 'km')
def job_every_month(): from k.manager.FinManager import FinManager from k.puller.SharePuller import SharePuller from k.manager.Kmanager import KManager from k.util.Logger import logger from k.util.DateUtil import DateUtil from k.util.PandasToMysql import pm if (isWeekDay()): return sp = SharePuller() sp.pull() FinManager.pull_data() FinManager.count_kpi()
def pull_data(): shp = SharePuller(); df = shp.query_from_mysql(); now = datetime.datetime.now() end = now.strftime('%Y-%m-%d'); for index, row in df.iterrows(): code = row['code']; start = str(row['timeToMarket']); #目前start和and 还未生效 fy = FinYearPuller(); fy.pull(code, start, end); fs = FinSeasonPuller(); fs.pull(code, start, end); fdp = FundsPuller(); fdp.pull(code, start, end)
def count_kpi(): shp = SharePuller(); df = shp.query_from_mysql(); for index, row in df.iterrows(): code = row['code']; nma=NetProfitMA(); nma.run(code,to_mysql=True,type=1); nms = NetProfitSort(); nms8= NetProfitSort8(); for dt in['2017-09-30','2017-12-31','2018-03-31','2018-06-30','2018-09-30']: nms.run(dt,to_mysql=True,type=1) nms8.run(dt,to_mysql=True,type=1) if(__name__=='__main__'): shp = SharePuller(); df = shp.query_from_mysql(); for index, row in df.iterrows(): code = row['code']; nms = NetProfitSort(); nms8 = NetProfitSort8(); for dt in [ '2018-09-30']: nms.run(dt, to_mysql=True, type=1) nms8.run(dt, to_mysql=True, type=1)
def main(argv): create_db = False pull_list = False pull_k_data = False pull_k_h_data = False pull_f_data = False pull_m_k_data = False kpi_k = False kpi_f = False retry = False start_date = datetime.datetime.now().strftime('%Y-%m-%d') start_code = None try: opts, args = getopt.getopt(argv, 'hcfkrs:', ['pl', 'pk', 'ph', 'pf', 'pmk', 'sc=']) except getopt.GetoptError: print_help() sys.exit(2) for opt, arg in opts: if opt == '-h': print_help() sys.exit() elif opt == '-c': create_db = True elif opt == '--pl': pull_list = True elif opt == '--pk': pull_k_data = True elif opt == '--ph': pull_k_h_data = True elif opt == '--pf': pull_f_data = True elif opt == '-k': kpi_k = True elif opt == '-f': kpi_f = True elif opt in ("-s", "--sdate"): start_date = arg elif opt == '-r': retry = True elif opt == '--sc': start_code = arg elif opt == '--pmk': pull_m_k_data = True from k.util.DbCreator import DbCreator from k.manager.Kmanager import KManager from k.manager.FinManager import FinManager from k.puller.SharePuller import SharePuller from GlobalConfig import ConfigDict if (retry): df = pd.DataFrame() if os.path.exists(ConfigDict['k_fail_log']): df = pd.read_csv(ConfigDict['k_fail_log'], index_col=0, dtype={ 'code': np.str, 'type': np.str }) print(df) if os.path.exists(ConfigDict['k_fail_log'] + '.pre'): os.remove(ConfigDict['k_fail_log'] + '.pre') os.rename(ConfigDict['k_fail_log'], ConfigDict['k_fail_log'] + '.pre') new_df = pd.DataFrame(data=None, columns=['type', 'code']) new_df.to_csv(ConfigDict['k_fail_log'], mode='w') for i in range(df.shape[0]): if (df.loc[i, 'type'] == 'pd'): K_RETRY['pd'].append(df.loc[i, 'code']) elif (df.loc[i, 'type'] == 'ph'): K_RETRY['ph'].append(df.loc[i, 'code']) elif (df.loc[i, 'type'] == 'ks'): K_RETRY['ks'].append(df.loc[i, 'code']) elif (df.loc[i, 'type'] == 'km'): K_RETRY['km'].append(df.loc[i, 'code']) print(K_RETRY) if (not os.path.exists(ConfigDict['k_fail_log'])): new_df = pd.DataFrame(data=None, columns=['date', 'type', 'code']) new_df.to_csv(ConfigDict['k_fail_log'], mode='w') if (create_db): dc = DbCreator() dc.init_create_table() if (pull_list): sp = SharePuller() sp.pull() if (pull_k_data): KManager.pull_data(start_date, start_code=start_code, retry=retry, retryDict=K_RETRY) KManager.pull_data_hk(start_date, start_code=start_code, retry=retry, retryDict=K_RETRY) if (pull_k_h_data): KManager.pull_data_hk(start_date, start_code=start_code, retry=retry, retryDict=K_RETRY) if (pull_m_k_data): KManager.pull_data_month_and_month_kpi(start_date, start_code=start_code, retry=retry, retryDict=K_RETRY) if (kpi_k): KManager.count_kpi(start_date, start_code=start_code, retry=retry, retryDict=K_RETRY) if (pull_f_data): FinManager.pull_data() if (kpi_f): FinManager.count_kpi() print('finish OK')