def pull_data_and_count_kpi(start_date=None,
start_code=None,
retry=False,
retryDict=None):
shp = SharePuller()
df = shp.query_from_mysql()
now = datetime.datetime.now()
end = now.strftime('%Y-%m-%d')
dict = {}
# 拉取数据&保存到DB
kp = Kpuller()
# 获取基本数据，按列表计算
for index, row in df.iterrows():
code = row['code']
start = str(row['timeToMarket'])
if (start_code and code < start_code):
continue
if (retry and code not in retryDict['pd']):
continue
if (start_date != None and start < start_date):
start = start_date
dict[code] = kp.pull(code, start, end)
time.sleep(1.5)
KManager.to_csv(dict, 'pd')
def count_kpi():
shp = SharePuller();
df = shp.query_from_mysql();
for index, row in df.iterrows():
code = row['code'];
nma=NetProfitMA();
nma.run(code,to_mysql=True,type=1);
nms = NetProfitSort();
nms8= NetProfitSort8();
for dt in['2017-09-30','2017-12-31','2018-03-31','2018-06-30','2018-09-30']:
nms.run(dt,to_mysql=True,type=1)
nms8.run(dt,to_mysql=True,type=1)
def pull_data_hk(start_date=None,
start_code=None,
retry=False,
retryDict=None):
shp = SharePuller()
df = shp.query_from_mysql()
dict = {}
hkp = HkHoldPuller()
for dt in DateUtil.getDateSeq(start_date):
if (retry and dt not in retryDict['ph']):
continue
dict[dt] = hkp.pull(dt)
time.sleep(3)
KManager.to_csv(dict, 'ph')
def count_kpi(start_date,
s=True,
m=True,
start_code=None,
retry=False,
retryDict=None):
shp = SharePuller()
df = shp.query_from_mysql()
end = datetime.datetime.now().strftime('%Y-%m-%d')
# ================================
## 单指标计算
# ================================
if (s):
dict = {}
for index, row in df.iterrows():
code = row['code']
start = str(row['timeToMarket'])
if (start_code and code < start_code):
continue
if (retry and code not in retryDict['ks']):
continue
if (start_date != None and start < start_date):
start = start_date
KManager.kpi_s(code, start_date, pm, dict)
KManager.to_csv(dict, 'ks')
# ================================
## 多指标计算
# ================================
if (m):
dict = {}
if (start_date == None):
start_date = end
dict[start_date] = KManager.kpi_m(start_date, pm)
elif (start_date <= end):
for dt in DateUtil.getDateSeq(start_date):
if (retry and code not in retryDict['km']):
continue
dict[dt] = KManager.kpi_m(dt, pm)
#更新数据
pm.execute(
"update share_data_day set valid=1 where trade_date='%s'"
% (dt))
KManager.to_csv(dict, 'km')
def job_every_month():
from k.manager.FinManager import FinManager
from k.puller.SharePuller import SharePuller
from k.manager.Kmanager import KManager
from k.util.Logger import logger
from k.util.DateUtil import DateUtil
from k.util.PandasToMysql import pm
if (isWeekDay()):
return
sp = SharePuller()
sp.pull()
FinManager.pull_data()
FinManager.count_kpi()
def pull_data():
shp = SharePuller();
df = shp.query_from_mysql();
now = datetime.datetime.now()
end = now.strftime('%Y-%m-%d');
for index, row in df.iterrows():
code = row['code'];
start = str(row['timeToMarket']);
#目前start和and 还未生效
fy = FinYearPuller();
fy.pull(code, start, end);
fs = FinSeasonPuller();
fs.pull(code, start, end);
fdp = FundsPuller();
fdp.pull(code, start, end)
def count_kpi():
shp = SharePuller();
df = shp.query_from_mysql();
for index, row in df.iterrows():
code = row['code'];
nma=NetProfitMA();
nma.run(code,to_mysql=True,type=1);
nms = NetProfitSort();
nms8= NetProfitSort8();
for dt in['2017-09-30','2017-12-31','2018-03-31','2018-06-30','2018-09-30']:
nms.run(dt,to_mysql=True,type=1)
nms8.run(dt,to_mysql=True,type=1)
if(__name__=='__main__'):
shp = SharePuller();
df = shp.query_from_mysql();
for index, row in df.iterrows():
code = row['code'];
nms = NetProfitSort();
nms8 = NetProfitSort8();
for dt in [ '2018-09-30']:
nms.run(dt, to_mysql=True, type=1)
nms8.run(dt, to_mysql=True, type=1)
def main(argv):
create_db = False
pull_list = False
pull_k_data = False
pull_k_h_data = False
pull_f_data = False
pull_m_k_data = False
kpi_k = False
kpi_f = False
retry = False
start_date = datetime.datetime.now().strftime('%Y-%m-%d')
start_code = None
try:
opts, args = getopt.getopt(argv, 'hcfkrs:',
['pl', 'pk', 'ph', 'pf', 'pmk', 'sc='])
except getopt.GetoptError:
print_help()
sys.exit(2)
for opt, arg in opts:
if opt == '-h':
print_help()
sys.exit()
elif opt == '-c':
create_db = True
elif opt == '--pl':
pull_list = True
elif opt == '--pk':
pull_k_data = True
elif opt == '--ph':
pull_k_h_data = True
elif opt == '--pf':
pull_f_data = True
elif opt == '-k':
kpi_k = True
elif opt == '-f':
kpi_f = True
elif opt in ("-s", "--sdate"):
start_date = arg
elif opt == '-r':
retry = True
elif opt == '--sc':
start_code = arg
elif opt == '--pmk':
pull_m_k_data = True
from k.util.DbCreator import DbCreator
from k.manager.Kmanager import KManager
from k.manager.FinManager import FinManager
from k.puller.SharePuller import SharePuller
from GlobalConfig import ConfigDict
if (retry):
df = pd.DataFrame()
if os.path.exists(ConfigDict['k_fail_log']):
df = pd.read_csv(ConfigDict['k_fail_log'],
index_col=0,
dtype={
'code': np.str,
'type': np.str
})
print(df)
if os.path.exists(ConfigDict['k_fail_log'] + '.pre'):
os.remove(ConfigDict['k_fail_log'] + '.pre')
os.rename(ConfigDict['k_fail_log'],
ConfigDict['k_fail_log'] + '.pre')
new_df = pd.DataFrame(data=None, columns=['type', 'code'])
new_df.to_csv(ConfigDict['k_fail_log'], mode='w')
for i in range(df.shape[0]):
if (df.loc[i, 'type'] == 'pd'):
K_RETRY['pd'].append(df.loc[i, 'code'])
elif (df.loc[i, 'type'] == 'ph'):
K_RETRY['ph'].append(df.loc[i, 'code'])
elif (df.loc[i, 'type'] == 'ks'):
K_RETRY['ks'].append(df.loc[i, 'code'])
elif (df.loc[i, 'type'] == 'km'):
K_RETRY['km'].append(df.loc[i, 'code'])
print(K_RETRY)
if (not os.path.exists(ConfigDict['k_fail_log'])):
new_df = pd.DataFrame(data=None, columns=['date', 'type', 'code'])
new_df.to_csv(ConfigDict['k_fail_log'], mode='w')
if (create_db):
dc = DbCreator()
dc.init_create_table()
if (pull_list):
sp = SharePuller()
sp.pull()
if (pull_k_data):
KManager.pull_data(start_date,
start_code=start_code,
retry=retry,
retryDict=K_RETRY)
KManager.pull_data_hk(start_date,
start_code=start_code,
retry=retry,
retryDict=K_RETRY)
if (pull_k_h_data):
KManager.pull_data_hk(start_date,
start_code=start_code,
retry=retry,
retryDict=K_RETRY)
if (pull_m_k_data):
KManager.pull_data_month_and_month_kpi(start_date,
start_code=start_code,
retry=retry,
retryDict=K_RETRY)
if (kpi_k):
KManager.count_kpi(start_date,
start_code=start_code,
retry=retry,
retryDict=K_RETRY)
if (pull_f_data):
FinManager.pull_data()
if (kpi_f):
FinManager.count_kpi()
print('finish OK')