def to_df(file_in):
try:
save_dir = os.path.join(data_dict.get("dfcf_gudonghushu"),"parse_data")
create_dir_if_not_exist(save_dir)
df1 = json_to_df(file_in)
df2 = df1.get("data_out")
with open(file_in) as f: ss = f.read()
df2.iloc[0,:] = [ss.split("HolderNum")[1].split(",")[0].replace(":",'').replace('\"','')]+df2.iloc[0,2:15].tolist()+["0"]
df2.columns = ["HolderNum","PreviousHolderNum","HolderNumChange"
,"HolderNumChangeRate","RangeChangeRate",
"EndDate","HolderAvgCapitalisation","HolderAvgStockQuantity",
"TotalCapitalisation","CapitalStock","NoticeDate",
"CapitalStockChange","CapitalStockChangeEvent",
"ClosePrice","test"]
del df2['test']
df2["dt"] = df2["EndDate"]
df2['stock_index'] = file_in[0:6]
df3 = dfProcess.col2Float(df2,["HolderNum","PreviousHolderNum","HolderNumChange"])
save_file = os.path.join(save_dir,file_in.replace(".txt",".csv"))
#save_file = "./test.csv"
df2.to_csv(save_file,index=None)
except:
save_dir = os.path.join(data_dict.get("dfcf_gudonghushu"),"parse_data")
#save_dir="./"
save_file = os.path.join(save_dir,file_in.replace(".txt",".csv"))
df2=pd.DataFrame()
df2.to_csv(save_file,index=None)
def save_the_table(new_table, dir_dadan, now_date, now_date_time, tag='no'):
save_dir = os.path.join(dir_dadan, now_date)
create_dir_if_not_exist(save_dir)
if tag == 'no':
save_file = os.path.join(save_dir, now_date_time + ".csv")
else:
save_file = os.path.join(save_dir, now_date_time + '_' + tag + ".csv")
new_table.to_csv(save_file, index=0)
def main():
now_date, now_date_time = get_the_datetime(
) ## the now_date is like "2019_11_08"
dir_dadan = data_dict.get("DADAN")
data_dir = os.path.join(dir_dadan, now_date)
df1 = combine_csv_in_folder(data_dir)
df1.columns = setColname().DADAN()
## merge the data
df_merge1 = DADAN_diff_stat(df1)
df_merge1.to_csv("DADAN_sample.csv", index=0)
print("=" * 50)
print(df_merge1[['stock_index', 'stock_name', 'buy_sale_diff']].head(50))
#print(df_merge1[['stock_index','stock_name','buy_sale_diff']][df_merge1['price']<25].head(50))
print(df_merge1.tail(50))
## save data
print("=" * 50)
save_dir = os.path.join(data_path, "DADAN_daily_report")
create_dir_if_not_exist(save_dir)
save_file = "DADAN_200_daily_report_" + now_date + ".csv"
save_file = os.path.join(save_dir, save_file)
df_merge1.to_csv(save_file, encoding="utf_8_sig")
return df_merge1
def save_the_table(new_table, dir_dadan, now_date, page):
save_dir = os.path.join(dir_dadan, now_date)
create_dir_if_not_exist(save_dir)
save_file = os.path.join(save_dir, page + ".csv")
new_table.to_csv(save_file, index=0)
df_merge = pd.merge(df_buy,df_sell,how='left',on = ["stock_index","stock_name"]) # merge join, on 'stock_index' and 'stock_name'
df_merge = df_merge.fillna(0)
df_merge["buy_sale_diff"] = df_merge["buy_num"]-df_merge["sale_num"]
df_merge["buy_sale_diff_shou"] = df_merge["buy_shou"]-df_merge["sale_shou"]
#df_merge["buy_sale_diff"] = (df_merge["buy_num"]-df_merge["sale_num"])/df_merge["price"]
df_merge1 = df_merge.sort_values("buy_sale_diff",ascending=False)
#print(df_merge1)
return df_merge1
def DADAN_columns():
return ["stock_index","stock_name", \
"trade_time","price","trade_num","trade_shou", \
"status","price_change_rate","price_change_ratio","look","date"]
if __name__ =='__main__':
#now_date,now_date_time = get_the_datetime() ## the now_date is like "2019_11_08"
now_date = "2020_01_23"
dir_dadan = data_dict.get("DADAN")
data_dir = os.path.join(dir_dadan,now_date)
print(data_dir)
df1 = combine_csv_in_folder(data_dir)
df1.columns = DADAN_columns()
df_merge1 = DADAN_diff_stat(df1)
df_merge1.to_csv("DADAN_sample.csv",index=0)
print(df_merge1.head(30))
print(df_merge1.tail(30))
save_dir = os.path.join(data_path,"DADAN_daily_report")
create_dir_if_not_exist(save_dir)
save_file="DADAN_daily_report_"+now_date+".csv"
save_file = os.path.join(save_dir,save_file)