def retrieve_interval_price_summaries(market: str, code: str, interval_minute: int, data_num: int = 1023):
params = {
"symbol": market + str(code),
"scale": interval_minute,
"datalen": data_num,
"ma": "no",
}
url = "http://money.finance.sina.com.cn/quotes_service/api/json_v2.php/CN_MarketData.getKLineData"
with urlopen("%s?%s" % (url, urlencode(params))) as response:
page = response.read()
result = json.loads(page)
df = DataFrame(result)
price_df = transform(df, [
Pipeline([], IntervalPriceSummary.code.key, lambda: code),
Pipeline([], IntervalPriceSummary.exchange.key, lambda: market),
Pipeline("day", IntervalPriceSummary.time.key,
lambda day_str: day_str.replace(" ", "T")),
Pipeline([], IntervalPriceSummary.interval_len.key, lambda: time(minute=interval_minute)),
], ["open", "close", "high", "low", "volume"])
insert_dataframe(price_df, IntervalPriceSummary)
def retrieve_company_info() -> DataFrame:
df = pro_api().stock_basic()
industry_df = unique_of(
transform(df, [Pipeline('industry', Industry.industry.key)]),
Industry.industry.key)
insert_dataframe(industry_df, Industry)
exchange_df = unique_of(
transform(df, [
Pipeline('ts_code', ExchangeMarkets.exchange.key,
lambda x: x.split('.')[-1]),
]), ExchangeMarkets.exchange.key)
insert_dataframe(exchange_df, ExchangeMarkets)
areas_df = unique_of(transform(df, [Pipeline('area', Areas.area.key)]),
Areas.area.key)
insert_dataframe(areas_df, Areas)
stocks_df = transform(df, [
Pipeline('symbol', Stocks.code.key),
Pipeline('name', Stocks.full_name.key),
Pipeline('ts_code', Stocks.exchange.key, lambda x: x.split('.')[-1]),
Pipeline('list_date', Stocks.list_date.key, ts_date_to_sql)
],
preserve_list=['industry', 'area'])
return stocks_df
def retrieve_daily_price_summaries(trade_date: datetime.date = None):
if not trade_date:
trade_date = datetime.now().date()
# use api to get daily stock summaries
df = pro_api().daily(trade_date=date_to_ts(trade_date))
# make sure exchange dependency met
exchange_df = ts_daily_to_exchange_table(df)
insert_dataframe(exchange_df, ExchangeMarkets)
# make sure stock dependency met
stocks_df = ts_daily_to_stocks_transform(df)
insert_dataframe(stocks_df, Stocks, duplicate_update=False)
# put daily stock data in
daily_price_summary_df = ts_daily_to_daily_price_summaries_table(df, trade_date)
insert_dataframe(daily_price_summary_df, DailyPriceSummaries)
from data_collection.sources.tushare.dataframe_requester import retrieve_company_info
from database.sql_tables import Stocks
from database.utils import insert_dataframe
if __name__ == "__main__":
insert_dataframe(retrieve_company_info(), Stocks)
def insert_df(ts_code: str):
daily_df = retrieve_history_daily_price(ts_code)
insert_dataframe(daily_df, DailyPriceSummaries)