from pathlib import Path
import click
import pandas as pd
from cnswd.mongodb import get_db
from cnswd.scripts.base import get_stock_status
from cnswd.utils import make_logger
from trading_calendars import get_calendar
from zipline.utils.cli import maybe_show_progress
from ..minute_bars import CN_EQUITIES_MINUTES_PER_DAY, BcolzMinuteBarWriter
from .core import load, most_recent_data
from .for_test_bundle import TEST_CODES
from .wy_minute_data import encode_index_code, fetch_single_minutely_equity
logger = make_logger('数据包', collection='zipline')
def info_func():
calendar = get_calendar('XSHG')
# 本地分钟级别数据开始日期
CALENDAR_START = pd.Timestamp('2020-06-29', tz='UTC')
now = pd.Timestamp('now')
if now.date() == calendar.actual_last_session.date() and now.hour < 15:
CALENDAR_STOP = calendar.actual_last_session - calendar.day
else:
CALENDAR_STOP = calendar.actual_last_session
return calendar, CALENDAR_START, CALENDAR_STOP
def execute(cmd):
from cnswd.utils import make_logger
from ..common import AD_FIELD_NAME, SID_FIELD_NAME, TS_FIELD_NAME
from .base import bcolz_table_path
from .wy_data import (get_dividend_data, get_investment_rating_data,
get_margin_data, get_short_name_changes)
from .wy_finance import _add_ggrq, _get_p_data, get_ggrq, get_q_indicator
from .wy_info import get_industry, get_info, get_ths_concept
# from .yahoo import YAHOO_ITEMS, read_item_data
# 设置显示日志
logbook.set_datetime_format('local')
logbook.StreamHandler(sys.stdout).push_application()
logger = make_logger('网易数据包')
def _fix_mixed_type(df):
# 1. 修复str类型中存在混合类型的列 如 ldf np.NaN lyy
# 2. bool 类型含空值
for col in df.columns:
if pd.api.types.is_string_dtype(df[col]):
# 注意，字符串为空，一律以 空白字符串 替代
df.fillna(value={col: ''}, inplace=True)
if pd.api.types.is_bool_dtype(df[col]):
df.fillna(value={col: False}, inplace=True)
if pd.api.types.is_integer_dtype(df[col]):
df.fillna(value={col: -1}, inplace=True)
if pd.api.types.is_datetime64tz_dtype(df[col]):
raise ValueError('时间列不得带时区信息')
"""
import time
import pandas as pd
from cnswd.utils import HotDataCache, make_logger
from . import core as bundles
from .adjusts import NON_ADJUSTED_COLUMN_FACTOR
from .utils import _exchanges
from .wy_data import (fetch_single_equity, fetch_single_quity_adjustments,
gen_asset_metadata)
from .wy_minute_data import fetch_single_minutely_equity
TODAY = pd.Timestamp('today').normalize()
log = make_logger('wydb', collection='zipline')
OHLCV_COLS = ['open', 'high', 'low', 'close', 'volume']
# 截取测试股票、指数代码 + 新股 + 中途停牌
TEST_SIDS = [
1, 2, 333, 2335, 2024, 600645, 300001, 300882, 300313, 1000002, 1000001
]
# TEST_SIDS = [2]
TEST_CODES = [str(x).zfill(6) for x in TEST_SIDS]
def _to_sid(x):
"""符号转换为sid"""
return int(x)
from itertools import product
from multiprocessing import Manager, Pool
import pandas as pd
from numpy.random import shuffle
from toolz import partition_all
from toolz.dicttoolz import valfilter
from cnswd.mongodb import get_db
from cnswd.setting.constants import MARKET_START, MAX_WORKER
from cnswd.utils import make_logger
from cnswd.websource.wy import fetch_company_info
from .base import get_stock_status
logger = make_logger('网易公司资料')
NAMES = ['公司简介', 'IPO资料']
START = MARKET_START.tz_localize(None)
def create_index_for(collection):
# 不存在索性信息时，创建索引
if not collection.index_information():
collection.create_index([("股票代码", 1)])
def need_refresh(collection2, code):
"""是否需要刷新
简单规则：
from itertools import product
from multiprocessing import Manager, Pool
import pandas as pd
from numpy.random import shuffle
from toolz import partition_all
from toolz.dicttoolz import valfilter
from cnswd.mongodb import get_db
from cnswd.setting.constants import MARKET_START, MAX_WORKER
from cnswd.utils import make_logger
from cnswd.websource.wy import fetch_financial_indicator
from .base import get_stock_status
logger = make_logger('网易主要财务指标')
TYPES = {'report': '按报告期', 'year': '按年度', 'season': '按单季度'}
NAMES = {
'zycwzb': '主要财务指标',
'ylnl': '盈利能力',
'chnl': '偿还能力',
'cznl': '成长能力',
'yynl': '营运能力'
}
START = MARKET_START.tz_localize(None)
DATE_KEY = '报告日期'
def create_index_for(collection):
# 不存在索性信息时，创建索引
import asyncio
import aiohttp
import pandas as pd
# import re
from cnswd.mongodb import get_db
from cnswd.utils import make_logger
logger = make_logger('同花顺财经')
HEADERS = {
'Host': 'news.10jqka.com.cn',
'User-Agent':
'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:77.0) Gecko/20100101 Firefox/77.0',
'Accept': 'application/json, text/javascript, */*; q=0.01',
'Accept-Language': 'zh-CN,en-US;q=0.7,en;q=0.3',
'Accept-Encoding': 'gzip, deflate, br',
'X-Requested-With': 'XMLHttpRequest',
'Connection': 'keep-alive',
'Referer': 'https://news.10jqka.com.cn/realtimenews.html',
}
DEFAULT_KWARGS = {'page': 1, 'tag': '', 'track': 'website', 'pagesize': 400}
url = 'https://news.10jqka.com.cn/tapp/news/push/stock/'
# T_PATT = re.compile(".*?time$")
# logger = make_logger('同花顺财经')
def to_timestamp(x):
# 输入字符串
dt = pd.Timestamp(int(x), unit='s', tz='Asia/Shanghai')
from itertools import product
from multiprocessing import Manager, Pool
import pandas as pd
from numpy.random import shuffle
from toolz import partition_all
from toolz.dicttoolz import valfilter
from cnswd.mongodb import get_db
from cnswd.setting.constants import MARKET_START, MAX_WORKER
from cnswd.utils import make_logger
from cnswd.websource.wy import fetch_yjyg
from .base import get_stock_status
logger = make_logger('网易业绩预告')
START = MARKET_START.tz_localize(None)
DATE_KEY_1 = '公告日期'
DATE_KEY_2 = '报告日期'
def create_index_for(collection):
# 不存在索性信息时，创建索引
if not collection.index_information():
collection.create_index([(DATE_KEY_1, 1), (DATE_KEY_2, 1),
("股票代码", 1)],
unique=True)
def need_refresh(collection, code):
from itertools import product
from multiprocessing import Manager, Pool
import pandas as pd
from numpy.random import shuffle
from toolz import partition_all
from toolz.dicttoolz import valfilter
from cnswd.mongodb import get_db
from cnswd.setting.constants import MARKET_START, MAX_WORKER
from cnswd.utils import make_logger
from cnswd.websource.wy import fetch_financial_report
from .base import get_stock_status
logger = make_logger('网易财务分析')
NAMES = {'zcfzb': '资产负债表', 'lrb': '利润表', 'xjllb': '现金流量表'}
# NAMES = {'xjllb': '现金流量表'} # 删除
START = MARKET_START.tz_localize(None)
DATE_KEY = '报告日期'
def create_index_for(collection):
# 不存在索性信息时，创建索引
if not collection.index_information():
collection.create_index([(DATE_KEY, 1), ("股票代码", 1)], unique=True)
def need_refresh(collection, code):
"""是否需要刷新