def __init__(self, name, date, q, event=None):
Thread.__init__(self,
group=None,
target=None,
name=name,
args=(),
kwargs=None,
daemon=False)
self.ready = event
self.log = Logger('lib')
self._index = schema.get_schema('INDEX')
self.meta_index = schema.get_schema('META_INDEX')
self.detail = schema.get_schema('DETAIL_MODEL')
# Collect basic data from current records in db
con = self._index.get_con()
self.meta = pd.read_sql_query('SELECT * FROM meta.index', con)
self.q = q
self.today = date
def __init__(self, name, **kwargs):
Thread.__init__(self,
group=None,
target=None,
name=name,
args=(),
kwargs=None,
daemon=False)
self.log = Logger('lib')
self.basic = schema.get_schema('BASIC_STOCKS')
self._meta = schema.get_schema('META_CODE')
self.detail = schema.get_schema('DETAIL_MODEL')
# Collect basic data from current records in db
con = self.basic.get_con()
self.meta = pd.read_sql_query('SELECT * FROM meta.code', con)
# self.hs = pd.DataFrame()
self.today = kwargs['today']
self.hs = kwargs['hs']
self.q = kwargs['q']
import time
import tushare as ts
import datetime as dt
from queue import Queue
import psycopg.pg_opt as pg
from threading import Thread
import psycopg.schema as schema
from Logger.Logger import Logger
from stock.collector import fill
from stock.collector import index
from urllib.error import HTTPError
log = Logger('lib')
today = dt.date.today().isoformat()
daily = schema.get_schema('DAILY')
RETRY_COUNT = 3
THREAD_NUM = 10
q = Queue()
def _check_update():
res = daily.select(["*"], date='\'%s\'' % today).execute().fetch()
if len(res):
return False
# Not done today
return True
def _update_meta(**kargs):
"""
# -*- coding:utf-8 -*-
import re
import numpy as np
import pandas as pd
import datetime as dt
from psycopg import schema
basic_schema = schema.get_schema('BASIC_STOCKS')
con = basic_schema.get_con()
# Filter conditions:
# SH(600*), SZ(00*), gem, sh50, hs300, zz500, sme, st
# industry
# total_assets
# time_to_market
# pe
class Pool():
def __init__(self, **conditions):
assert not con.closed, 'DB connection not avaliable now.'
_stock_frame = pd.read_sql_query(
'SELECT * FROM basic.stock_basic ORDER BY code;', con)
assert not _stock_frame.empty, 'Cannot load stocks from DB or is empty.'
self.stocks = _stock_frame
self.filter = {
'SH': self._SH,
'SZ': self._SZ,
'gem': self._gem,
'sh50': self._sh50,
'hs300': self._hs300,
'zz500': self._zz500,
# -*- coding:utf-8 -*-
import re
import uuid
import pandas as pd
import psycopg.schema as schema
from Logger.Logger import Logger
logger = Logger('lib')
db_o = schema.get_schema('CONCEPT')
logger.info('Start filter concept...')
persistence = db_o.select().execute().fetch()
origin = set()
for i in persistence:
origin.add(i['name'])
def do(hs):
hs = hs.reindex(['code','name','concept'], axis=1)
concepts = set()
for item in list(hs['concept']):
if item != '':
split = item.split(',')
for sub in split:
concepts.add(sub)
for item in concepts:
group = hs[hs['concept'].apply(lambda i: (','+item in i) or (item+',' in i))].reindex(['name','code'], axis=1)
concept_d = list(group.to_dict('index').values())
concept_str = str(concept_d).replace('\'', '"')
if item in origin:
# -*- coding:utf-8 -*-
import uuid
import pandas as pd
import psycopg.schema as schema
from Logger.Logger import Logger
logger = Logger('lib')
db_o = schema.get_schema('INDUSTRY')
logger.info('Start filter industry...')
persistence = db_o.select().execute().fetch()
origin = set()
for i in persistence:
origin.add(i['name'])
def do(hs):
hs = hs.reindex(['name','code','industry'], axis=1)
industries = set(hs['industry'])
for item in industries:
group = hs[hs['industry']==item].reindex(['code','name'],axis=1)
industry_d = list(group.to_dict('index').values())
industry_str = str(industry_d).replace('\'', '"')
if item in origin:
logger.info('Update existing industry \'%s\''%item)
old_item = list(filter(lambda it: item == it['name'], persistence))[0]
guid = old_item['guid']
db_o.update(uuid=guid, list=industry_str).execute()
else:
logger.info('Creating new industry \'%s\''%item)
uid = uuid.uuid4()
def __init__(self, code):
assert code, 'Must provied stock code to fetch data!'
self.stock = code
self.basic = schema.get_schema('BASIC_STOCKS')
self.detail = schema.get_schema('DETAIL_MODEL')
self.init()
import uuid
import math
import copy
import numpy as np
import pandas as pd
import tushare as ts
import datetime as dt
import psycopg.schema as schema
from queue import Queue
from threading import Thread
from Logger.Logger import Logger
from stock.collector.basic import get_basics
logger = Logger('lib')
db_o = schema.get_schema('REPORT')
_today = dt.date.today()
YEAR = _today.year
QUARTER = math.ceil(_today.month / 3)
def get_profit(year, quarter, q):
"""
eps: 每股收益
eps_yoy: 每股收益同比
bvps: 每股净资产
roe: 净资产收益率
epcf: 每股现金流
net_profits: 净利润（万元）
profits_yoy: 净利润同比