def set_duration_console(du_date):
        if cct.isMac():
            cct.set_console(width, height)
        else:
            cct.set_console(width, height, title=str(du_date))
    status = False
    vol = ct.json_countVol
    type = ct.json_countType
    success = 0
    top_all = pd.DataFrame()
    time_s = time.time()
    delay_time = 720000
    # delay_time = cct.get_delay_time()
    First = True
    blkname = '062.blk'
    block_path = tdd.get_tdx_dir_blocknew() + blkname
    status_change = False
    lastpTDX_DF = pd.DataFrame()
    # dl=30
    ptype = 'low'
    # op, ra, duration_date, days = pct.get_linear_model_status('999999', filter='y', dl=dl, ptype=ptype, days=1)
    # duration_date = int(ct.duration_date * 1.4)
    duration_date = ct.duration_date_l
#    duration_date = 120
    # duration_date = 300
    du_date = duration_date
    resample = ct.resample_dtype
    newdays = 5
    end_date = None
    ptype = 'low'
    filter = 'y'
    if cct.isMac():
        cct.set_console(165, 16)
    else:
        cct.set_console(160, 15)
    status = False
    vol = '0'
    type = '2'
    # cut_num=10000
    success = 0
    top_all = pd.DataFrame()
    time_s = time.time()
    # delay_time = 3600
    delay_time = cct.get_delay_time()
    First = True
    base_path = tdd.get_tdx_dir()
    block_path = tdd.get_tdx_dir_blocknew() + '062.blk'
    status_change = False

    # all_diffpath = tdd.get_tdx_dir_blocknew() + '062.blk'
    while 1:
        try:
            # df = sina_data.Sina().all
            df = rl.get_sina_Market_json('all')
            top_now = rl.get_market_price_sina_dd_realTime(df, vol, type)
            # print top_now.loc['300208','name']
            # top_now.to_hdf("testhdf5", 'marketDD', format='table', complevel=9)
            df_count = len(df)
            now_count = len(top_now)
            del df
            gc.collect()
            radio_t = cct.get_work_time_ratio()
예제 #3
0
    if cct.isMac():
        cct.set_console(90, 16)
    else:
        cct.set_console(82, 16)
    status = False
    vol = "0"
    type = "2"
    cut_num = 20000
    success = 0
    top_all = pd.DataFrame()
    time_s = time.time()
    # delay_time = 7200
    delay_time = cct.get_delay_time()
    base_path = tdd.get_tdx_dir()
    block_path = tdd.get_tdx_dir_blocknew() + "064.blk"
    while 1:
        try:
            df = rl.get_sina_all_json_dd(vol, type)
            if len(df) > cut_num:
                df = df[:cut_num]
                print len(df),
            top_now = rl.get_sina_dd_count_price_realTime(df)
            # print len(top_now)
            time_d = time.time()
            if time_d - time_s > delay_time:
                status_change = True
                time_s = time.time()
                top_all = pd.DataFrame()

            else:
예제 #4
0
# -*- coding:utf-8 -*-
import datetime
import time

import pandas as pd

import JohnsonUtil.commonTips as cct
# from JSONData import LineHistogram as lrh
from JSONData import powerCompute as pct
from JSONData import realdatajson as rl
from JSONData import tdx_data_Day as tdd

base_path = tdd.get_tdx_dir()
block_path = tdd.get_tdx_dir_blocknew() + '068.blk'
block_path_d = tdd.get_tdx_dir_blocknew() + '069.blk'
if __name__ == "__main__":
    # df=sina_data.Sina().all.set_index('code')
    today = datetime.date.today()
    print today

    def get_market_status(mk, type='m', start=None, end=None):
        df = rl.get_sina_Market_json(mk).set_index('code')
        time_s = time.time()
        code_l = []
        code_d = []
        for code in df.index:
            status, lenday, diff = pct.get_linear_model_status_LSH(code,
                                                                   type=type,
                                                                   start=start,
                                                                   end=end)
            # op, ra, status, dss = pct.get_linear_model_status(code, type=type, start=start, end=end)
예제 #5
0
# -*- coding:utf-8 -*-
import time
import datetime
import pandas as pd
import LineHistogram as lrh
from JSONData import realdatajson as rl
import JohhnsonUtil.commonTips as cct
from JSONData import tdx_data_Day as tdd

base_path = tdd.get_tdx_dir()
block_path = tdd.get_tdx_dir_blocknew() + '068.blk'
block_path_d = tdd.get_tdx_dir_blocknew() + '069.blk'
if __name__ == "__main__":
    # df=sina_data.Sina().all.set_index('code')
    today = datetime.date.today()
    print today

    def get_market_status(mk, type='m'):
        df = rl.get_sina_Market_json(mk).set_index('code')
        time_s = time.time()
        code_l = []
        code_d = []
        for code in df.index:
            status, lenday, diff = lrh.get_linear_model_status(code, type)
            if status:
                # print(df.loc[code,:])
                # if df.loc[code, 'percent'] > 0 and lenday > 200:
                if df.loc[code, 'trade'] != 0 and lenday > 200:
                    df.loc[code, 'diff'] = abs(diff)
                    code_l.append(df.loc[code, :])
            elif df.loc[code, 'trade'] != 0 and lenday > 200:
예제 #6
0
def get_roll_mean_all(single=True,
                      tdx=False,
                      app=True,
                      duration=100,
                      ma_250_l=1.02,
                      ma_250_h=1.11):
    # df = tdd.search_Tdx_multi_data_duration('tdx_all_df_300', 'all_300', df=None,code_l=code_list, start=start, end=None, freq=None, col=None, index='date')
    block_path = tdd.get_tdx_dir_blocknew() + '060.blk'

    if not app and cct.get_file_size(
            block_path) > 100 and cct.creation_date_duration(block_path) == 0:
        print "It's Today Update"
        return True
    code_list = sina_data.Sina().market('all').index.tolist()
    print "all code:", len(code_list)
    if duration < 300:
        h5_fname = 'tdx_all_df' + '_' + str(300)
        h5_table = 'all' + '_' + str(300)
    else:
        h5_fname = 'tdx_all_df' + '_' + str(900)
        h5_table = 'all' + '_' + str(900)
    # df = tdd.search_Tdx_multi_data_duration('tdx_all_df_300', 'all_300', df=None,code_l=code_list, start='20150501', end=None, freq=None, col=None, index='date')
    df = tdd.search_Tdx_multi_data_duration(h5_fname,
                                            h5_table,
                                            df=None,
                                            code_l=code_list,
                                            start=None,
                                            end=None,
                                            freq=None,
                                            col=None,
                                            index='date')
    # df = tdd.search_Tdx_multi_data_duration(h5_fname, h5_table, df=None,code_l=code_list, start=None, end=None, freq=None, col=None, index='date',tail=1)

    code_uniquelist = df.index.get_level_values('code').unique()

    code_select = code_uniquelist[random.randint(0, len(code_uniquelist) - 1)]
    print round(time.time() - time_s, 2), df.index.get_level_values(
        'code').unique().shape, code_select, df.loc[code_select].shape
    # df.groupby(level=[0]),df.index.get_level_values(0)
    # len(df.index.get_level_values('code').unique())
    # df = df[~df.index.duplicated(keep='first')]
    dfs = df

    def get_groupby_mean_median_close(dfs):

        groupd = dfs.groupby(level=[0])
        df = groupd['close'].agg({'median': 'median', 'mean': 'mean'})
        df['close'] = groupd.tail(1).reset_index().set_index(['code'])['close']
        # dfs['mean'] = groupd['close'].agg('mean')
        # dfs['median'] = groupd['close'].agg('median')

        # dfs = dfs.fillna(0)
        # idx = pd.IndexSlice
        # mask = ( (dfs['mean'] > dfs['median'])
        #         & (dfs['close'] > dfs['mean'])
        #         )
        # df=dfs.loc[idx[mask, :]]

        df = df[(df['mean'] > df['median']) & (df['close'] > df['mean'])]

        # dt_low = None
        # if dl == 1:
        #     dfs = groupd.tail(1)
        #     print("dfs tail1")
        # else:
        #     dl = 30
        #     dindex = tdd.get_tdx_Exp_day_to_df(
        #         '999999', dl=dl).sort_index(ascending=False)
        #     dt = tdd.get_duration_price_date('999999', df=dindex)
        #     dt = dindex[dindex.index >= dt].index.values
        #     dt_low = dt[-1]
        #     dtlen = len(dt) if len(dt) >0 else 1
        #     dfs = groupd.tail(dtlen)
        #     print("dfs tail:%s dt:%s"%(dtlen,dt))
        #     dfs = get_multi_date_duration(dfs,dt[-1])
        return df

    groupd = dfs.groupby(level=[0])

    # rollma = ['5','10','60','100','200']
    # rollma = ['5','10','250']
    if duration < 300:
        rollma = ['10']
    else:
        rollma = ['10', '250']

    rollma.extend([str(duration)])

    # import ipdb;ipdb.set_trace()
    # df.loc['300130'][:2]

    # dfs['mean'] = groupd['close'].agg('mean')
    # dfs['median'] = groupd['close'].agg('median')

    for da in rollma:
        cumdays = int(da)
        dfs['ma%d' % cumdays] = groupd['close'].apply(pd.rolling_mean, cumdays)
        if cumdays == 10:
            dfs['upper'] = dfs['ma%d' % cumdays].apply(lambda x: round(
                (1 + 11.0 / 100) * x, 1))
            dfs['lower'] = dfs['ma%d' % cumdays].apply(lambda x: round(
                (1 - 9.0 / 100) * x, 1))
            dfs['ene'] = map(lambda x, y: round((x + y) / 2, 1), dfs['upper'],
                             dfs['lower'])
        # df['upper'] = map(lambda x: round((1 + 11.0 / 100) * x, 1), df.ma10d)
        # df['lower'] = map(lambda x: round((1 - 9.0 / 100) * x, 1), df.ma10d)
        # df['ene'] = map(lambda x, y: round((x + y) / 2, 1), df.upper, df.lower)
        # dfs['amount%d'%cumdays] = groupd['amount'].apply(pd.rolling_mean, cumdays)
    # df.ix[df.index.levels[0]]
    #df.ix[df.index[len(df.index)-1][0]] #last row
    # dfs = tdd.search_Tdx_multi_data_duration(df=dfs,code_l=code_list, start='20170918', end='20170918', freq=None, col=None, index='date')

    # print dfs[:1],len(dfs)
    # groupd.agg({'low': 'min'})
    # '''idx mask filter'''
    # '''
    dt_low = None
    df_idx = None
    if single:
        dfs = groupd.tail(1)
        print("dfs tail1")
    else:
        dl = 30
        dindex = tdd.get_tdx_Exp_day_to_df('999999',
                                           dl=dl).sort_index(ascending=False)
        dt = tdd.get_duration_price_date('999999', df=dindex)
        dt = dindex[dindex.index >= dt].index.values
        dt_low = dt[-1]
        dtlen = len(dt) if len(dt) > 0 else 1
        dfs = groupd.tail(dtlen)
        # import ipdb;ipdb.set_trace()
        df_idx = get_groupby_mean_median_close(dfs)

        print("dfs tail:%s dt:%s" % (dtlen, dt))
        dfs = get_multi_date_duration(dfs, dt[-1])

        # groupd2 = dfs.groupby(level=[0])
        # dfs['ma%d'%cumdays] = groupd['close'].apply(pd.rolling_mean, cumdays)

        # dfs.reset_index().groupby(['code'])['date'].transform('count')
        single = True

    dfs = dfs.fillna(0)
    idx = pd.IndexSlice
    # mask = (dfs[('ma%s')%(rollma[0])] > dfs[('ma%s')%(rollma[1])]) & (dfs[('ma%s')%(rollma[-1])] > 0) & (dfs[('close')] > dfs[('ma%s')%(rollma[0])])  & (dfs[('close')] > dfs[('ma%s')%(rollma[-1])])
    # mask = (dfs[('ma%s')%(rollma[0])] > dfs[('ma%s')%(rollma[1])]) & (dfs[('ma%s')%(rollma[-1])] > 0) & (dfs[('close')] > dfs[('ma%s')%(rollma[1])])  & (dfs[('close')] > dfs[('ma%s')%(rollma[-1])])
    # mask = (dfs[('ma%s')%(rollma[0])] > dfs[('ma%s')%(rollma[1])]) & (dfs[('ma%s')%(rollma[-1])] > 0) &  (dfs[('close')] > dfs[('ma%s')%(rollma[-1])])

    # mask = ( (dfs[('ma%s')%(rollma[0])] > 0) & (dfs[('ma%s')%(rollma[-1])] > 0) & (dfs[('close')] > dfs[('ma%s')%(rollma[-1])]) & (dfs[('close')] > dfs[('ma%s')%(rollma[0])]))

    # mask = ( (dfs[('ma%s')%(rollma[0])] > 0) & (dfs[('ma%s')%(rollma[-1])] > 0)
    #         & (dfs[('close')] > dfs[('ma%s')%(rollma[-1])]*ma_250_l)
    #         & (dfs[('close')] < dfs[('ma%s')%(rollma[-1])]*ma_250_h)
    #         & (dfs[('close')] > dfs[('ma%s')%(rollma[0])]))

    # & (dfs['mean'] > dfs['median'])
    # & (dfs['close'] > dfs['mean'])

    if len(rollma) > 1:

        mask = ((dfs[('ma%s') % (rollma[0])] > 0) & (dfs[('ma%s') %
                                                         (rollma[-1])] > 0)
                & (dfs[('ma%s') % (rollma[0])] > dfs[('ma%s') % (rollma[-1])])
                & (dfs[('close')] > dfs[('ma%s') % (rollma[0])])
                & (dfs[('close')] > dfs[('ma%s') % (rollma[-1])] * ma_250_h)
                & ((dfs[('close')] > dfs['ene']) |
                   (dfs[('close')] > dfs['upper'])))
    else:
        mask = ((dfs[('ma%s') % (rollma[0])] > 0)
                & (dfs[('close')] > dfs[('ma%s') % (rollma[0])])
                & ((dfs[('close')] > dfs['ene']) |
                   (dfs[('close')] > dfs['upper'])))

    # mask = ((dfs[('close')] > dfs[('ma%s')%(rollma[-1])]))
    df = dfs.loc[idx[mask, :]]
    df = get_multi_code_count(df)
    print(df.couts[:5])

    # import ipdb;ipdb.set_trace()
    # df.sort_values(by='couts',ascending=0)
    # groupd.first()[:2],groupd.last()[:2]
    # groupd = df250.groupby(level=[0])
    # '''
    # groupd.transform(lambda x: x.iloc[-1])
    # groupd.last()
    # groupd.apply(lambda x: x.close > x.ma250)
    # df.shape,df.sort_index(ascending=False)[:5]
    # ?groupd.agg
    # groupd = df.groupby(level=[0])
    # groupd['close'].apply(pd.rolling_mean, 250, min_periods=1)
    #ex:# Group df by df.platoon, then apply a rolling mean lambda function to df.casualties
    # df.groupby('Platoon')['Casualties'].apply(lambda x:x.rolling(center=False,window=2).mean())

    code_uniquelist = df.index.get_level_values('code').unique()
    code_select = code_uniquelist[random.randint(0, len(code_uniquelist) - 1)]

    if app:
        print round(time.time() - time_s, 2), 's', df.index.get_level_values(
            'code').unique().shape, code_select, df.loc[code_select][-1:]

    if single:
        # groupd = df.groupby(level=[0])
        if tdx:
            # block_path = tdd.get_tdx_dir_blocknew() + '060.blk'
            # if cct.get_work_time():
            #     codew = df[df.date == cct.get_today()].index.tolist()

            if dt_low is not None:
                groupd2 = df.groupby(level=[0])
                df = groupd2.tail(1)
                df = df.reset_index().set_index('code')
                # import ipdb;ipdb.set_trace()

                # df = df[(df.date >= dt_low) & (df.date <= cct.get_today())]
                dd = df[(df.date == dt_low)]
                df = df[(df.date >= cct.last_tddate(1))]
                # import ipdb;ipdb.set_trace()
                print("df:%s df_idx:%s" % (len(df), len(df_idx)))

                if df_idx is not None and len(df_idx) > 0:
                    df = df.loc[df_idx.index, :].dropna()
                print("Main Down dd :%s MainUP df:%s couts std:%0.1f " %
                      (len(dd), len(df), df.couts.std()))
                # print df.date.mode()[0]
                df = df.sort_values(by='couts', ascending=1)
                df = df[df.couts > df.couts.std()]
                # df = df[(df.date >= df.date.mode()[0]) & (df.date <= cct.get_today())]
                codew = df.index.tolist()

                if app:
                    print round(time.time() - time_s, 2), 'groupd2', len(df)

            else:
                df = df.reset_index().set_index('code')
                df = df[(df.date >= cct.last_tddate(days=10))
                        & (df.date <= cct.get_today())]
                codew = df.index.tolist()

            top_temp = tdd.get_sina_datadf_cnamedf(codew, df)
            top_temp = top_temp[(~top_temp.index.str.contains('688'))
                                & (~top_temp.name.str.contains('ST'))]
            codew = top_temp.index.tolist()

            #clean st and 688

            if app:
                hdf5_wri = cct.cct_raw_input("rewrite code [Y] or append [N]:")
                if hdf5_wri == 'y' or hdf5_wri == 'Y':
                    append_status = False
                else:
                    append_status = True
            else:
                append_status = False

            if len(codew) > 10:
                cct.write_to_blocknew(block_path,
                                      codew,
                                      append_status,
                                      doubleFile=False,
                                      keep_last=0)
                print "write:%s block_path:%s" % (len(codew), block_path)
            else:
                print "write error:%s block_path:%s" % (len(codew), block_path)

        # df['date'] = df['date'].apply(lambda x:(x.replace('-','')))
        # df['date'] = df['date'].astype(int)
        # print df.loc[code_select].T,df.shape
        MultiIndex = False
    else:
        MultiIndex = True
    h5a.write_hdf_db('all300',
                     df,
                     table='roll200',
                     index=False,
                     baseCount=500,
                     append=False,
                     MultiIndex=MultiIndex)
    return df