コード例 #1
0
def import_datayes_daily_data(start_date, end_date, cont_list = [], is_replace = False):
    numdays = (end_date - start_date).days + 1
    date_list = [start_date + datetime.timedelta(days=x) for x in range(0, numdays) ]
    date_list = [ d for d in date_list if (d.weekday()< 5) and (d not in misc.CHN_Holidays)]
    for d in date_list:
        cnt = 0
        dstring = d.strftime('%Y%m%d')
        ts.set_token(misc.datayes_token)
        mkt = ts.Market()
        df = mkt.MktFutd(tradeDate = dstring)
        if len(df.ticker) == 0:
            continue
        for cont in df.ticker:
            if (len(cont_list) > 0) and (cont not in cont_list):
                continue
            data = df[df.ticker==cont]
            if len(data) == 0:
                print 'no data for %s for %s' % (cont, dstring)
            else:
                data_dict = {}
                data_dict['date']  = d
                data_dict['open']  = float(data.openPrice)
                data_dict['close'] = float(data.closePrice)
                data_dict['high']  = float(data.highestPrice)
                data_dict['low'] = float(data.lowestPrice)
                data_dict['volume'] = int(data.turnoverVol)
                data_dict['openInterest'] = int(data.openInt)
                if data_dict['volume'] > 0:
                    cnt += 1
                    db.insert_daily_data(cont, data_dict, is_replace = is_replace, dbtable = 'fut_daily')
        print 'date=%s, insert count = %s' % (d, cnt)
コード例 #2
0
def fix_daily_by_tick(contlist, sdate, edate, is_forced=False):
    res = {}
    for inst in contlist:
        product = misc.inst2product(inst)
        start_tick = 1500000
        end_tick = 2100000
        if product in misc.night_session_markets:
            start_tick = 300000
        elif product in ['IF', 'TF']:
            start_tick = 1515000
            end_tick = 2115000
        ddf = mysqlaccess.load_daily_data_to_df('fut_daily', inst, sdate,
                                                edate)
        tdf = mysqlaccess.load_tick_to_df('fut_tick',
                                          inst,
                                          sdate,
                                          edate,
                                          start_tick=start_tick,
                                          end_tick=end_tick)
        for d in list(set(tdf.date)):
            if (is_forced) or (d not in ddf.index) or (ddf.ix(d, 'open') == 0):
                df = tdf[tdf['date'] == d].sort(['tick_id'])
                ddata = {}
                ddata['date'] = d
                ddata['open'] = float(df.iloc[0].price)
                ddata['close'] = float(df.iloc[-1].price)
                ddata['high'] = float(df.iloc[-1].high)
                ddata['low'] = float(df.iloc[-1].low)
                ddata['volume'] = int(df.iloc[-1].volume)
                ddata['openInterest'] = int(df.iloc[-1].openInterest)
                print inst, ddata
                res[(inst, d)] = ddata
                mysqlaccess.insert_daily_data(inst, ddata, is_forced)
    return res
コード例 #3
0
def fix_daily_data(contlist, sdate, edate):
    for inst in contlist:
        ddf = mysqlaccess.load_daily_data_to_df('fut_daily', inst, sdate,
                                                edate)
        mdf = mysqlaccess.load_min_data_to_df('fut_min',
                                              inst,
                                              sdate,
                                              edate,
                                              minid_start=300,
                                              minid_end=2115)
        dailydata = data_handler.conv_ohlc_freq(mdf, 'D')
        for dd in dailydata.index:
            d = dd.date()
            dslice = dailydata.ix[dd]
            if d not in ddf.index:
                ddata = {}
                ddata['date'] = d
                ddata['open'] = float(dslice.open)
                ddata['close'] = float(dslice.close)
                ddata['high'] = float(dslice.high)
                ddata['low'] = float(dslice.low)
                ddata['volume'] = int(dslice.volume)
                ddata['openInterest'] = int(dslice.openInterest)
                print inst, ddata
                mysqlaccess.insert_daily_data(inst, ddata)
コード例 #4
0
ファイル: fix_db_data.py プロジェクト: Timelordt/pyktrader
def fix_daily_by_tick(contlist, sdate, edate, is_forced=False):
    res = {}
    for inst in contlist:
        product = misc.inst2product(inst)
        start_tick= 1500000
        end_tick  = 2100000
        if product in misc.night_session_markets:
            start_tick = 300000
        elif product in ['IF','TF']:
            start_tick = 1515000
            end_tick   = 2115000
        ddf = mysqlaccess.load_daily_data_to_df('fut_daily', inst, sdate, edate)
        tdf = mysqlaccess.load_tick_to_df('fut_tick', inst, sdate, edate, start_tick=start_tick, end_tick = end_tick)
        for d in list(set(tdf.date)):
            if (is_forced) or (d not in ddf.index) or (ddf.ix(d, 'open')==0):
                df = tdf[tdf['date']==d].sort(['tick_id'])
                ddata = {}
                ddata['date'] = d
                ddata['open'] = float(df.iloc[0].price)
                ddata['close'] = float(df.iloc[-1].price)
                ddata['high'] = float(df.iloc[-1].high)
                ddata['low'] = float(df.iloc[-1].low)
                ddata['volume'] = int(df.iloc[-1].volume)
                ddata['openInterest'] = int(df.iloc[-1].openInterest)
                print inst, ddata
                res[(inst,d)] = ddata
                mysqlaccess.insert_daily_data(inst, ddata, is_forced)
    return res
コード例 #5
0
ファイル: wind_data.py プロジェクト: yongfuyang/pyktrader2
def load_csv_to_db(edate,
                   save_loc='C:\\dev\\data\\',
                   freq='m',
                   is_replace=False):
    cont_list = misc.filter_main_cont(edate, False)
    if freq not in ['m', 'd']:
        return False
    for cont in cont_list:
        if freq == 'm':
            filename = save_loc + cont + '_min.csv'
        else:
            filename = save_loc + cont + '_daily.csv'
        if not os.path.isfile(filename):
            continue
        data_reader = csv.reader(file(filename, 'rb'))
        mindata_list = []
        for idx, line in enumerate(data_reader):
            if idx > 0:
                if 'nan' in [line[0], line[2], line[3], line[4], line[6]]:
                    continue
                min_data = {}
                min_data['volume'] = int(float(line[0]))
                if min_data['volume'] <= 0: continue
                dtime = datetime.datetime.strptime(line[1],
                                                   '%Y-%m-%d %H:%M:%S.%f')
                if freq == 'm':
                    min_data['datetime'] = dtime.replace(microsecond=0)
                else:
                    min_data['date'] = dtime.date()
                min_data['high'] = float(line[2])
                min_data['low'] = float(line[3])
                min_data['close'] = float(line[4])
                if line[5] == 'nan':
                    oi = 0
                else:
                    oi = int(float(line[5]))
                min_data['openInterest'] = oi
                min_data['open'] = float(line[6])
                if freq == 'm':
                    min_data['min_id'] = get_min_id(dtime)
                    trading_date = dtime.date()
                    if min_data['min_id'] < 600:
                        trading_date = misc.day_shift(trading_date, '1b')
                    min_data['date'] = trading_date
                    mindata_list.append(min_data)
                else:
                    print cont
                    db.insert_daily_data(cont,
                                         min_data,
                                         is_replace=is_replace,
                                         dbtable='fut_daily')
        if freq == 'm':
            print cont
            db.bulkinsert_min_data(cont, mindata_list, is_replace=is_replace)
    return True
コード例 #6
0
 def write_mkt_data(self, event):
     inst = event.dict['instID']
     type = event.dict['type']
     data = event.dict['data']
     if type == EVENT_MIN_BAR:
         mysqlaccess.insert_min_data(inst, data, dbtable = self.min_db_table)
     elif type == EVENT_TICK:
         mysqlaccess.bulkinsert_tick_data(inst, data, dbtable = self.tick_db_table)
     elif type == EVENT_MKTDATA_EOD:
         mysqlaccess.insert_daily_data(inst, data, dbtable = self.daily_db_table)
     else:
         pass
コード例 #7
0
ファイル: tradeagent.py プロジェクト: keel1982/pyktrader
 def write_mkt_data(self, event):
     inst = event.dict['instID']
     type = event.dict['type']
     data = event.dict['data']
     if type == EVENT_MIN_BAR:
         mysqlaccess.insert_min_data(inst, data, dbtable = self.min_db_table)
     elif type == EVENT_TICK:
         mysqlaccess.bulkinsert_tick_data(inst, data, dbtable = self.tick_db_table)
     elif type == EVENT_MKTDATA_EOD:
         mysqlaccess.insert_daily_data(inst, data, dbtable = self.daily_db_table)
     else:
         pass
コード例 #8
0
ファイル: fix_db_data.py プロジェクト: Timelordt/pyktrader
def fix_daily_data(contlist, sdate, edate):
    for inst in contlist:
        ddf = mysqlaccess.load_daily_data_to_df('fut_daily', inst, sdate, edate)
        mdf = mysqlaccess.load_min_data_to_df('fut_min', inst, sdate, edate, minid_start=300, minid_end = 2115)
        dailydata = data_handler.conv_ohlc_freq(mdf, 'D')
        for dd in dailydata.index:
            d = dd.date()
            dslice = dailydata.ix[dd]
            if d not in ddf.index:
                ddata = {}
                ddata['date'] = d
                ddata['open'] = float(dslice.open)
                ddata['close'] = float(dslice.close)
                ddata['high'] = float(dslice.high)
                ddata['low'] = float(dslice.low)
                ddata['volume'] = int(dslice.volume)
                ddata['openInterest'] = int(dslice.openInterest)
                print inst, ddata
                mysqlaccess.insert_daily_data(inst, ddata)