Exemplo n.º 1
0
def getTradingDates():
    '''Get futures trading dates.

Returns
-------
tradingDates : list of datetime.date
    return list of trading dates in datetime.date.

Exceptions
----------
    raise Exception when duplicated records found.
    '''
    username, password = config.MONGODB_CRED
    db = dMongodb.getAuthenticatedConnection(config.MONGODB_URL,
                                             config.MONGODB_PORT, username,
                                             password, 'universe')

    cursor = db.refdata.find({'Name': 'TradingDates', 'Country': 'CN'})
    nRecords = cursor.count()
    if nRecords == 0:
        tradingDates = None
    elif nRecords > 1:
        raise Exception('Duplicated {n:d} records found.'.format(n=nRecords))
    else:
        data = cursor.next()
        tradingDates = [d.date() for d in data['Data']]

    return tradingDates
Exemplo n.º 2
0
def updateFuturesUniverse( asOfDate ):
    '''Update futures universe data.

Parameters
----------
asOfDate : datetime.date
    Data date.

Returns
-------
None
    '''
    futData = []

    for exch in futures.FUTURES_EXCHANGES:
        logging.info( 'Get futures contracts in {e:s}...'.format( e=exch ) )
        exchFutures = futures.getFuturesContracts( exch )
        futData.append( exchFutures )

    futData = pd.concat( futData )
    futData.reset_index( drop=True, inplace=True )

    # write to MongoDB
    username, password = MONGODB_CRED
    db = mongodb.getAuthenticatedConnection( MONGODB_URL, MONGODB_PORT,
                                              username, password, DB_NAME )

    mongoDate = dt.datetime.combine( asOfDate,
                           dt.datetime.min.time() )
    record = { 'Date': mongoDate,
               'Data': futData.to_json(),
               'Country': 'CN' }

    db.futures.update( { 'Date': mongoDate, 'Country': 'CN' }, record, upsert=True )
Exemplo n.º 3
0
def updateStockUniverse( asOfDate ):
    '''Update stock universe data.

Parameters
----------
asOfDate : datetime.date
    Data date.

Returns
-------
None
    '''
    # get the universe data.
    industryClassification = stocks.getIndustryClassification()
    stockInfo              = stocks.getStocks()

    # extract MongoDB credential
    username, password = MONGODB_CRED
    db = mongodb.getAuthenticatedConnection( MONGODB_URL, MONGODB_PORT,
                                             username, password, DB_NAME )
    # write to MongoDB
    # the data is enhanced with an asOfDate date and the market identifier
    mongoDate = dt.datetime.combine( asOfDate,
                           dt.datetime.min.time() )
    record = { 'Date': mongoDate,
               'Stocks':  stockInfo.to_json(),
               'Sectors': industryClassification.to_json(),
               'Country': 'CN' }

    db.stocks.update( { 'Date': mongoDate, 'Country': 'CN' }, record, upsert=True )
Exemplo n.º 4
0
def main():
    '''Entry point of the job.
    '''
    # runtime date
    asOfDate = dt.date.today()

    # get all stocks in the universe
    universe = stockApi.getExchangeStockNames(asOfDate)
    universe.sort()
    # idx = universe.index( '603085.XSHG' )
    universe = universe[idx:]

    # initialize MongoDB connection
    username, password = config.MONGODB_CRED
    db = mongodb.getAuthenticatedConnection(config.MONGODB_URL,
                                            config.MONGODB_PORT, username,
                                            password, 'binData')

    nStocks = len(universe)
    logging.info(
        'Backfill bin volume for {ns:d} stocks in total...'.format(ns=nStocks))

    # for bin data backfill, stocks are updated one-by-one and by month
    for i, s in enumerate(universe):
        for startDate, endDate in DATE_RANGE:
            logging.info(
                'Backfilling bin volume for {sec:s} ({idx:d}/{n:d}) from {sd:s} to {ed:s}...'
                .format(sec=s,
                        idx=i + 1,
                        n=nStocks,
                        sd=str(startDate),
                        ed=str(endDate)))
            data = stockTrading.getHistoryBinData(s,
                                                  startDate=startDate,
                                                  endDate=endDate)

            try:
                groupedData = data.groupby(['dataDate'])
                records = []

                for dataDate, tData in groupedData:
                    record = {
                        'SecID': s,
                        'Date': dt.datetime.strptime(dataDate, '%Y-%m-%d'),
                        'Data': tData.to_json(),
                        'Country': 'CN'
                    }
                    records.append(record)

                db.stocks.insert_many(records)
            except KeyError as e:
                logging.warning(
                    'Error when updating {sec:s} from {sd:s} to {ed:s}: {msg:s}.'
                    .format(sec=s,
                            sd=str(startDate),
                            ed=str(endDate),
                            msg=str(e)))

    logging.info('Bin data backfill done.')
Exemplo n.º 5
0
def getDailyData(secId, startDate=dt.date(2012, 1, 1),
                 endDate=dt.date.today()):
    '''Get daily data for the given futures during the date range.

Parameters
----------
secId : str
    Security ID of the futures;
startDate : datetime.date
    Start date of the daily data queried inclusively;
endDate : datetime.date
    End date of the daily data queried inclusively.

Returns
-------
dailyData : pandas.DataFrame
    Requested daily data in pandas.DataFrame.

Exceptions
----------
    raise Exception when duplicated records found on the given futures name.
    '''
    # Get authenticated MongoDB connection
    username, password = config.MONGODB_CRED
    db = dMongodb.getAuthenticatedConnection(config.MONGODB_URL,
                                             config.MONGODB_PORT, username,
                                             password, 'dailyData')

    # Query data
    cursor = db.futures.find({'SecID': secId})

    # Sanity check
    nRecords = cursor.count()
    if nRecords == 0:
        dailyData = None
    elif nRecords > 1:
        raise Exception(
            'Duplicated {n:d} records found for futures {s:s}.'.format(
                n=nRecords, s=secId))
    else:
        data = cursor.next()
        dailyData = pd.read_json(data['Data'])
        dailyData.sort_index(inplace=True)

        # Filtered by date
        startDateStr = startDate.strftime('%Y-%m-%d')
        endDateStr = endDate.strftime('%Y-%m-%d')

        greaterDates = dailyData.tradeDate >= startDateStr
        smallerDates = dailyData.tradeDate <= endDateStr
        dailyData = dailyData[np.logical_and(greaterDates, smallerDates)]
        # reset index from 0 onwards
        dailyData.reset_index(drop=True, inplace=True)

    return dailyData
Exemplo n.º 6
0
def getFuturesInformation(asOfDate, ticker=None, listed=True, country='CN'):
    '''Get all futures contract information.

Parameters
----------
asOfDate : datetime.date
    Data date of the futures information or
    None if not found;
ticker : str
    Ticker name of the futures, if not given, return all futures;
listed : bool or None
    Contract listed status, True for listed contracts by the asOfDate, otherwise return all available
    contracts;
country : str
    Country identifier, currently, only CN supported.

Returns
-------
futuresInfo : pandas.DataFrame
    Futures information.

Exceptions
----------
    raise Exception when duplicated records found.
    '''
    username, password = config.MONGODB_CRED
    db = dMongodb.getAuthenticatedConnection(config.MONGODB_URL,
                                             config.MONGODB_PORT, username,
                                             password, 'universe')

    # find the latest records
    validDate = dt.datetime(2020, 1, 1)
    data = db.futures.find_one({
        'Date': {
            '$lte': validDate
        },
        'Country': 'CN'
    },
                               sort=[('Date', dMongodb.pymongo.DESCENDING)])

    futuresInfo = pd.read_json(data['Data'])
    futuresInfo.sort_index(inplace=True)

    if ticker is not None:
        futuresInfo = futuresInfo[futures.ticker == ticker]
        futuresInfo.reset_index(drop=True, inplace=True)

    if listed:
        sAsOfDate = str(asOfDate)
        isListed = np.logical_and(futuresInfo.listDate <= sAsOfDate,
                                  futuresInfo.lastTradeDate >= sAsOfDate)
        futuresInfo = futuresInfo[isListed]
        futuresInfo.reset_index(drop=True, inplace=True)

    return futuresInfo
Exemplo n.º 7
0
def main():
    '''Entry point of the job.
    '''
    # runtime
    asOfDate = dt.date.today()
    logging.info(
        'Updating minute bin data for futures on date {d:s}...'.format(
            d=str(asOfDate)))

    # get all futures in the universe
    futuresInfo = futuresApi.getFuturesInformation(asOfDate)
    universe = dict(zip(futuresInfo.ticker, futuresInfo.secID))

    # initialize MongoDB connection
    username, password = config.MONGODB_CRED
    db = mongodb.getAuthenticatedConnection(config.MONGODB_URL,
                                            config.MONGODB_PORT, username,
                                            password, 'binData')

    nFutures = len(universe)
    logging.info(
        'Minute bin volume for {ns:d} futures in total to be updated...')

    # for bin data, futures are updated one-by-one
    for i, ids in enumerate(universe.items()):
        futures, secId = ids
        futures = futures.upper()
        logging.info(
            'Updating minute bin data for {s:s} ({idx:d}/{n:d})...'.format(
                s=secId, idx=i + 1, n=nFutures))

        data = futuresTrading.getBinData(futures, dataDate=asOfDate)

        if len(data) > 0:
            mongoDate = dt.datetime.combine(asOfDate, dt.datetime.min.time())
            record = {
                'SecID': secId,
                'Date': mongoDate,
                'Data': data.to_json(),
                'Country': 'CN'
            }

            db.futures.update(
                {
                    'SecID': secId,
                    'Date': mongoDate,
                    'Country': 'CN'
                },
                record,
                upsert=True)
        else:
            logging.warning('Empty data for {secId:s}'.format(secId=secId))

    logging.info('All futures updated.')
Exemplo n.º 8
0
def getBinData(secId, startDate=dt.date(2012, 1, 1), endDate=dt.date.today()):
    '''Get minute-by-minute data for the given futures during the date range.

Parameters
----------
secId : str
    Security ID of the futures;
startDate : datetime.date
    Start date of the bin data required inclusively,
endDate : datetime.date
    End date of the bin data required inclusively.

Returns
-------
binData : pandas.Panel
    Requested bin data in pandas.Panel.

Exceptoins
----------
    raise Exception when duplicated records found on the given futures name.
    '''
    # Get authenticated MongoDB connection
    username, password = config.MONGODB_CRED
    db = dMongodb.getAuthenticatedConnection(config.MONGODB_URL,
                                             config.MONGODB_PORT, username,
                                             password, 'binData')

    # Query data
    cursor = db.futures.find({
        'SecID': secId,
        'Date': {
            '$gte': dt.datetime.combine(startDate, dt.datetime.min.time()),
            '$lte': dt.datetime.combine(endDate, dt.datetime.min.time())
        }
    })

    data = {}
    for item in cursor:
        # Build DataFrame's to convert to a Panel.
        date = item['Date'].date()
        if date in data:
            raise Exception(
                'Duplicated records on {d:s} found.'.format(d=str(date)))
        else:
            dayBinData = pd.read_json(item['Data'])
            dayBinData.sort_index(inplace=True)
            data[date] = dayBinData

    return pd.Panel(data)
Exemplo n.º 9
0
def main():
    '''Entry point of the job.
    '''
    # runtime
    asOfDate = dt.date.today()
    logging.info('Updating minute bin data for stocks on date {d:s}...'.format(
        d=str(asOfDate)))

    # get all stocks in the universe
    universe = stockApi.getExchangeStockNames(asOfDate)

    # initialize MongoDB connection
    username, password = config.MONGODB_CRED
    db = mongodb.getAuthenticatedConnection(config.MONGODB_URL,
                                            config.MONGODB_PORT, username,
                                            password, 'binData')

    nStocks = len(universe)
    logging.info(
        'Minute bin volume for {ns:d} stocks in total to be updated...'.format(
            ns=nStocks))

    # for bin data, stocks are updated one-by-one
    for i, stock in enumerate(universe):
        logging.info(
            'Updating minute bin data for {s:s} ({idx:d}/{n:d})...'.format(
                s=stock, idx=i + 1, n=nStocks))

        data = stockTrading.getBinData(stock, dataDate=asOfDate)

        mongoDate = dt.datetime.combine(asOfDate, dt.datetime.min.time())
        record = {
            'SecID': stock,
            'Date': mongoDate,
            'Data': data.to_json(),
            'Country': 'CN'
        }

        db.stocks.update({
            'SecID': stock,
            'Date': mongoDate,
            'Country': 'CN'
        },
                         record,
                         upsert=True)

    logging.info('All stocks updated.')
Exemplo n.º 10
0
def main():
    '''Entry point of the job.
    '''
    # runtime date
    asOfDate = dt.date.today()
    mongoDate = dt.datetime.combine(asOfDate, dt.datetime.min.time())

    # get all stocks in the universe
    universe = stockApi.getExchangeStockNames(asOfDate)

    # initialize MongoDB connetion
    username, password = config.MONGODB_CRED
    db = mongodb.getAuthenticatedConnection(config.MONGODB_URL,
                                            config.MONGODB_PORT, username,
                                            password, 'dailyData')

    nStocks = len(universe)
    logging.info(
        'Daily volume for {ns:d} stocks in total to be updated...'.format(
            ns=nStocks))

    # for daily data, stocks are updated one by one
    for i, s in enumerate(universe):
        logging.info(
            'Updating daily data for {sec:s} ({idx:d}/{n:d})...'.format(
                sec=s, idx=i + 1, n=nStocks))
        data = stockTrading.getAdjustedDailyData(s)

        # compose record
        record = {
            'SecID': s,
            'Data': data.to_json(),
            'LastModified': mongoDate,
            'Country': 'CN'
        }

        # save data to mongo
        db.stocks.update({'SecID': s, 'Country': 'CN'}, record, upsert=True)

    logging.info('Daily data updated done.')
Exemplo n.º 11
0
def main():
    '''Main body of the job.
    '''
    asOfDate = dt.date(2016, 4, 25)
    endDate = dt.datetime.combine(dt.date(2016, 4, 22), dt.datetime.min.time())

    # get all available futures since the very beginning.
    futuresInfo = futuresApi.getFuturesInformation(asOfDate, listed=False)

    # initialize MongoDB connection
    username, password = config.MONGODB_CRED
    db = mongodb.getAuthenticatedConnection(config.MONGODB_URL,
                                            config.MONGODB_PORT, username,
                                            password, 'binData')

    # contract metadata
    contractMeta = list(
        zip(futuresInfo.ticker, futuresInfo.secID, futuresInfo.listDate,
            futuresInfo.lastTradeDate))
    # contractMeta = contractMeta[ 23 : ]
    nContracts = len(contractMeta)

    for i, meta in enumerate(contractMeta):
        instId, secId, listedDate, tradeDate = meta
        logging.info('Processing futures {f:s} ({i:d}/{n:d})...'.format(
            f=secId, i=i + 1, n=nContracts))
        if listedDate <= '2012-01-01':
            logging.warning(
                'Too senior contract for {sec:s}.'.format(sec=secId))
            continue

        # normalize instrument ID
        instId = instId.upper()
        startDate = dt.datetime.strptime(listedDate, '%Y-%m-%d')
        endDate = min(endDate, dt.datetime.strptime(tradeDate, '%Y-%m-%d'))

        curDate = startDate
        records = []
        while curDate <= endDate:
            logging.info(
                'Processing {iid:s} ({i:d}/{n:d}) on {sd:s}...'.format(
                    iid=instId, sd=str(curDate), i=i, n=nContracts))
            try:
                dailyBin = futuresTrading.getBinData(instId, dataDate=curDate)
                if len(dailyBin) > 0:
                    record = {
                        'SecID': secId,
                        'Date': curDate,
                        'Data': dailyBin.to_json(),
                        'Country': 'CN'
                    }
                    records.append(record)
                else:
                    logging.warning('Empty data for {sec:s} on {sd:s}.'.format(
                        sec=secId, sd=str(curDate)))
            except KeyError as e:
                logging.warning(
                    'Error when updating {sec:s} on {sd:s}.'.format(
                        sec=secId, sd=str(curDate)))

            curDate += dt.timedelta(1)

        if len(records) > 0:
            db.futures.insert_many(records)
        else:
            logging.warning(
                'Empty data to be inserted into database for {sec:s}'.format(
                    sec=secId))

    logging.info('Bin data backfill done.')