def loadMcCsv(fileName, dbName, symbol): """将Multicharts导出的csv格式的历史数据插入到Mongo数据库中""" import csv start = time() print u'开始读取CSV文件%s中的数据插入到%s的%s中' % (fileName, dbName, symbol) # 锁定集合,并创建索引 host, port = loadMongoSetting() client = pymongo.MongoClient(host, port) collection = client[dbName][symbol] collection.ensure_index([('datetime', pymongo.ASCENDING)], unique=True) # 读取数据和插入到数据库 reader = csv.DictReader(file(fileName, 'r')) for d in reader: bar = CtaBarData() bar.vtSymbol = symbol bar.symbol = symbol bar.open = float(d['Open']) bar.high = float(d['High']) bar.low = float(d['Low']) bar.close = float(d['Close']) bar.date = datetime.strptime(d['Date'], '%Y/%m/%d').strftime('%Y%m%d') bar.time = d['Time'] bar.datetime = datetime.strptime(bar.date + ' ' + bar.time, '%Y%m%d %H:%M:%S') bar.volume = d['TotalVolume'] flt = {'datetime': bar.datetime} collection.update_one(flt, {'$set': bar.__dict__}, upsert=True) print bar.date, bar.time print u'插入完毕,耗时:%s' % (time() - start)
def dbConnect(self): """连接MongoDB数据库""" if not self.dbClient: # 读取MongoDB的设置 host, port, logging = loadMongoSetting() try: # 设置MongoDB操作的超时时间为0.5秒 self.dbClient = MongoClient(host, port, connectTimeoutMS=500) # 调用server_info查询服务器状态,防止服务器异常并未连接成功 self.dbClient.server_info() self.writeLog(text.DATABASE_CONNECTING_COMPLETED) # 连接状态 self.db_has_connected = True # 如果启动日志记录,则注册日志事件监听函数 # if logging: # self.eventEngine.register(EVENT_LOG, self.dbLogging) except ConnectionFailure: self.dbClient = None self.writeError(text.DATABASE_CONNECTING_FAILED) self.db_has_connected = False
def loadMcCsv(fileName, dbName, symbol): """将Multicharts导出的csv格式的历史数据插入到Mongo数据库中""" import csv start = time() print( u'开始读取CSV文件%s中的数据插入到%s的%s中' %(fileName, dbName, symbol)) # 锁定集合,并创建索引 host, port = loadMongoSetting() client = pymongo.MongoClient(host, port) collection = client[dbName][symbol] collection.ensure_index([('datetime', pymongo.ASCENDING)], unique=True) # 读取数据和插入到数据库 f = open(fileName,'r',encoding='utf8') reader = csv.DictReader(f) for d in reader: bar = CtaBarData() bar.vtSymbol = symbol bar.symbol = symbol bar.open = float(d['Open']) bar.high = float(d['High']) bar.low = float(d['Low']) bar.close = float(d['Close']) bar.date = datetime.strptime(d['Date'], '%Y/%m/%d').strftime('%Y%m%d') bar.time = d['Time'] bar.datetime = datetime.strptime(bar.date + ' ' + bar.time, '%Y%m%d %H:%M:%S') bar.volume = d['TotalVolume'] flt = {'datetime': bar.datetime} collection.update_one(flt, {'$set':bar.__dict__}, upsert=True) print( bar.date, bar.time) print( u'插入完毕,耗时:%s' % (time()-start))
def dbConnect(self, force=False): if (self.dbClient is None or self.db_has_connected == False) or force: host, port, logging = loadMongoSetting() print('connecting to Mongo:{}:{}'.format(host, port)) try: # 设置MongoDB操作的超时时间为0.5秒 self.dbClient = MongoClient(host, port, connectTimeoutMS=500) # 调用server_info查询服务器状态,防止服务器异常并未连接成功 self.dbClient.server_info() self.db_has_connected = True print('mongo db connected') except Exception as ex: self.dbClient = None self.db_has_connected = False self.writeError(u'connect to mongo {}:{} exception:{}'.format( host, port, str(ex)))
def __init__(self): """Constructor""" host, port = loadMongoSetting() self.dbClient = pymongo.MongoClient(host, port) self.datayesClient = DatayesClient()
def __init__(self): """Constructor""" host, port = loadMongoSetting() self.dbClient = pymongo.MongoClient(host, port) self.datayesClient = DatayesClient()