def init_task(self):
config = self.config
dest_dir = config['hdf5']['dir']
sqlite_file_name = dest_dir + "/stock.db"
self.tasks = []
if self.config.getboolean('weight', 'enable', fallback=False):
self.tasks.append(
ImportWeightToSqliteTask(self.queue, sqlite_file_name,
dest_dir))
if self.config.getboolean('finance', 'enable', fallback=False):
self.tasks.append(ImportHistoryFinanceTask(self.queue, dest_dir))
task_count = 0
if self.config.getboolean('ktype', 'day', fallback=False):
task_count += 2
if self.config.getboolean('ktype', 'min5', fallback=False):
task_count += 2
if self.config.getboolean('ktype', 'min', fallback=False):
task_count += 2
if self.config.getboolean('ktype', 'trans', fallback=False):
task_count += 2
if self.config.getboolean('ktype', 'time', fallback=False):
task_count += 2
self.send_message(['INFO', '搜索通达信服务器'])
self.hosts = search_best_tdx()
if not self.hosts:
self.send_message(['INFO', '无法连接通达信行情服务器！请检查网络设置！'])
return
if task_count == 0:
return
use_tdx_number = min(
task_count, len(self.hosts),
self.config.getint('pytdx', 'use_tdx_number', fallback=10))
split = task_count // use_tdx_number
use_hosts = []
for i in range(use_tdx_number):
for j in range(split):
use_hosts.append((self.hosts[i][2], self.hosts[i][3]))
if len(use_hosts) < task_count:
for i in range(task_count - len(use_hosts)):
use_hosts.insert(0, (self.hosts[0][2], self.hosts[0][3]))
# for i in range(len(use_hosts)):
# print(i, use_hosts[i])
cur_host = 0
# 以下按数据量从大到小依次使用速度从高到低的TDX服务器
if self.config.getboolean('ktype', 'trans', fallback=False):
today = datetime.date.today()
trans_start_date = datetime.datetime.strptime(
config['ktype']['trans_start_date'], '%Y-%m-%d').date()
trans_max_days = (today - trans_start_date).days + 1
self.tasks.append(
ImportPytdxTransToH5(self.queue, sqlite_file_name, 'SH',
self.quotations, use_hosts[cur_host][0],
use_hosts[cur_host][1], dest_dir,
trans_max_days))
cur_host += 1
self.tasks.append(
ImportPytdxTransToH5(self.queue, sqlite_file_name, 'SZ',
self.quotations, use_hosts[cur_host][0],
use_hosts[cur_host][1], dest_dir,
trans_max_days))
cur_host += 1
if self.config.getboolean('ktype', 'min', fallback=False):
start_date = datetime.datetime.strptime(
config['ktype']['min_start_date'], '%Y-%m-%d').date()
self.tasks.append(
ImportPytdxToH5(
self.queue, sqlite_file_name, 'SH', '1MIN',
self.quotations, use_hosts[cur_host][0],
use_hosts[cur_host][1], dest_dir,
start_date.year * 100000000 + start_date.month * 1000000 +
start_date.day * 10000))
cur_host += 1
self.tasks.append(
ImportPytdxToH5(
self.queue, sqlite_file_name, 'SZ', '1MIN',
self.quotations, use_hosts[cur_host][0],
use_hosts[cur_host][1], dest_dir,
start_date.year * 100000000 + start_date.month * 1000000 +
start_date.day * 10000))
cur_host += 1
if self.config.getboolean('ktype', 'time', fallback=False):
today = datetime.date.today()
time_start_date = datetime.datetime.strptime(
config['ktype']['time_start_date'], '%Y-%m-%d').date()
time_max_days = (today - time_start_date).days + 1
self.tasks.append(
ImportPytdxTimeToH5(self.queue, sqlite_file_name, 'SH',
self.quotations, use_hosts[cur_host][0],
use_hosts[cur_host][1], dest_dir,
time_max_days))
cur_host += 1
self.tasks.append(
ImportPytdxTimeToH5(self.queue, sqlite_file_name, 'SZ',
self.quotations, use_hosts[cur_host][0],
use_hosts[cur_host][1], dest_dir,
time_max_days))
cur_host += 1
if self.config.getboolean('ktype', 'min5', fallback=False):
start_date = datetime.datetime.strptime(
config['ktype']['min5_start_date'], '%Y-%m-%d').date()
self.tasks.append(
ImportPytdxToH5(
self.queue, sqlite_file_name, 'SH', '5MIN',
self.quotations, use_hosts[cur_host][0],
use_hosts[cur_host][1], dest_dir,
start_date.year * 100000000 + start_date.month * 1000000 +
start_date.day * 10000))
cur_host += 1
self.tasks.append(
ImportPytdxToH5(
self.queue, sqlite_file_name, 'SZ', '5MIN',
self.quotations, use_hosts[cur_host][0],
use_hosts[cur_host][1], dest_dir,
start_date.year * 100000000 + start_date.month * 1000000 +
start_date.day * 10000))
cur_host += 1
if self.config.getboolean('ktype', 'day', fallback=False):
start_date = datetime.datetime.strptime(
config['ktype']['day_start_date'], '%Y-%m-%d').date()
self.tasks.append(
ImportPytdxToH5(
self.queue, sqlite_file_name, 'SH', 'DAY', self.quotations,
use_hosts[cur_host][0], use_hosts[cur_host][1], dest_dir,
start_date.year * 100000000 + start_date.month * 1000000 +
start_date.day * 10000))
cur_host += 1
self.tasks.append(
ImportPytdxToH5(
self.queue, sqlite_file_name, 'SZ', 'DAY', self.quotations,
use_hosts[cur_host][0], use_hosts[cur_host][1], dest_dir,
start_date.year * 100000000 + start_date.month * 1000000 +
start_date.day * 10000))
cur_host += 1
def __call__(self):
total_count = 0
try:
if self.config.getboolean('hdf5', 'enable', fallback=True):
sqlite_file = "{}/stock.db".format(self.config['hdf5']['dir'])
connect = sqlite3.connect(sqlite_file, timeout=1800)
pytdx_import_weight = pytdx_import_weight_to_sqlite
self.logger.debug('use sqlite import weight')
else:
db_config = {
'user': self.config['mysql']['usr'],
'password': self.config['mysql']['pwd'],
'host': self.config['mysql']['host'],
'port': self.config['mysql']['port']
}
connect = mysql.connector.connect(**db_config)
pytdx_import_weight = pytdx_import_weight_to_mysql
self.logger.debug('use mysql import weight')
except Exception as e:
#self.queue.put([self.msg_name, str(e), -1, 0, total_count])
self.queue.put([self.msg_name, 'INFO', str(e), 0, 0])
self.queue.put([self.msg_name, '', 0, None, total_count])
return
try:
"""
download_dir = self.dest_dir + "/downloads"
if not os.path.lexists(download_dir):
os.makedirs(download_dir)
self.queue.put([self.msg_name, '正在下载钱龙权息信息...', 0, 0, 0])
net_file = urllib.request.urlopen(
'http://www.qianlong.com.cn/download/history/weight.rar', timeout=60
)
buffer = net_file.read()
self.queue.put([self.msg_name, '钱龙权息信息下载完成，正在校验是否存在更新...', 0, 0, 0])
new_md5 = hashlib.md5(buffer).hexdigest()
dest_filename = download_dir + '/weight.rar'
old_md5 = None
if os.path.exists(dest_filename):
with open(dest_filename, 'rb') as oldfile:
old_md5 = hashlib.md5(oldfile.read()).hexdigest()
#如果没变化不需要解压导入
if new_md5 != old_md5:
with open(dest_filename, 'wb') as file:
file.write(buffer)
self.queue.put([self.msg_name, '正在解压钱龙权息信息...', 0, 0, 0])
x = os.system('unrar x -o+ -inul {} {}'.format(dest_filename, download_dir))
if x != 0:
raise Exception("无法找到unrar命令！")
self.queue.put([self.msg_name, '正在导入钱龙权息数据...', 0, 0, 0])
total_count = qianlong_import_weight(connect, download_dir + '/weight', 'SH')
total_count += qianlong_import_weight(connect, download_dir + '/weight', 'SZ')
self.queue.put([self.msg_name, '导入钱龙权息数据完毕!', 0, 0, total_count])
else:
self.queue.put([self.msg_name, '钱龙权息数据无变化', 0, 0, 0])
"""
hosts = search_best_tdx()
api = TdxHq_API()
api.connect(hosts[0][2], hosts[0][3])
self.queue.put([self.msg_name, '正在导入权息数据...', 0, 0, 0])
total_count = pytdx_import_weight(api, connect, "SH")
total_count += pytdx_import_weight(api, connect, "SZ")
self.queue.put([self.msg_name, '导入权息数据完毕!', 0, 0, total_count])
#self.queue.put([self.msg_name, '下载通达信财务信息(上证)...', 0, 0, 0])
#x = pytdx_import_finance(connect, api, "SH")
#self.queue.put([self.msg_name, '下载通达信财务信息(深证)...', 0, 0, 0])
#x += pytdx_import_finance(connect, api, "SZ")
#self.queue.put([self.msg_name, '导入通达信财务信息完毕!', 0, 0, x])
api.disconnect()
except Exception as e:
self.logger.error(e)
#self.queue.put([self.msg_name, str(e), -1, 0, total_count])
self.queue.put([self.msg_name, 'INFO', str(e), 0, 0])
finally:
connect.commit()
connect.close()
self.queue.put([self.msg_name, '', 0, None, total_count])
def __call__(self):
total_count = 0
try:
connect = sqlite3.connect(self.sqlitefile)
except Exception as e:
#self.queue.put([self.msg_name, str(e), -1, 0, total_count])
self.queue.put([self.msg_name, 'INFO', str(e), 0, 0])
self.queue.put([self.msg_name, '', 0, None, total_count])
return
try:
download_dir = self.dest_dir + "/downloads"
if not os.path.lexists(download_dir):
os.makedirs(download_dir)
self.queue.put([self.msg_name, '正在下载钱龙权息信息...', 0, 0, 0])
net_file = urllib.request.urlopen(
'http://www.qianlong.com.cn/download/history/weight.rar',
timeout=60)
buffer = net_file.read()
self.queue.put(
[self.msg_name, '钱龙权息信息下载完成，正在校验是否存在更新...', 0, 0, 0])
new_md5 = hashlib.md5(buffer).hexdigest()
dest_filename = download_dir + '/weight.rar'
old_md5 = None
if os.path.exists(dest_filename):
with open(dest_filename, 'rb') as oldfile:
old_md5 = hashlib.md5(oldfile.read()).hexdigest()
#如果没变化不需要解压导入
if new_md5 != old_md5:
with open(dest_filename, 'wb') as file:
file.write(buffer)
self.queue.put([self.msg_name, '正在解压钱龙权息信息...', 0, 0, 0])
x = os.system('unrar x -o+ -inul {} {}'.format(
dest_filename, download_dir))
if x != 0:
raise Exception("无法找到unrar命令！")
self.queue.put([self.msg_name, '正在导入钱龙权息数据...', 0, 0, 0])
total_count = qianlong_import_weight(connect,
download_dir + '/weight',
'SH')
total_count += qianlong_import_weight(connect,
download_dir + '/weight',
'SZ')
self.queue.put(
[self.msg_name, '导入钱龙权息数据完毕!', 0, 0, total_count])
else:
self.queue.put([self.msg_name, '钱龙权息数据无变化', 0, 0, 0])
hosts = search_best_tdx()
api = TdxHq_API()
api.connect(hosts[0][2], hosts[0][3])
self.queue.put([self.msg_name, '下载通达信权息信息(上证)...', 0, 0, 0])
x = pytdx_import_finance(connect, api, "SH")
self.queue.put([self.msg_name, '下载通达信权息信息(深证)...', 0, 0, 0])
x += pytdx_import_finance(connect, api, "SZ")
self.queue.put([self.msg_name, '导入通达信权息信息完毕!', 0, 0, x])
api.disconnect()
except Exception as e:
#self.queue.put([self.msg_name, str(e), -1, 0, total_count])
self.queue.put([self.msg_name, 'INFO', str(e), 0, 0])
finally:
connect.commit()
connect.close()
self.queue.put([self.msg_name, '', 0, None, total_count])
def init_task(self):
config = self.config
dest_dir = config['hdf5']['dir']
sqlite_file_name = dest_dir + "/stock.db"
self.tasks = []
if self.config.getboolean('weight', 'enable', fallback=False):
self.tasks.append(ImportWeightToSqliteTask(self.queue, sqlite_file_name, dest_dir))
if self.config.getboolean('finance', 'enable', fallback=False):
self.tasks.append(ImportHistoryFinanceTask(self.queue, dest_dir))
task_count = 0
if self.config.getboolean('ktype', 'day', fallback=False):
task_count += 2
if self.config.getboolean('ktype', 'min5', fallback=False):
task_count += 2
if self.config.getboolean('ktype', 'min', fallback=False):
task_count += 2
if self.config.getboolean('ktype', 'trans', fallback=False):
task_count += 2
if self.config.getboolean('ktype', 'time', fallback=False):
task_count += 2
self.send_message(['INFO', '搜索通达信服务器'])
self.hosts = search_best_tdx()
if not self.hosts:
self.send_message(['INFO', '无法连接通达信行情服务器！请检查网络设置！'])
return
if task_count == 0:
return
use_tdx_number = min(task_count, len(self.hosts), self.config.getint('pytdx', 'use_tdx_number', fallback=10))
split = task_count // use_tdx_number
use_hosts = []
for i in range(use_tdx_number):
for j in range(split):
use_hosts.append((self.hosts[i][2], self.hosts[i][3]))
if len(use_hosts) < task_count:
for i in range(task_count - len(use_hosts)):
use_hosts.insert(0, (self.hosts[0][2], self.hosts[0][3]))
# for i in range(len(use_hosts)):
# print(i, use_hosts[i])
cur_host = 0
# 以下按数据量从大到小依次使用速度从高到低的TDX服务器
if self.config.getboolean('ktype', 'trans', fallback=False):
today = datetime.date.today()
trans_start_date = datetime.datetime.strptime(config['ktype']['trans_start_date'], '%Y-%m-%d').date()
trans_max_days = (today - trans_start_date).days + 1
self.tasks.append(
ImportPytdxTransToH5(
self.queue, sqlite_file_name, 'SH', self.quotations,
use_hosts[cur_host][0], use_hosts[cur_host][1],
dest_dir, trans_max_days
)
)
cur_host += 1
self.tasks.append(
ImportPytdxTransToH5(
self.queue, sqlite_file_name, 'SZ', self.quotations,
use_hosts[cur_host][0], use_hosts[cur_host][1],
dest_dir, trans_max_days
)
)
cur_host += 1
if self.config.getboolean('ktype', 'min', fallback=False):
start_date = datetime.datetime.strptime(config['ktype']['min_start_date'], '%Y-%m-%d').date()
self.tasks.append(
ImportPytdxToH5(
self.queue, sqlite_file_name, 'SH', '1MIN', self.quotations,
use_hosts[cur_host][0], use_hosts[cur_host][1],
dest_dir, start_date.year * 100000000 + start_date.month * 1000000 + start_date.day * 10000
)
)
cur_host += 1
self.tasks.append(
ImportPytdxToH5(
self.queue, sqlite_file_name,
'SZ', '1MIN', self.quotations,
use_hosts[cur_host][0], use_hosts[cur_host][1],
dest_dir, start_date.year * 100000000 + start_date.month * 1000000 + start_date.day * 10000
)
)
cur_host += 1
if self.config.getboolean('ktype', 'time', fallback=False):
today = datetime.date.today()
time_start_date = datetime.datetime.strptime(config['ktype']['time_start_date'], '%Y-%m-%d').date()
time_max_days = (today - time_start_date).days + 1
self.tasks.append(ImportPytdxTimeToH5(self.queue, sqlite_file_name, 'SH',
self.quotations,
use_hosts[cur_host][0], use_hosts[cur_host][1],
dest_dir,
time_max_days))
cur_host += 1
self.tasks.append(ImportPytdxTimeToH5(self.queue, sqlite_file_name, 'SZ',
self.quotations,
use_hosts[cur_host][0], use_hosts[cur_host][1],
dest_dir,
time_max_days))
cur_host += 1
if self.config.getboolean('ktype', 'min5', fallback=False):
start_date = datetime.datetime.strptime(config['ktype']['min5_start_date'], '%Y-%m-%d').date()
self.tasks.append(
ImportPytdxToH5(
self.queue, sqlite_file_name, 'SH', '5MIN',
self.quotations,
use_hosts[cur_host][0], use_hosts[cur_host][1],
dest_dir,
start_date.year * 100000000 + start_date.month * 1000000 + start_date.day * 10000
)
)
cur_host += 1
self.tasks.append(
ImportPytdxToH5(
self.queue, sqlite_file_name, 'SZ', '5MIN',
self.quotations,
use_hosts[cur_host][0], use_hosts[cur_host][1],
dest_dir,
start_date.year * 100000000 + start_date.month * 1000000 + start_date.day * 10000
)
)
cur_host += 1
if self.config.getboolean('ktype', 'day', fallback=False):
start_date = datetime.datetime.strptime(config['ktype']['day_start_date'], '%Y-%m-%d').date()
self.tasks.append(
ImportPytdxToH5(
self.queue, sqlite_file_name, 'SH', 'DAY',
self.quotations,
use_hosts[cur_host][0], use_hosts[cur_host][1],
dest_dir,
start_date.year * 100000000 + start_date.month * 1000000 + start_date.day * 10000
)
)
cur_host += 1
self.tasks.append(
ImportPytdxToH5(
self.queue, sqlite_file_name, 'SZ', 'DAY',
self.quotations,
use_hosts[cur_host][0], use_hosts[cur_host][1],
dest_dir,
start_date.year * 100000000 + start_date.month * 1000000 + start_date.day * 10000
)
)
cur_host += 1
def __call__(self):
total_count = 0
try:
connect = sqlite3.connect(self.sqlitefile)
except Exception as e:
#self.queue.put([self.msg_name, str(e), -1, 0, total_count])
self.queue.put([self.msg_name, 'INFO', str(e), 0, 0])
self.queue.put([self.msg_name, '', 0, None, total_count])
return
try:
download_dir = self.dest_dir + "/downloads"
if not os.path.lexists(download_dir):
os.makedirs(download_dir)
self.queue.put([self.msg_name, '正在下载钱龙权息信息...', 0, 0, 0])
net_file = urllib.request.urlopen('http://www.qianlong.com.cn/download/history/weight.rar', timeout=60)
buffer = net_file.read()
self.queue.put([self.msg_name, '钱龙权息信息下载完成，正在校验是否存在更新...', 0, 0, 0])
new_md5 = hashlib.md5(buffer).hexdigest()
dest_filename = download_dir + '/weight.rar'
old_md5 = None
if os.path.exists(dest_filename):
with open(dest_filename, 'rb') as oldfile:
old_md5 = hashlib.md5(oldfile.read()).hexdigest()
#如果没变化不需要解压导入
if new_md5 != old_md5:
with open(dest_filename, 'wb') as file:
file.write(buffer)
self.queue.put([self.msg_name, '正在解压钱龙权息信息...', 0, 0, 0])
x = os.system('unrar x -o+ -inul {} {}'.format(dest_filename, download_dir))
if x != 0:
raise Exception("无法找到unrar命令！")
self.queue.put([self.msg_name, '正在导入钱龙权息数据...', 0, 0, 0])
total_count = qianlong_import_weight(connect, download_dir + '/weight', 'SH')
total_count += qianlong_import_weight(connect, download_dir + '/weight', 'SZ')
self.queue.put([self.msg_name, '导入钱龙权息数据完毕!', 0, 0, total_count])
else:
self.queue.put([self.msg_name, '钱龙权息数据无变化', 0, 0, 0])
hosts = search_best_tdx()
api = TdxHq_API()
api.connect(hosts[0][2], hosts[0][3])
self.queue.put([self.msg_name, '下载通达信权息信息(上证)...', 0, 0, 0])
x = pytdx_import_finance(connect, api, "SH")
self.queue.put([self.msg_name, '下载通达信权息信息(深证)...', 0, 0, 0])
x += pytdx_import_finance(connect, api, "SZ")
self.queue.put([self.msg_name, '导入通达信权息信息完毕!', 0, 0, x])
api.disconnect()
except Exception as e:
#self.queue.put([self.msg_name, str(e), -1, 0, total_count])
self.queue.put([self.msg_name, 'INFO', str(e), 0, 0])
finally:
connect.commit()
connect.close()
self.queue.put([self.msg_name, '', 0, None, total_count])
