def crack(self, *args, **kwargs):
ip = args[1]['ip']
port = args[1]['port']
service = args[1]['service']
if not ip or not port or not service:
return None
userpath = '%s/%s' % (self.__get_crack_dic_path(), self.user_dict)
passpath = '%s/%s' % (self.__get_crack_dic_path(), self.pass_dict)
self.command = "%s -h %s -n %s -U %s -P %s -e ns -M %s -f -v 4 -t 16 -R 0" % (self.medusa_script, ip, str(port), userpath, passpath, service)
print self.command
start = datetime.datetime.now()
process = subprocess.Popen(self.command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
while process.poll() is None:
time.sleep(0.5)
now = datetime.datetime.now()
if (now - start).seconds > self.timeout:
try:
print process.pid
self.normal_exit = False
#process.terminate() #这里就不能在return 了
logger.info("medusa will be stopped because of crack [%s:%s] time out." % (ip, str(port)))
except Exception,e:
logger.error('Exception:%s' % str(e))
process.terminate() #这里就不能在return 了
process.kill()
time.sleep(2)
def async_deal_into_db(self, bdomain, taskid):
'''
入库
:return:
'''
while not self.threadpool.resultQueue.empty():
try:
result_dit = self.threadpool.resultQueue.get(block = False)
figerinfo = result_dit['figerinfo']
exist_result_list = result_dit['result_list']
for exist_result in exist_result_list:
vulurl = {}
vulurl['sid'] = taskid
vulurl['url'] = exist_result['url']
vulurl['title'] = exist_result['title']
vulurl['keyword'] = ''
vulurl['code'] = exist_result['http_code']
vulurl['figerinfo'] = figerinfo
vulurl['first_time'] = getCurTime()
if bdomain == DOMAIN_TYPE[0]:
#sql = "insert into t_web_vulurl(`sid`, `url`, `title`, `keyword`, `code`, `figerinfo`, `first_time`) values('%s', '%s', '%s', '%s', '%s', '%s', '%s')" % (str(taskid), str(exist_result['url']) , str(exist_result['title']), '', str(exist_result['http_code']), figerinfo, getCurTime())
self.plugin_db.insert_by_dict(WEBVUL_TABLE , vulurl)
elif bdomain == DOMAIN_TYPE[1]:
self.plugin_db.insert_by_dict(WEBIPVUL_TABLE, vulurl)
except Exception,e:
logger.error(str(e))
break
def __del__(self):
'''
析构函数，关闭数据库的连接
'''
try:
self.__conn.close()
except MySQLdb.Error,e:
logger.error('[-]....Mysql Error:%s' % e)
def createConnection(self):
'''
创建数据库连接
'''
if not self.__isMysqlDb():
return False
try:
self.__conn = MySQLdb.connect(host=self.__host,port=self.__port,user=self.__user,passwd=self.__passwd,charset='utf8')
self.__conn.select_db(self.__db)
except MySQLdb.Error,e:
logger.error('[-]......Mysql Error:%s' % e)
#self.__del__()
return False
def __burlaccess(self):
'''
判断目标URL是否可以访问
:return:
'''
try:
request = requests.get(url = self.url, timeout = self.timeout, headers=HTTP_HEADERS, allow_redirects = True, verify = False)
respinfo = HtmlRespInfo(request.status_code, request.text)
self.exist_result_que.put_nowait({'respinfo':respinfo.trans(), 'url':self.url})
try:
self.__whatweb() #获取网站指纹
except Exception,e:
logger.error('[webscan]error:%s' % str(e))
return True
def executeUpdate(self, sql):
'''
对数据库进行添加，修改和删除的操作
'''
if sql is None:
return False
if self.__conn is None:
self.createConnection()
self.__cursor = self.cursor()
try:
self.__cursor.execute(sql)
self.__conn.commit()
return int(self.__cursor.lastrowid)
except MySQLdb.Error, e:
logger.error('[-].....Mysql update/insert Error:%s' % e)
return False
def executeQuery(self, sql):
'''
对数据库进行查询操作
'''
if sql is None:
return None
if self.__conn is None:
self.createConnection()
self.__cursor = self.cursor()
try:
self.__cursor.execute(sql)
result = self.__cursor.fetchall()
return result
except MySQLdb.Error, e:
logger.error('[-]......Mysql Error:%s' % e)
return None
def run(self):
while True:
try:
if self.daemon:
callbackfunc, args, kwargs = self.workQueue.get(block = True) #如果主线程退出，子线程也退出。这里设置阻塞
else:
callbackfunc, args, kwargs = self.workQueue.get(block = True, timeout = self.timeout)
res = callbackfunc(args, kwargs)
#执行结果加入到结果队列中
if res is not None:
self.resultQueue.put_nowait(res)
except Queue.Empty:
logger.info('work queue empty!')
break
except:
logger.error(sys.exc_info())
raise
time.sleep(0.3)
def send(cls, sub, content):
'''''
send_mail("*****@*****.**","sub","content")
'''
me = MAIL_USER+"<"+MAIL_USER+"@"+MAIL_POSTFIX+">"
msg = MIMEText(content,_charset='gbk')
msg['Subject'] = sub
msg['From'] = me
msg['To'] = ";".join(MAILTO_LIST)
try:
s = smtplib.SMTP()
s.connect(MAIL_HOST)
s.login(MAIL_USER,MAIL_PASS)
s.sendmail(me, MAILTO_LIST, msg.as_string())
s.close()
return True
except Exception, e:
logger.error(str(e))
return False
def download_history_future_basis():
url_model = 'http://www.100ppi.com/graph/future.php?f=graph_echarts&id=%s&source=%s'
future_basis_downloaded_origin_path = get_future_basis_downloaded_origin_path()
future_basis_history_downloaded_origin_path = os.path.join(future_basis_downloaded_origin_path, 'history')
if not os.path.exists(future_basis_history_downloaded_origin_path):
os.makedirs(future_basis_history_downloaded_origin_path)
for future_code in FUTURES_BASIS.keys():
# "铝": {"url": "http://www.100ppi.com/sf/827.html"},
future_id = FUTURES_BASIS[future_code]['url'].split('sf/')[1].split('.')[0]
logger.info('>>>>>>>>>>期货品种：'+future_code+', future_id='+future_id)
# 现期图, ['现货价格','最近合约','主力合约']
request_url_one = url_model % (future_id, '1')
content_html_one = download_page(request_url_one)
if content_html_one is None:
exit_now()
content_json_one = parse_history_page(content_html_one)
filename_one_path = \
os.path.join(future_basis_history_downloaded_origin_path,
future_code+'_'+FUTURES_BASIS[future_code]['name']+'_'+future_id+'_'+'现期图.js')
with open(filename_one_path, 'w', encoding='utf-8') as f:
f.write(json.dumps(content_json_one))
# 基差图, ['最近基差','主力基差']
request_url_two = url_model % (future_id, '2')
content_html_two = download_page(request_url_two)
if content_html_two is None:
exit_now()
content_json_two = parse_history_page(content_html_two)
filename_two_path = \
os.path.join(future_basis_history_downloaded_origin_path,
future_code+'_'+FUTURES_BASIS[future_code]['name']+'_'+future_id+'_'+'基差图.js')
with open(filename_two_path, 'w', encoding='utf-8') as f:
f.write(json.dumps(content_json_two))
# 基差率图, ['最近基差率','主力基差率']
request_url_thr = url_model % (future_id, '3')
content_html_thr = download_page(request_url_thr)
if content_html_thr is None:
exit_now()
content_json_thr = parse_history_page(content_html_thr)
filename_thr_path = \
os.path.join(future_basis_history_downloaded_origin_path,
future_code+'_'+FUTURES_BASIS[future_code]['name']+'_'+future_id+'_'+'基差率图.js')
with open(filename_thr_path, 'w', encoding='utf-8') as f:
f.write(json.dumps(content_json_thr))
# 交易日期,品种代码,现货价格,最近合约,主力合约,最近基差,主力基差,最近基差率,主力基差率
conbine_all_list = list()
conbine_all_list.append(FUTURE_BASIS_CSV_TITLE)
if FUTURES_BASIS[future_code]['name'] != '':
one_trade_date_list = content_json_one['xAxis']['data']
one_0_list = content_json_one['series'][0]['data']
one_1_list = content_json_one['series'][1]['data']
one_2_list = content_json_one['series'][2]['data']
two_trade_date_list = content_json_two['xAxis']['data']
two_0_list = content_json_two['series'][0]['data']
two_1_list = content_json_two['series'][1]['data']
thr_trade_date_list = content_json_thr['xAxis']['data']
thr_0_list = content_json_thr['series'][0]['data']
thr_1_list = content_json_thr['series'][1]['data']
if len(one_trade_date_list) != len(two_trade_date_list) or len(one_trade_date_list) != len(thr_1_list)\
or len(one_trade_date_list) != len(thr_trade_date_list) \
or len(one_trade_date_list) != len(one_0_list) or len(one_trade_date_list) != len(one_1_list) \
or len(one_trade_date_list) != len(one_2_list) or len(one_trade_date_list) != len(two_0_list) \
or len(one_trade_date_list) != len(two_1_list) or len(one_trade_date_list) != len(thr_0_list):
logger.error('数据格式不对请检查')
exit_now()
for i in range(len(one_trade_date_list)):
conbine_all_list.append(','.join([one_trade_date_list[i], future_code, str(one_0_list[i]),
str(one_1_list[i]), str(one_2_list[i]), str(two_0_list[i]),
str(two_1_list[i]), str(thr_0_list[i]), str(thr_1_list[i])]))
future_basis_file_path = get_future_basis_file_path(future_code)
with open(future_basis_file_path, 'w', encoding='utf-8') as f:
f.write('\n'.join(conbine_all_list))
def close(self):
try:
self.__conn.close()
except MySQLdb.Error, e:
logger.error('[-]....Mysql Error:%s' % e)
def parse_one(trade_date):
trading_future_contract_name_list = list()
future_price_downloaded_origin_file_path = \
get_future_price_downloaded_origin_file_path(trade_date, FuturesExchange.CZCE)
if not os.path.exists(future_price_downloaded_origin_file_path):
logger.error('%s 文件不存在！' % future_price_downloaded_origin_file_path)
return False
with open(future_price_downloaded_origin_file_path, 'r',
encoding='utf-8') as f:
_items = f.read().split('小计')
for index, item in enumerate(_items[:-1]):
lines = item.strip().replace('\n\n', '\n').replace('\t', '').replace(
' ', '').replace(',', '').split('\n')
# logger.info('>' * 10+trade_date+'\t'+'\n'.join(lines))
if index == 0:
if not lines[1].startswith('合约代码'):
logger.error(
'数据格式有变动请检查, future_price_downloaded_origin_file_path=' +
future_price_downloaded_origin_file_path)
exit_now()
lines = lines[2:]
lines.insert(0, '|')
# 检查最后一行
if not lines[0].startswith('|'):
logger.error(
'数据格式有变动请检查, future_price_downloaded_origin_file_path=' +
future_price_downloaded_origin_file_path)
exit_now()
lines = lines[1:]
# logger.info('>>>>>>>>>> trade_date='+trade_date+'\n'+str(lines))
# 开始解析
for line in lines:
# 交易日期,合约名称,昨结算,开盘价,最高价,最低价,收盘价,今结算,成交量,持仓量,增减量,成交额(万元),交割结算价
# 合约代码|昨结算|今开盘|最高价|最低价|今收盘|今结算|涨跌1|涨跌2|成交量|持仓量|增减量|成交额(万元)|交割结算价
line = line.split('|')
czce_future_contract_name = line[0]
# 提取期货品种代码
if len(czce_future_contract_name) < 4 or czce_future_contract_name[-1] not in __digit or \
czce_future_contract_name[-2] not in __digit or czce_future_contract_name[-3] not in __digit:
logger.error('合约代码有问题, czce_future_contract_name: %s' %
czce_future_contract_name)
exit_now()
future_variety_code = czce_future_contract_name[:-3]
# 重新命名合约名称
future_contract_name = rename_czce_future_contract_name(
trade_date, czce_future_contract_name, future_variety_code)
logger.info(czce_future_contract_name + ' >>> ' +
future_contract_name)
# 交易日期,合约名称,昨结算,开盘价,最高价,最低价,收盘价,今结算,成交量,持仓量,增减量,成交额(万元),交割结算价
new_line = [
trade_date, future_contract_name, line[1], line[2], line[3],
line[4], line[5], line[6], line[9], line[10], line[11],
line[12], line[13]
]
logger.info('new_line=' + str(new_line))
# 将该条数据插入文件
future_price_contract_file_path = \
get_future_price_contract_file_path(future_variety_code, future_contract_name)
logger.info('future_price_contract_file_path=' +
future_price_contract_file_path)
insert_one_day_future_price_data_to_csv(
future_price_contract_file_path, new_line)
trading_future_contract_name_list.append(future_contract_name)
# 正在上市交易的合约
record_trading_future_contract_name_file_path = \
get_record_trading_future_contract_name_file_path(FuturesExchange.CZCE)
with open(record_trading_future_contract_name_file_path,
'w',
encoding='utf-8') as f:
f.write('\n'.join(trading_future_contract_name_list))
return True
def get(self):
logger.error('here')
return {'message': 'ok'}, 200
def parse_one(trade_date):
"""
python3 zipfile 解压文件时出现文件名为乱码情况的解决方案：
这个问题我在自己代码中跟中转换编码都不行，总出错！
介绍一种便捷方法，直接改源码：
直接找到zipfile.py文件
第一处：
if zinfo.flag_bits & 0x800:
# UTF-8 filename
fname_str = fname.decode("utf-8")
else:
# fname_str = fname.decode("cp437")
fname_str = fname.decode("gbk")
第二处：
if flags & 0x800:
# UTF-8 file names extension
filename = filename.decode('utf-8')
else:
# Historical ZIP filename encoding
# filename = filename.decode('cp437')
filename = filename.decode('gbk')
:param trade_date:
:return:
"""
future_holding_downloaded_origin_file_path = \
get_future_holding_downloaded_origin_file_path(trade_date, FuturesExchange.DCE)
if not os.path.exists(future_holding_downloaded_origin_file_path):
logger.error('%s 文件不存在！' % future_holding_downloaded_origin_file_path)
return False
zip_file = zipfile.ZipFile(future_holding_downloaded_origin_file_path, "r")
for file_name in zip_file.namelist():
try:
content = zip_file.read(file_name).decode('utf-8').strip().replace('\r', '').replace('\n\n', '\n')
except UnicodeDecodeError:
logger.info('数据编码有变动请检查, future_holding_downloaded_origin_file_path=' +
future_holding_downloaded_origin_file_path+file_name)
content = zip_file.read(file_name).decode('gbk').strip().replace('\r', '').replace('\n\n', '\n')
# 检查名次到总计的间隔
check_content_one = re.findall(r'名次.*总计', content, re.S)
if not check_content_one or len(check_content_one) != 1:
logger.error('数据名次到总计的间隔有变动请检查, future_holding_downloaded_origin_file_path=' +
future_holding_downloaded_origin_file_path + file_name)
return False
check_content_one_list = check_content_one[0].split('\n')
if len(check_content_one_list) == 6:
logger.info(content)
logger.info('名次到总计的间隔为6，空！！！')
continue
lines = content.split('\n')
if len(lines) < 8:
logger.error('数据不足8行有变动请检查, future_holding_downloaded_origin_file_path=' +
future_holding_downloaded_origin_file_path+file_name)
return False
# 检查前两行
__repair_trade_date = None
if '大连商品交易所' not in lines[0] or '合约代码：' not in lines[1] or 'Date：' not in lines[1]:
tmp_title = file_name.split('_')
if len(tmp_title) != 5 or len(tmp_title[1]) <= 4 or tmp_title[0] != trade_date:
logger.error('数据格式有变动或日期对不上请检查, future_holding_downloaded_origin_file_path=' +
future_holding_downloaded_origin_file_path+file_name)
return False
future_variety_code = tmp_title[1][:-4].upper()
_tmp_delivery_month = tmp_title[1][-4:]
else:
tmp_title = lines[1].replace('合约代码：', '').replace('Date：', '').replace('-', '').strip().split()
if trade_date == '20161115' and tmp_title[1] == '20161114': # 修复大连期货交易所提供的龙虎榜数据
logger.warn('repair_trade_date, future_holding_downloaded_origin_file_path=' +
future_holding_downloaded_origin_file_path+file_name)
__repair_trade_date = tmp_title[1]
elif tmp_title[1] != trade_date:
logger.info(str(tmp_title)+' | '+trade_date)
logger.error('数据日期对不上请检查, future_holding_downloaded_origin_file_path=' +
future_holding_downloaded_origin_file_path+file_name)
return False
if len(tmp_title) != 2 or len(tmp_title[0]) <= 4:
logger.error('数据格式有变动请检查, future_holding_downloaded_origin_file_path=' +
future_holding_downloaded_origin_file_path+file_name)
return False
future_variety_code = tmp_title[0][:-4].upper()
if future_variety_code not in FUTURE_VARIETIES:
logger.error('期货品种代码有问题请检查, future_holding_downloaded_origin_file_path=' +
future_holding_downloaded_origin_file_path+file_name)
return False
_tmp_delivery_month = tmp_title[0][-4:]
# 重新命名合约名称
future_contract_name = '%s%s%s' % (future_variety_code, trade_date[:2], _tmp_delivery_month)
# 检查数据是否完整
content = '\n'.join(lines).strip()
items = content.split('名次')
if len(items) != 4:
logger.error('数据格式有变动三个不全请检查, future_holding_downloaded_origin_file_path=' +
future_holding_downloaded_origin_file_path+file_name)
return False
content_volume = items[1].strip()
content_buy = items[2].strip()
content_sell = items[3].strip()
logger.info('%s, %s, %s | %s, %s' % (tmp_title, future_variety_code, future_contract_name,
file_name, future_holding_downloaded_origin_file_path))
# 开始解析
future_holding_dict = get_future_holding_dict()
future_holding_dict['code'] = future_contract_name
future_holding_dict['date'] = trade_date
if __repair_trade_date is not None: # 修复大连期货交易所提供的龙虎榜数据
future_holding_dict['date'] = __repair_trade_date
# 解析成交量
lines_volume = content_volume.split('\n')
if '成交量' not in lines_volume[0] or '总计' not in lines_volume[-1]:
logger.error('成交量数据格式表头有变动请检查, future_holding_downloaded_origin_file_path=' +
future_holding_downloaded_origin_file_path+file_name)
return False
if len(lines_volume) > 2:
tmp_str_total = lines_volume[-1].replace('总计', '').strip().replace(',', '').split()
if len(tmp_str_total) == 1:
total_v, tmp_total_v = int(tmp_str_total[0]), 0
total_i_a_d, tmp_total_i_a_d = 0, 0
elif len(tmp_str_total) == 2:
total_v, tmp_total_v = int(tmp_str_total[0]), 0
total_i_a_d, tmp_total_i_a_d = int(tmp_str_total[1]), 0
else:
logger.error('成交量数据格式表尾有变动请检查, future_holding_downloaded_origin_file_path=' +
future_holding_downloaded_origin_file_path+file_name)
return False
for line_volume in lines_volume[1:-1]:
line_volume = line_volume.split()
if len(line_volume) != 4:
logger.error('成交量数据格式有变动请检查, future_holding_downloaded_origin_file_path=' +
future_holding_downloaded_origin_file_path+file_name)
return False
v = int(line_volume[2].replace(',', ''))
i_a_d = int(line_volume[3].replace(',', ''))
future_holding_dict['volume'][int(line_volume[0])] = [line_volume[1], v, i_a_d]
tmp_total_v += v
tmp_total_i_a_d += i_a_d
logger.info(line_volume)
if total_v != tmp_total_v:
logger.error('成交量各期货会员总持仓核对不上, future_holding_downloaded_origin_file_path=' +
future_holding_downloaded_origin_file_path+file_name)
return False
if total_i_a_d > 0 and total_i_a_d != tmp_total_i_a_d:
logger.error('成交量各期货会员总持仓核对不上, future_holding_downloaded_origin_file_path=' +
future_holding_downloaded_origin_file_path+file_name)
return False
future_holding_dict['total_volume'] = [tmp_total_v, tmp_total_i_a_d]
# 解析持买单量
lines_buy = content_buy.split('\n')
if '持买单量' not in lines_buy[0] or '总计' not in lines_buy[-1]:
logger.error('持买单量数据格式表头有变动请检查, future_holding_downloaded_origin_file_path=' +
future_holding_downloaded_origin_file_path+file_name)
return False
if len(lines_buy) > 2:
tmp_str_total = lines_buy[-1].replace('总计', '').strip().replace(',', '').split()
if len(tmp_str_total) == 1:
total_v, tmp_total_v = int(tmp_str_total[0]), 0
total_i_a_d, tmp_total_i_a_d = 0, 0
elif len(tmp_str_total) == 2:
total_v, tmp_total_v = int(tmp_str_total[0]), 0
total_i_a_d, tmp_total_i_a_d = int(tmp_str_total[1]), 0
else:
logger.error('成交量数据格式表尾有变动请检查, future_holding_downloaded_origin_file_path=' +
future_holding_downloaded_origin_file_path+file_name)
return False
for line_buy in lines_buy[1:-1]:
line_buy = line_buy.split()
if len(line_buy) != 4:
logger.error('持买单量数据格式有变动请检查, future_holding_downloaded_origin_file_path=' +
future_holding_downloaded_origin_file_path+file_name)
return False
v = int(line_buy[2].replace(',', ''))
i_a_d = int(line_buy[3].replace(',', ''))
future_holding_dict['buy'][int(line_buy[0])] = [line_buy[1], v, i_a_d]
tmp_total_v += v
tmp_total_i_a_d += i_a_d
logger.info(line_buy)
if total_v != tmp_total_v:
logger.error('持买单量各期货会员总持仓核对不上, future_holding_downloaded_origin_file_path=' +
future_holding_downloaded_origin_file_path+file_name)
return False
if total_i_a_d > 0 and total_i_a_d != tmp_total_i_a_d:
logger.error('成交量各期货会员总持仓核对不上, future_holding_downloaded_origin_file_path=' +
future_holding_downloaded_origin_file_path+file_name)
return False
future_holding_dict['total_buy'] = [tmp_total_v, tmp_total_i_a_d]
# 解析持卖单量
lines_sell = list()
tmp_lines_sell = content_sell.split('\n')
for tmp_line in tmp_lines_sell:
lines_sell.append(tmp_line)
if '总计' in tmp_line:
break
if '持卖单量' not in lines_sell[0] or '总计' not in lines_sell[-1]:
logger.error('持卖单量数据格式表头有变动请检查, future_holding_downloaded_origin_file_path=' +
future_holding_downloaded_origin_file_path+file_name)
return False
if len(lines_sell) > 2:
tmp_str_total = lines_sell[-1].replace('总计', '').strip().replace(',', '').split()
if len(tmp_str_total) == 1:
total_v, tmp_total_v = int(tmp_str_total[0]), 0
total_i_a_d, tmp_total_i_a_d = 0, 0
elif len(tmp_str_total) == 2:
total_v, tmp_total_v = int(tmp_str_total[0]), 0
total_i_a_d, tmp_total_i_a_d = int(tmp_str_total[1]), 0
else:
logger.error('成交量数据格式表尾有变动请检查, future_holding_downloaded_origin_file_path=' +
future_holding_downloaded_origin_file_path+file_name)
return False
for line_sell in lines_sell[1:-1]:
line_sell = line_sell.split()
if len(line_sell) != 4:
logger.error('持卖单量数据格式有变动请检查, future_holding_downloaded_origin_file_path=' +
future_holding_downloaded_origin_file_path+file_name)
return False
v = int(line_sell[2].replace(',', ''))
i_a_d = int(line_sell[3].replace(',', ''))
future_holding_dict['sell'][int(line_sell[0])] = [line_sell[1], v, i_a_d]
tmp_total_v += v
tmp_total_i_a_d += i_a_d
logger.info(line_sell)
if total_v != tmp_total_v:
logger.error('持卖单量各期货会员总持仓核对不上, future_holding_downloaded_origin_file_path=' +
future_holding_downloaded_origin_file_path+file_name)
return False
if total_i_a_d > 0 and total_i_a_d != tmp_total_i_a_d:
logger.error('成交量各期货会员总持仓核对不上, future_holding_downloaded_origin_file_path=' +
future_holding_downloaded_origin_file_path+file_name)
return False
future_holding_dict['total_sell'] = [tmp_total_v, tmp_total_i_a_d]
# 写入文件
future_holding_contract_file_path = \
get_future_holding_contract_file_path(future_variety_code, future_contract_name)
insert_one_day_future_holding_data_to_js(future_holding_contract_file_path, future_holding_dict)
# break
zip_file.close()
return True
def parse_downloaded_origin_file_from_20100818_to_20150918(
trade_date, future_holding_downloaded_origin_file_path):
with open(future_holding_downloaded_origin_file_path,
'r',
encoding='utf-8') as f:
items = f.read().replace('合约', '品种').split('品种')
for index, item in enumerate(items):
lines = item.strip().replace('\n\n',
'\n').replace(' ',
'').replace('\t',
'').split('\n')
logger.info('>' * 10 + trade_date + '\t' + '\n'.join(lines))
# 检查最后一行，合计
if '合计' not in lines[-1]:
if index == 0:
continue
else:
logger.error('缺少合计, ' + str(lines[-1]))
exit_now()
# 检查第一行, ：棉花日期：2010-08-18 ：ER105日期：2010-08-18
__future_varietties_dict = {
'CF': '棉花',
'RI': '早籼',
'OI': '菜油',
'SR': '白糖',
'TA': 'PTA',
'WS': '强麦',
'WT': '硬麦',
'MA': '甲醇',
'PM': '普麦',
'FG': '玻璃',
'RM': '菜粕',
'RS': '菜籽',
# 'ZC': '动力煤',
'TC': '煤',
'JR': '粳稻',
'LR': '晚籼',
'SF': '硅铁',
'SM': '锰硅',
}
check_res = 0
future_variety_code = ''
for _future_variety_code in FUTURE_VARIETIES.keys():
if _future_variety_code in lines[0]:
check_res += 1
future_variety_code = _future_variety_code
for _future_variety_code, _future_variety_name in __future_varietties_dict.items(
):
if _future_variety_name in lines[0]:
check_res += 1
future_variety_code = _future_variety_code
if check_res != 1 and future_variety_code not in ('TA', ):
logger.error('合约代码对不上, lines[0]: ' + str(lines[0]) +
'|future_variety_code=' + future_variety_code)
exit_now()
# 重新命名合约名称 (：棉花日期：2010-08-18 ：ER105日期：2010-08-18)
future_contract_name = None
czce_future_contract_name, future_date_str = \
lines[0].strip().replace('-', '').replace(':', '').replace('：', '').split('日期')
if czce_future_contract_name in FUTURE_VARIETIES[future_variety_code][
'品种全称']:
future_contract_name = 'total'
elif future_variety_code in czce_future_contract_name:
future_contract_name = \
rename_czce_future_contract_name(trade_date, czce_future_contract_name, future_variety_code)
logger.info(czce_future_contract_name + ' >>> ' +
future_contract_name)
else:
logger.error('合约代码有问题, lines[0]: ' + str(lines[0]) + ' | 品种全称=' +
FUTURE_VARIETIES[future_variety_code]['品种全称'])
exit_now()
# 检查日期
if future_date_str != trade_date or len(future_date_str) != 8:
logger.error('日期对不上, lines[0]: ' + str(lines[0]))
exit_now()
# 解析数据
future_holding_dict = get_future_holding_dict()
future_holding_dict['code'] = future_contract_name
future_holding_dict['date'] = trade_date
for line in lines[1:-1]:
if '.0' not in line:
logger.error('格式不对请检查, line=' + line)
exit_now()
line = line.replace('.0', '')
line = line.strip().split(',')
logger.info('line=' + str(line))
if line[1] != '':
future_holding_dict['volume'][int(
line[0])] = [line[1], int(line[2]),
int(line[3])]
if line[4] != '':
future_holding_dict['buy'][int(
line[0])] = [line[4], int(line[5]),
int(line[6])]
if line[7] != '':
future_holding_dict['sell'][int(
line[0])] = [line[7], int(line[8]),
int(line[9])]
line = lines[-1]
if '.0' not in line:
logger.error('格式不对请检查, line=' + line)
exit_now()
line = line.replace('.0', '')
line = line.strip().split(',')
if line[1] != '':
line.insert(1, '')
logger.info('合计 line=' + str(line))
future_holding_dict['total_volume'] = [int(line[2]), int(line[3])]
future_holding_dict['total_buy'] = [int(line[5]), int(line[6])]
future_holding_dict['total_sell'] = [int(line[8]), int(line[9])]
# 写入文件
future_holding_contract_file_path = \
get_future_holding_contract_file_path(future_variety_code, future_contract_name)
insert_one_day_future_holding_data_to_js(
future_holding_contract_file_path, future_holding_dict)
if __name__ == "__main__":
logger.info("Checking for updates...")
os.system("git pull")
logger.info(
"If updates were done, restart this script by using CTRL-C to terminate it, and re run it."
)
# Make connection to Discord
try:
RPC = Presence(client_id) # Initialize the Presence class
RPC.connect() # Start the handshake loop
except pypresence.exceptions.InvalidPipe:
logger.error(
"Could not connect to the discord pipe. Please ensure it's running."
)
exit(1)
except FileNotFoundError:
logger.error(
"Could not connect to the discord pipe. Please ensure it's running."
)
exit(1)
# Load our current config
config = nso_functions.get_config_file()
# We run this just to ensure the data collection is A-OK
get_minutes_since()
def get(self):
ret = db.session.query(Device).first()
logger.error('test')
return {'message': ret.name, 'status': 0}
def main():
logger.info("Checking for updates...")
os.system("git pull")
logger.info(
"If updates were done, restart this script by using CTRL-C to terminate it, and re run it."
)
# Make connection to Discord
try:
RPC = Presence(client_id) # Initialize the Presence class
RPC.connect() # Start the handshake loop
except pypresence.exceptions.InvalidPipe:
logger.error(
"Could not connect to the discord pipe. Please ensure it's running."
)
exit(1)
except FileNotFoundError:
logger.error(
"Could not connect to the discord pipe. Please ensure it's running."
)
exit(1)
# Load our current config
config = nso_functions.get_config_file()
logger.info("Check discord!")
# get friend code from config, and add config option if does not exist
try:
friend_code = config['friend_code']
except KeyError:
config['friend_code'] = 'Unset'
config_f = open("config.txt", "w")
config_f.write(json.dumps(
config,
sort_keys=True,
indent=4,
))
config_f.close()
friend_code = config['friend_code']
while True: # The presence will stay on as long as the program is running
for i in range(0, 4):
minutes_since, last_match = get_minutes_since()
# int is here so we don't have funky floating point madness
seconds_since = int(minutes_since * 60)
hours_since = int(minutes_since / 60)
if minutes_since >= 60:
details = "Last match: {} hour(s) ago".format(hours_since)
elif minutes_since > 1:
details = "Last match: {} minute(s) ago".format(
math.floor(minutes_since))
else:
details = "Last match: {} second(s) ago".format(seconds_since)
# job_result is only present in salmon run JSON
if last_match.get('job_result') is not None:
gamemode_key = "salmon_run"
if last_match['job_result']['is_clear']:
outcome = "winning"
else:
outcome = "losing"
large_text = "Last match was Salmon Run, {} with {} eggs".format(
outcome, last_match['job_score'])
if i == 0:
state = "Grade: {}".format(
(last_match["grade"])["long_name"])
elif i == 1:
state = "Friend code: {}".format(friend_code)
elif i == 2:
state = "Difficulty: {}".format(
str(last_match["danger_rate"]))
elif i == 3:
state = "Played on {}".format(
last_match['schedule']['stage']['name'])
else:
large_text = "Last match was {}, {} on {}".format(
last_match["game_mode"]["name"],
last_match["rule"]["name"], last_match["stage"]["name"])
gamemode_key = last_match["rule"]["key"]
if i == 0:
state = "Friend code: {}".format(friend_code)
elif i == 1:
state = "K/D: {}/{}".format(
last_match["player_result"]["kill_count"],
last_match["player_result"]["death_count"])
elif i == 2:
details = last_match["my_team_result"]["name"]
try:
state = "{}% vs {}%".format(
last_match["my_team_percentage"],
last_match["other_team_percentage"])
except KeyError:
try:
state = "{} vs {}".format(
last_match["my_team_count"],
last_match["other_team_count"])
except KeyError:
state = "Gamemode not yet supported"
elif i == 3:
state = "{}p".format(
last_match["player_result"]["game_paint_point"])
if show_weapon:
details = "{}".format(last_match["player_result"]
["player"]["weapon"]["name"])
else:
pass
if minutes_since < timeout_minutes:
RPC.update(details=details,
state=state,
large_image=gamemode_key,
small_image="default",
large_text=large_text)
else:
RPC.clear()
logger.debug("RPC cleared, not in game long enough")
time.sleep(time_interval)
def parse_one(trade_date):
"""
说明：
(1) 价格：自2019年12月02日起，纤维板报价单位由元/张改为元/立方米
(2) 价格：元/吨，鸡蛋为元/500千克，纤维板为元/立方米，胶合板为元/张
(3) 成交量、持仓量：手（按单边计算）
(4) 成交额：万元（按单边计算）
(5) 涨跌＝收盘价－前结算价
(6) 涨跌1=今结算价-前结算价
(7) 合约系列：具有相同月份标的期货合约的所有期权合约的统称
(8) 隐含波动率：根据期权市场价格，利用期权定价模型计算的标的期货合约价格波动率
:param trade_date:
:return:
"""
trading_future_contract_name_list = list()
future_price_downloaded_origin_file_path = \
get_future_price_downloaded_origin_file_path(trade_date, FuturesExchange.DCE)
if not os.path.exists(future_price_downloaded_origin_file_path):
logger.error('%s 文件不存在！' % future_price_downloaded_origin_file_path)
return False
with open(future_price_downloaded_origin_file_path, 'r',
encoding='utf-8') as f:
content_html = f.read().strip().replace(' ', '').replace(
'\t', '').replace(' ', '').replace('\n', '')
table_obj = re.findall(r'<table.*?</table>', content_html, re.S)
if not table_obj or len(table_obj) != 1:
logger.error(
'html数据格式有变动请检查, future_price_downloaded_origin_file_path=' +
future_price_downloaded_origin_file_path)
return False
table_obj = table_obj[0]
tr_obj_list = re.findall(r'<tr>.*?</tr>', table_obj, re.S)
if not tr_obj_list or len(tr_obj_list) <= 1:
logger.error(
'html数据格式有变动请检查, future_price_downloaded_origin_file_path=' +
future_price_downloaded_origin_file_path)
return False
for index, tr_obj in enumerate(tr_obj_list):
if index == 0:
if '收盘价' not in tr_obj or '<th>' not in tr_obj:
logger.error(
'html数据格式有变动请检查, future_price_downloaded_origin_file_path='
+ future_price_downloaded_origin_file_path)
return False
continue
td_obj_list = re.findall(r'<td>(.*?)</td>', tr_obj, re.S)
if len(td_obj_list) != 14:
logger.error(
'html数据格式有变动请检查, future_price_downloaded_origin_file_path=' +
future_price_downloaded_origin_file_path)
return False
if '小计' in td_obj_list[0] or '总计' in td_obj_list[0]:
continue
# 重新命名合约名称
if td_obj_list[0] not in MAPPING_DCE_FUTURE_VARIETIES_CODE:
logger.error(
'MAPPING_DCE_FUTURE_VARIETIES_CODE ERROR: %s, %s' %
(td_obj_list[0], future_price_downloaded_origin_file_path))
return False
future_variety_code = MAPPING_DCE_FUTURE_VARIETIES_CODE[td_obj_list[0]]
delivery_month = td_obj_list[1]
if len(delivery_month) != 4:
logger.error(
'交割月份 ERROR: %s, %s' %
(td_obj_list[1], future_price_downloaded_origin_file_path))
return False
future_contract_name = '%s%s%s' % (future_variety_code, trade_date[:2],
delivery_month)
# 行中各字段数据解析
# 商品名称 交割月份 开盘价 最高价 最低价 收盘价 前结算价 结算价 涨跌 涨跌1 成交量 持仓量 持仓量变化 成交额
# 交易日期,合约名称,昨结算,开盘价,最高价,最低价,收盘价,今结算,成交量,持仓量,增减量,成交额(万元),交割结算价
new_line = [
trade_date, future_contract_name,
td_obj_list[6].replace(',', '') if td_obj_list[6] != '-' else '',
td_obj_list[2].replace(',', '') if td_obj_list[2] != '-' else '',
td_obj_list[3].replace(',', '') if td_obj_list[3] != '-' else '',
td_obj_list[4].replace(',', '') if td_obj_list[4] != '-' else '',
td_obj_list[5].replace(',', '') if td_obj_list[5] != '-' else '',
td_obj_list[7].replace(',', '') if td_obj_list[7] != '-' else '',
td_obj_list[10].replace(',', '') if td_obj_list[10] != '-' else '',
td_obj_list[11].replace(',', '') if td_obj_list[11] != '-' else '',
td_obj_list[12].replace(',', '') if td_obj_list[12] != '-' else '',
td_obj_list[13].replace(',', '') if td_obj_list[13] != '-' else '',
''
]
logger.info('new_line=' + str(new_line))
# 将该条数据插入文件
future_price_contract_file_path = \
get_future_price_contract_file_path(future_variety_code, future_contract_name)
logger.info('future_price_contract_file_path=' +
future_price_contract_file_path)
insert_one_day_future_price_data_to_csv(
future_price_contract_file_path, new_line)
trading_future_contract_name_list.append(future_contract_name)
# 正在上市交易的合约
record_trading_future_contract_name_file_path = \
get_record_trading_future_contract_name_file_path(FuturesExchange.DCE)
with open(record_trading_future_contract_name_file_path,
'w',
encoding='utf-8') as f:
f.write('\n'.join(trading_future_contract_name_list))
return True
def read_data(self):
future_price_dict = self.get_future_price_contract_data_by_future_contract_name(
self.future_variety_code, self.future_contract_name)
future_holding_dict = self.get_future_holding_contract_data_by_future_contract_name(
self.future_variety_code, self.future_contract_name)
future_members_buy_holding_dict = self.get_future_members_holding_dict(
len(future_price_dict.keys()))
future_members_sell_holding_dict = self.get_future_members_holding_dict(
len(future_price_dict.keys()))
index = 0
for trade_date, line in future_price_dict.items():
if trade_date not in self.trade_calendar:
logger.error('交易日历有问题，请检查！')
exit_now()
self.trade_date_list.append(trade_date)
# 收盘价
if float(line[6]) == 0:
self.price_list.append(None)
else:
self.price_list.append(float(line[6]))
# 持仓量
total_holding = float(line[9])
self.total_holding_list.append(total_holding)
# 警戒线
self.cordon_list.append(0.5)
if trade_date not in future_holding_dict:
self.buy_list.append(None)
self.sell_list.append(None)
self.individual_buy_list.append(None)
self.individual_sell_list.append(None)
else:
buy_holding = float(
future_holding_dict[trade_date]['total_buy'][0])
sell_holding = float(
future_holding_dict[trade_date]['total_sell'][0])
self.buy_list.append(
(buy_holding / total_holding) * 100 / 100.0)
self.sell_list.append(
(sell_holding / total_holding) * 100 / 100.0)
self.individual_buy_list.append(1 -
(buy_holding / total_holding) *
100 / 100.0)
self.individual_sell_list.append(
1 - (sell_holding / total_holding) * 100 / 100.0)
# 多单持仓
buy = future_holding_dict[trade_date]['buy']
for value in buy.values():
future_members_buy_holding_dict[value[0]][index] = value[1]
# 空单持仓
sell = future_holding_dict[trade_date]['sell']
for value in sell.values():
future_members_sell_holding_dict[
value[0]][index] = value[1]
index += 1
for future_member_name, future_member_holding_list in future_members_buy_holding_dict.items(
):
total_num = 0
for v in future_member_holding_list:
if v is None:
continue
total_num += v
if total_num != 0:
self.new_future_members_buy_holding_dict[
future_member_name] = future_member_holding_list
for future_member_name, future_member_holding_list in future_members_sell_holding_dict.items(
):
total_num = 0
for v in future_member_holding_list:
if v is None:
continue
total_num += v
if total_num != 0:
self.new_future_members_sell_holding_dict[
future_member_name] = future_member_holding_list
# 交易日历
__trade_calendar_file_path = os.path.join(os.path.join(work_path, 'resources'),
'trade_calendar.csv')
__trade_calendar = list()
today = time.strftime("%Y%m%d", time.localtime())
logger.info('today=' + today)
with open(__trade_calendar_file_path, 'r', encoding='utf-8') as f:
_tmp = f.read().strip().split('\n')
for item_date in _tmp:
item_date = item_date.split(',')[0]
if item_date > today:
continue
__trade_calendar.append(item_date)
if len(__trade_calendar) == 0 or today > _tmp[-1]:
logger.error('请检查交易日历！')
exit(1)
TRADE_CALENDAR = __trade_calendar
# 期货交易数据表头
# 涨跌1=收盘价-昨结算
# 涨跌2=结算价-昨结算
# 持仓量=空盘量
# 价格：元 / 吨
# 成交量、持仓量：手（单边计算）
# 成交额：万元（单边计算）
FUTURE_PRICE_CSV_TITLE = '交易日期,合约名称,昨结算,开盘价,最高价,最低价,收盘价,今结算,成交量,持仓量,增减量,成交额(万元),交割结算价'
FUTURE_BASIS_CSV_TITLE = '交易日期,品种代码,现货价格,最近合约,主力合约,最近基差,主力基差,最近基差率,主力基差率'
# 期货会员, 共147个
FUTURE_MEMBERS = {
def main():
logger.info("Checking for updates...")
os.system("git pull")
logger.info(
"If updates were done, restart this script by using CTRL-C to terminate it, and re run it."
)
# Make connection to Discord
try:
RPC = Presence(client_id) # Initialize the Presence class
RPC.connect() # Start the handshake loop
except pypresence.exceptions.InvalidPipe:
logger.error(
"Could not connect to the discord pipe. Please ensure it's running."
)
exit(1)
except FileNotFoundError:
logger.error(
"Could not connect to the discord pipe. Please ensure it's running."
)
exit(1)
# Load our current config
config = nso_functions.get_config_file()
logger.info("Check discord!")
# Get friend code from config, and add config option if does not exist
try:
friend_code = config['friend_code']
except KeyError:
config['friend_code'] = 'Unset'
config_f = open("config.txt", "w")
config_f.write(json.dumps(
config,
sort_keys=True,
indent=4,
))
config_f.close()
friend_code = config['friend_code']
while True: # The presence will stay on as long as the program is running
for i in range(0, 5):
minutes_since, last_match = get_minutes_since()
# Calculating the secs/hours/days since Last Match/Run
seconds_since = int(minutes_since * 60)
hours_since = int(minutes_since / 60)
days_since = int(minutes_since / 1440)
# When Previous Match was Salmon Run
# job_result is only present in salmon run JSON
if last_match.get('job_result') is not None:
# Sets Gamemode Key in order to change the Picture
gamemode_key = "salmon_run"
# Decides if last Run is shown in days, hours, minutes or seconds
# In Days
if minutes_since >= 1440:
details = "Last Run: {} day{} ago".format(
days_since, plural_logic(days_since))
# In Hours
elif minutes_since >= 60:
details = "Last Run: {} h{} ago".format(
hours_since, plural_logic(hours_since))
# In Minutes
elif minutes_since > 1:
details = "Last Run: {} min{} ago".format(
math.floor(minutes_since),
plural_logic(math.floor(minutes_since)))
# In Seconds
else:
details = "Last Run: {} sec{} ago".format(
seconds_since, plural_logic(seconds_since))
# Deciding the Result
if last_match['job_result']['is_clear']:
outcome = "WON"
else:
outcome = "LOST"
### Checks how many waves were played on last Run
# If all 3 Waves were played
if last_match["wave_details"][2]:
goldEgg = last_match["wave_details"][0]["golden_ikura_num"] + \
last_match["wave_details"][1]["golden_ikura_num"] + \
last_match["wave_details"][2]["golden_ikura_num"]
powEgg = last_match["wave_details"][0]["ikura_num"] + \
last_match["wave_details"][1]["ikura_num"] + \
last_match["wave_details"][2]["ikura_num"]
# If only 2 Waves were played
elif not last_match["wave_details"][2] and last_match[
"wave_details"][1]:
goldEgg = last_match["wave_details"][0][
"golden_ikura_num"] + last_match["wave_details"][1][
"golden_ikura_num"]
powEgg = last_match["wave_details"][0][
"ikura_num"] + last_match["wave_details"][1][
"ikura_num"]
# If only 1 Wave was played
else:
goldEgg = last_match["wave_details"][0]["golden_ikura_num"]
powEgg = last_match["wave_details"][0]["ikura_num"]
# When hovering on the Picture
large_text = "Last match was Salmon Run on {}".format(
last_match['schedule']['stage']['name'])
# IGN and Salmon Run Rank
if i == 0:
details = "IGN: {}".format(last_match["my_result"]["name"])
state = "{} {}".format((last_match["grade"])["long_name"],
last_match["grade_point"])
# Friend code
elif i == 1:
if not friend_code:
state = "FC: Not Given"
else:
state = "FC: {}".format(friend_code)
# Hazard Level
elif i == 2:
state = "Hazard Level: {}".format(
str(last_match["danger_rate"]) + "%")
# Result and Total Collected Golden Eggs / Power Eggs
elif i == 3:
details = "GoldEgg/PowEgg ({})".format(outcome)
state = "{} / {}".format(goldEgg, powEgg)
# Save / Death Ratio
elif i == 4:
state = "Save/Death Ratio: {}/{}".format(
last_match["my_result"]["help_count"],
last_match["my_result"]["dead_count"])
if minutes_since < timeout_minutes:
RPC.update(details=details,
state=state,
large_image=gamemode_key,
small_image="default",
large_text=large_text)
else:
RPC.clear()
logger.debug("RPC cleared, not in game long enough")
time.sleep(time_interval)
# When Previous Match was Turf, Ranked, League or Private
else:
# Decides if last Match is shown in days, hours, minutes or seconds
# In Days
if minutes_since >= 1440:
details = "Last Match: {} day{} ago".format(
days_since, plural_logic(days_since))
# In Hours
elif minutes_since >= 60:
details = "Last Match: {} h{} ago".format(
hours_since, plural_logic(hours_since))
# In Minutes
elif minutes_since > 1:
details = "Last Match: {} min{} ago".format(
math.floor(minutes_since),
plural_logic(math.floor(minutes_since)))
# In Seconds
else:
details = "Last Match: {} sec{} ago".format(
seconds_since, plural_logic(seconds_since))
# When hovering on the Picture
large_text = "Last match was {}, {} on {}".format(
last_match["game_mode"]["name"],
last_match["rule"]["name"], last_match["stage"]["name"])
# Gets Gamemode Key in order to change the Picture
gamemode_key = last_match["rule"]["key"]
# Gets Lobby Key
lobby_key = last_match["game_mode"]["key"]
# IGN and Level (+ Rank)
if i == 0:
details = "IGN: {}".format(
last_match["player_result"]["player"]["nickname"])
# Checks if player has a Level Star
# If player has no Level Star (yet XP)
if not last_match["star_rank"]:
# If last match was in a Regular Lobby (Turf War) or Private Lobby
if lobby_key == "regular" or lobby_key == "private":
state = "Level: {}".format(
last_match["player_result"]["player"]
["player_rank"], )
# If last match was in a Ranked Solo Lobby
elif lobby_key == "gachi":
# If last match was Splat Zones
if gamemode_key == "splat_zones":
# If player has S+ Rank
if last_match["udemae"]["name"] == "S+":
state = "Lvl: {}/R(SZ): {}{}".format(
last_match["player_result"]["player"]
["player_rank"],
last_match["udemae"]["name"],
last_match["udemae"]["s_plus_number"])
# If player has X Rank
elif last_match["udemae"]["name"] == "X":
# Checks if Player has any X Power
# If Player has no X Power (yet XP)
if not last_match["x_power"]:
state = "Lvl: {}/R(SZ): X(TBD)".format(
last_match["player_result"]
["player"]["player_rank"], )
# If Player has X Power
else:
state = "Lvl: {}/R(SZ): X({})".format(
last_match["player_result"]
["player"]["player_rank"],
last_match["x_power"])
# If player has other Ranks
else:
state = "Lvl: {}/R(SZ): {}".format(
last_match["player_result"]["player"]
["player_rank"],
last_match["udemae"]["name"])
# If last match was Tower Control
elif gamemode_key == "tower_control":
# If player has S+ Rank
if last_match["udemae"]["name"] == "S+":
state = "Lvl: {}/R(TC): {}{}".format(
last_match["player_result"]["player"]
["player_rank"],
last_match["udemae"]["name"],
last_match["udemae"]["s_plus_number"])
# If player has X Rank
elif last_match["udemae"]["name"] == "X":
# Checks if Player has any X Power
# If Player has no X Power (yet XP)
if not last_match["x_power"]:
state = "Lvl: {}/R(TC): X(TBD)".format(
last_match["player_result"]
["player"]["player_rank"], )
# If Player has X Power
else:
state = "Lvl: {}/R(TC): X({})".format(
last_match["player_result"]
["player"]["player_rank"],
last_match["x_power"])
# If player has other Ranks
else:
state = "Lvl: {}/R(TC): {}".format(
last_match["player_result"]["player"]
["player_rank"],
last_match["udemae"]["name"])
# If last match was Rainmaker
elif gamemode_key == "rainmaker":
# If player has S+ Rank
if last_match["udemae"]["name"] == "S+":
state = "Lvl: {}/R(RM): {}{}".format(
last_match["player_result"]["player"]
["player_rank"],
last_match["udemae"]["name"],
last_match["udemae"]["s_plus_number"])
# If player has X Rank
elif last_match["udemae"]["name"] == "X":
# Checks if Player has any X Power
# If Player has no X Power (yet XP)
if not last_match["x_power"]:
state = "Lvl: {}/R(RM): X(TBD)".format(
last_match["player_result"]
["player"]["player_rank"], )
# If Player has X Power
else:
state = "Lvl: {}/R(RM): X({})".format(
last_match["player_result"]
["player"]["player_rank"],
last_match["x_power"])
# If player has other Ranks
else:
state = "Lvl: {}/R(RM): {}".format(
last_match["player_result"]["player"]
["player_rank"],
last_match["udemae"]["name"])
# If last match was Clam Blitz
else:
# If player has S+ Rank
if last_match["udemae"]["name"] == "S+":
state = "Lvl: {}/R(CB): {}{}".format(
last_match["player_result"]["player"]
["player_rank"],
last_match["udemae"]["name"],
last_match["udemae"]["s_plus_number"])
# If player has X Rank
elif last_match["udemae"]["name"] == "X":
# Checks if Player has any X Power
# If Player has no X Power (yet XP)
if not last_match["x_power"]:
state = "Lvl: {}/R(CB): X(TBD)".format(
last_match["player_result"]
["player"]["player_rank"], )
# If Player has X Power
else:
state = "Lvl: {}/R(CB): X({})".format(
last_match["player_result"]
["player"]["player_rank"],
last_match["x_power"])
# If player has other Ranks
else:
state = "Lvl: {}/R(CB): {}".format(
last_match["player_result"]["player"]
["player_rank"],
last_match["udemae"]["name"])
# If last match was in a League Pair/Team Lobby
elif lobby_key == "league_pair" or lobby_key == "league_team":
# Checks if Player has League Power
# If Player has no League Power (yet XP)
if not last_match["league_point"]:
state = "Lvl: {}/Power: TBD".format(
last_match["player_result"]["player"]
["player_rank"])
# If Player has League Power
else:
state = "Lvl: {}/Power: {}".format(
last_match["player_result"]["player"]
["player_rank"],
last_match["league_point"])
# If player has a Level Star
else:
# If last match was in a Regular Lobby (Turf War) or Private Lobby
if lobby_key == "regular" or lobby_key == "private":
state = "Level: {}☆{}".format(
last_match["player_result"]["player"]
["player_rank"],
last_match["player_result"]["player"]
["star_rank"],
)
# If last match was in a Ranked Solo Lobby
elif lobby_key == "gachi":
# If last match was Splat Zones
if gamemode_key == "splat_zones":
# If player has S+ Rank
if last_match["udemae"]["name"] == "S+":
state = "Lvl: {}☆{}/R(SZ): {}{}".format(
last_match["player_result"]["player"]
["player_rank"],
last_match["player_result"]["player"]
["star_rank"],
last_match["udemae"]["name"],
last_match["udemae"]["s_plus_number"])
# If player has X Rank
elif last_match["udemae"]["name"] == "X":
# Checks if Player has any X Power
# If Player has no X Power (yet XP)
if not last_match["x_power"]:
state = "Lvl: {}☆{}/R(SZ): X(TBD)".format(
last_match["player_result"]
["player"]["player_rank"],
last_match["player_result"]
["player"]["star_rank"],
)
# If Player has X Power
else:
state = "Lvl: {}☆{}/R(SZ): X({})".format(
last_match["player_result"]
["player"]["player_rank"],
last_match["player_result"]
["player"]["star_rank"],
last_match["x_power"])
# If player has other Ranks
else:
state = "Lvl: {}☆{}/R(SZ): {}".format(
last_match["player_result"]["player"]
["player_rank"],
last_match["player_result"]["player"]
["star_rank"],
last_match["udemae"]["name"])
# If last match was Tower Control
elif gamemode_key == "tower_control":
# If player has S+ Rank
if last_match["udemae"]["name"] == "S+":
state = "Lvl: {}☆{}/R(TC): {}{}".format(
last_match["player_result"]["player"]
["player_rank"],
last_match["player_result"]["player"]
["star_rank"],
last_match["udemae"]["name"],
last_match["udemae"]["s_plus_number"])
# If player has X Rank
elif last_match["udemae"]["name"] == "X":
# Checks if Player has any X Power
# If Player has no X Power (yet XP)
if not last_match["x_power"]:
state = "Lvl: {}☆{}/R(TC): X(TBD)".format(
last_match["player_result"]
["player"]["player_rank"],
last_match["player_result"]
["player"]["star_rank"],
)
# If Player has X Power
else:
state = "Lvl: {}☆{}/R(TC): X({})".format(
last_match["player_result"]
["player"]["player_rank"],
last_match["player_result"]
["player"]["star_rank"],
last_match["x_power"])
# If player has other Ranks
else:
state = "Lvl: {}☆{}/R(TC): {}".format(
last_match["player_result"]["player"]
["player_rank"],
last_match["player_result"]["player"]
["star_rank"],
last_match["udemae"]["name"])
# If last match was Rainmaker
elif gamemode_key == "rainmaker":
# If player has S+ Rank
if last_match["udemae"]["name"] == "S+":
state = "Lvl: {}☆{}/R(RM): {}{}".format(
last_match["player_result"]["player"]
["player_rank"],
last_match["player_result"]["player"]
["star_rank"],
last_match["udemae"]["name"],
last_match["udemae"]["s_plus_number"])
# If player has X Rank
elif last_match["udemae"]["name"] == "X":
# Checks if Player has any X Power
# If Player has no X Power (yet XP)
if not last_match["x_power"]:
state = "Lvl: {}☆{}/R(RM): X(TBD)".format(
last_match["player_result"]
["player"]["player_rank"],
last_match["player_result"]
["player"]["star_rank"],
)
# If Player has X Power
else:
state = "Lvl: {}☆{}/R(RM): X({})".format(
last_match["player_result"]
["player"]["player_rank"],
last_match["player_result"]
["player"]["star_rank"],
last_match["x_power"])
# If player has other Ranks
else:
state = "Lvl: {}☆{}/R(RM): {}".format(
last_match["player_result"]["player"]
["player_rank"],
last_match["player_result"]["player"]
["star_rank"],
last_match["udemae"]["name"])
# If last match was Clam Blitz
elif gamemode_key == "clam_blitz":
# If player has S+ Rank
if last_match["udemae"]["name"] == "S+":
state = "Lvl: {}☆{}/R(CZ): {}{}".format(
last_match["player_result"]["player"]
["player_rank"],
last_match["player_result"]["player"]
["star_rank"],
last_match["udemae"]["name"],
last_match["udemae"]["s_plus_number"])
# If player has X Rank
elif last_match["udemae"]["name"] == "X":
# Checks if Player has any X Power
# If Player has no X Power (yet XP)
if not last_match["x_power"]:
state = "Lvl: {}☆{}/R(CZ): X(TBD)".format(
last_match["player_result"]
["player"]["player_rank"],
last_match["player_result"]
["player"]["star_rank"],
)
# If Player has X Power
else:
state = "Lvl: {}☆{}/R(CZ): X({})".format(
last_match["player_result"]
["player"]["player_rank"],
last_match["player_result"]
["player"]["star_rank"],
last_match["x_power"])
# If player has other Ranks
else:
state = "Lvl: {}☆{}/R(CZ): {}".format(
last_match["player_result"]["player"]
["player_rank"],
last_match["player_result"]["player"]
["star_rank"],
last_match["udemae"]["name"])
# If last match was in a League Pair/Team Lobby
elif lobby_key == "league_pair" or lobby_key == "league_team":
# Checks if Player has League Power
# If Player has no League Power (yet XP)
if not last_match["league_point"]:
state = "Lvl: {}☆{}/Power: TBD".format(
last_match["player_result"]["player"]
["player_rank"],
last_match["player_result"]["player"]
["star_rank"],
)
# If Player has League Power
else:
state = "Lvl: {}☆{}/Power: {}".format(
last_match["player_result"]["player"]
["player_rank"],
last_match["player_result"]["player"]
["star_rank"], last_match["league_point"])
# Friend Code
elif i == 1:
if not friend_code:
state = "FC: Not Given"
else:
state = "FC: {}".format(friend_code)
# Kill (Assist) / Death Ratio
elif i == 2:
state = "K(A)/D: {}({})/{}".format(
last_match["player_result"]["kill_count"],
last_match["player_result"]["assist_count"],
last_match["player_result"]["death_count"])
# Result and Percentages
elif i == 3:
details = last_match["my_team_result"]["name"]
try:
state = "{}% vs {}%".format(
last_match["my_team_percentage"],
last_match["other_team_percentage"])
except KeyError:
try:
state = "{} vs {}".format(
last_match["my_team_count"],
last_match["other_team_count"])
except KeyError:
state = "Gamemode not yet supported"
# Used Weapon and Total Points
elif i == 4:
state = "{}p".format(
last_match["player_result"]["game_paint_point"])
if show_weapon:
details = "{}".format(last_match["player_result"]
["player"]["weapon"]["name"])
else:
pass
if minutes_since < timeout_minutes:
RPC.update(details=details,
state=state,
large_image=gamemode_key,
small_image="default",
large_text=large_text)
else:
RPC.clear()
logger.debug("RPC cleared, not in game long enough")
time.sleep(time_interval)
def close(self):
try:
self.__conn.close()
except MySQLdb.Error,e:
logger.error('[-]....Mysql Error:%s' % e)
import os
import sys
import time
from shutil import copyfile, rmtree
from config.logger import logger
quiet = True # set this to false if you're having issues installing to debug stuff
if quiet:
quiet = "-q"
else:
quiet = ""
logger.info("Installing requirements...")
response = os.system(
'{} -m pip install -r requirements.txt {}'.format(sys.executable, quiet))
if response != 0:
logger.error(
"Failed to instal system wide (we recommend a venv if you're not!")
logger.info("Attempting to install as a user package")
response = os.system(
'{} -m pip install -r requirements.txt {} --user'.format(sys.executable, quiet))
if response == 0:
logger.info("Requirements installed! Go have some fun!")
sys.exit(0)
else:
logger.info(
"Something went wrong while installing requirements. Check your python setup and permissions.")
sys.exit(1)
# @Time : 2021/5/29 8:00 # @Author : 胡远 # @Github : https://github.com/QuixoteHY # @Email : [email protected] # @Describe : from config.logger import logger from config.utils import exit_now from downloader.video.youtube import video_youtube_download_path, download_youtube_video_by_youtube_dl_in_cmd import os from bs4 import BeautifulSoup andy_lee_work_path = os.path.join(video_youtube_download_path, 'Andy_Lee') if not os.path.exists(andy_lee_work_path): logger.error('工作目录不存在！%s' % andy_lee_work_path) exit_now() andy_lee_video_path = os.path.join(andy_lee_work_path, 'video') if not os.path.exists(andy_lee_video_path): os.makedirs(andy_lee_video_path) andy_lee_all_video_ids_file_path = os.path.join(andy_lee_work_path, 'all_video_ids.txt') andy_lee_downloaded_record_file_path = os.path.join(andy_lee_work_path, 'downloaded_video_id.txt') def read_downloaded_video_ids(): downloaded_video_ids = list() for file_name in os.listdir(andy_lee_video_path): video_id = file_name[9:20] if video_id not in downloaded_video_ids: downloaded_video_ids.append(video_id) return downloaded_video_ids
def set_record_shfe_future_holding_parse_date(downloaded_date):
if downloaded_date not in TRADE_CALENDAR:
logger.error('set_record_shfe_future_holding_parse_date(), %s日期不在交易日里内请检查！' % downloaded_date)
exit_now()
with open(record_shfe_future_holding_parse_date_path, 'w', encoding='utf-8') as f:
f.write(downloaded_date)
def parse_downloaded_origin_file_from_20150921(
trade_date, future_holding_downloaded_origin_file_path):
with open(future_holding_downloaded_origin_file_path,
'r',
encoding='utf-8') as f:
items = f.read().replace('合约', '品种').split('品种')
for index, item in enumerate(items):
lines = item.strip().replace('\n\n',
'\n').replace(' ',
'').replace(',',
'').split('\n')
logger.info('>' * 10 + trade_date + '\t' + '\n'.join(lines))
# 检查最后一行，合计
if '合计' not in lines[-1]:
if index == 0:
continue
else:
logger.error('缺少合计, ' + str(lines[-1]))
exit_now()
# 检查第二行表头
if '名次' not in lines[1]:
logger.error('缺少表头, ' + str(lines[1]))
exit_now()
# 第一行提取期货品种代码， 棉花CF日期：：2015-09-21
future_variety_code = None
czce_future_contract_name = lines[0].replace(
'：', '').split('日期')[0].strip()
check_res = 0
for __code, __item in FUTURE_VARIETIES.items():
if czce_future_contract_name == __item['品种全称']:
check_res += 1
future_variety_code = __code
if check_res > 1:
logger.error('合约代码与品种全称匹配数大于1, czce_future_contract_name: %s, %s' %
(czce_future_contract_name, lines[0]))
exit_now()
if future_variety_code is None:
if len(czce_future_contract_name) < 4 or czce_future_contract_name[-1] not in __digit or \
czce_future_contract_name[-2] not in __digit or czce_future_contract_name[-3] not in __digit:
logger.error('合约代码有问题, czce_future_contract_name: %s, %s' %
(czce_future_contract_name, lines[0]))
exit_now()
future_variety_code = czce_future_contract_name[:-3]
# 重新命名合约名称
future_contract_name = None
czce_future_contract_name, future_date_str = \
lines[0].strip().replace('-', '').replace(':', '').replace('：', '').split('日期')
if czce_future_contract_name == FUTURE_VARIETIES[future_variety_code][
'品种全称']:
future_contract_name = 'total'
elif future_variety_code in czce_future_contract_name:
future_contract_name = \
rename_czce_future_contract_name(trade_date, czce_future_contract_name, future_variety_code)
logger.info(czce_future_contract_name + ' >>> ' +
future_contract_name)
else:
logger.error('合约代码有问题, lines[0]: ' + str(lines[0]))
exit_now()
# 检查日期
if future_date_str != trade_date or len(future_date_str) != 8:
logger.error('日期对不上, lines[0]: ' + str(lines[0]))
exit_now()
# 解析数据
future_holding_dict = get_future_holding_dict()
future_holding_dict['code'] = future_contract_name
future_holding_dict['date'] = trade_date
for line in lines[2:-1]:
line = line.strip().split('|')
if line[1] != '-':
future_holding_dict['volume'][int(
line[0])] = [line[1], int(line[2]),
int(line[3])]
if line[4] != '-':
future_holding_dict['buy'][int(
line[0])] = [line[4], int(line[5]),
int(line[6])]
if line[7] != '-':
future_holding_dict['sell'][int(
line[0])] = [line[7], int(line[8]),
int(line[9])]
line = lines[-1].strip().split('|')
future_holding_dict['total_volume'] = [int(line[2]), int(line[3])]
future_holding_dict['total_buy'] = [int(line[5]), int(line[6])]
future_holding_dict['total_sell'] = [int(line[8]), int(line[9])]
# 写入文件
future_holding_contract_file_path = \
get_future_holding_contract_file_path(future_variety_code, future_contract_name)
insert_one_day_future_holding_data_to_js(
future_holding_contract_file_path, future_holding_dict)
