def crack(self, *args, **kwargs): ip = args[1]['ip'] port = args[1]['port'] service = args[1]['service'] if not ip or not port or not service: return None userpath = '%s/%s' % (self.__get_crack_dic_path(), self.user_dict) passpath = '%s/%s' % (self.__get_crack_dic_path(), self.pass_dict) self.command = "%s -h %s -n %s -U %s -P %s -e ns -M %s -f -v 4 -t 16 -R 0" % (self.medusa_script, ip, str(port), userpath, passpath, service) print self.command start = datetime.datetime.now() process = subprocess.Popen(self.command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) while process.poll() is None: time.sleep(0.5) now = datetime.datetime.now() if (now - start).seconds > self.timeout: try: print process.pid self.normal_exit = False #process.terminate() #这里就不能在return 了 logger.info("medusa will be stopped because of crack [%s:%s] time out." % (ip, str(port))) except Exception,e: logger.error('Exception:%s' % str(e)) process.terminate() #这里就不能在return 了 process.kill() time.sleep(2)
def async_deal_into_db(self, bdomain, taskid): ''' 入库 :return: ''' while not self.threadpool.resultQueue.empty(): try: result_dit = self.threadpool.resultQueue.get(block = False) figerinfo = result_dit['figerinfo'] exist_result_list = result_dit['result_list'] for exist_result in exist_result_list: vulurl = {} vulurl['sid'] = taskid vulurl['url'] = exist_result['url'] vulurl['title'] = exist_result['title'] vulurl['keyword'] = '' vulurl['code'] = exist_result['http_code'] vulurl['figerinfo'] = figerinfo vulurl['first_time'] = getCurTime() if bdomain == DOMAIN_TYPE[0]: #sql = "insert into t_web_vulurl(`sid`, `url`, `title`, `keyword`, `code`, `figerinfo`, `first_time`) values('%s', '%s', '%s', '%s', '%s', '%s', '%s')" % (str(taskid), str(exist_result['url']) , str(exist_result['title']), '', str(exist_result['http_code']), figerinfo, getCurTime()) self.plugin_db.insert_by_dict(WEBVUL_TABLE , vulurl) elif bdomain == DOMAIN_TYPE[1]: self.plugin_db.insert_by_dict(WEBIPVUL_TABLE, vulurl) except Exception,e: logger.error(str(e)) break
def __del__(self): ''' 析构函数,关闭数据库的连接 ''' try: self.__conn.close() except MySQLdb.Error,e: logger.error('[-]....Mysql Error:%s' % e)
def createConnection(self): ''' 创建数据库连接 ''' if not self.__isMysqlDb(): return False try: self.__conn = MySQLdb.connect(host=self.__host,port=self.__port,user=self.__user,passwd=self.__passwd,charset='utf8') self.__conn.select_db(self.__db) except MySQLdb.Error,e: logger.error('[-]......Mysql Error:%s' % e) #self.__del__() return False
def __burlaccess(self): ''' 判断目标URL是否可以访问 :return: ''' try: request = requests.get(url = self.url, timeout = self.timeout, headers=HTTP_HEADERS, allow_redirects = True, verify = False) respinfo = HtmlRespInfo(request.status_code, request.text) self.exist_result_que.put_nowait({'respinfo':respinfo.trans(), 'url':self.url}) try: self.__whatweb() #获取网站指纹 except Exception,e: logger.error('[webscan]error:%s' % str(e)) return True
def executeUpdate(self, sql): ''' 对数据库进行添加,修改和删除的操作 ''' if sql is None: return False if self.__conn is None: self.createConnection() self.__cursor = self.cursor() try: self.__cursor.execute(sql) self.__conn.commit() return int(self.__cursor.lastrowid) except MySQLdb.Error, e: logger.error('[-].....Mysql update/insert Error:%s' % e) return False
def executeQuery(self, sql): ''' 对数据库进行查询操作 ''' if sql is None: return None if self.__conn is None: self.createConnection() self.__cursor = self.cursor() try: self.__cursor.execute(sql) result = self.__cursor.fetchall() return result except MySQLdb.Error, e: logger.error('[-]......Mysql Error:%s' % e) return None
def run(self): while True: try: if self.daemon: callbackfunc, args, kwargs = self.workQueue.get(block = True) #如果主线程退出,子线程也退出。这里设置阻塞 else: callbackfunc, args, kwargs = self.workQueue.get(block = True, timeout = self.timeout) res = callbackfunc(args, kwargs) #执行结果加入到结果队列中 if res is not None: self.resultQueue.put_nowait(res) except Queue.Empty: logger.info('work queue empty!') break except: logger.error(sys.exc_info()) raise time.sleep(0.3)
def send(cls, sub, content): ''''' send_mail("*****@*****.**","sub","content") ''' me = MAIL_USER+"<"+MAIL_USER+"@"+MAIL_POSTFIX+">" msg = MIMEText(content,_charset='gbk') msg['Subject'] = sub msg['From'] = me msg['To'] = ";".join(MAILTO_LIST) try: s = smtplib.SMTP() s.connect(MAIL_HOST) s.login(MAIL_USER,MAIL_PASS) s.sendmail(me, MAILTO_LIST, msg.as_string()) s.close() return True except Exception, e: logger.error(str(e)) return False
def download_history_future_basis(): url_model = 'http://www.100ppi.com/graph/future.php?f=graph_echarts&id=%s&source=%s' future_basis_downloaded_origin_path = get_future_basis_downloaded_origin_path() future_basis_history_downloaded_origin_path = os.path.join(future_basis_downloaded_origin_path, 'history') if not os.path.exists(future_basis_history_downloaded_origin_path): os.makedirs(future_basis_history_downloaded_origin_path) for future_code in FUTURES_BASIS.keys(): # "铝": {"url": "http://www.100ppi.com/sf/827.html"}, future_id = FUTURES_BASIS[future_code]['url'].split('sf/')[1].split('.')[0] logger.info('>>>>>>>>>>期货品种:'+future_code+', future_id='+future_id) # 现期图, ['现货价格','最近合约','主力合约'] request_url_one = url_model % (future_id, '1') content_html_one = download_page(request_url_one) if content_html_one is None: exit_now() content_json_one = parse_history_page(content_html_one) filename_one_path = \ os.path.join(future_basis_history_downloaded_origin_path, future_code+'_'+FUTURES_BASIS[future_code]['name']+'_'+future_id+'_'+'现期图.js') with open(filename_one_path, 'w', encoding='utf-8') as f: f.write(json.dumps(content_json_one)) # 基差图, ['最近基差','主力基差'] request_url_two = url_model % (future_id, '2') content_html_two = download_page(request_url_two) if content_html_two is None: exit_now() content_json_two = parse_history_page(content_html_two) filename_two_path = \ os.path.join(future_basis_history_downloaded_origin_path, future_code+'_'+FUTURES_BASIS[future_code]['name']+'_'+future_id+'_'+'基差图.js') with open(filename_two_path, 'w', encoding='utf-8') as f: f.write(json.dumps(content_json_two)) # 基差率图, ['最近基差率','主力基差率'] request_url_thr = url_model % (future_id, '3') content_html_thr = download_page(request_url_thr) if content_html_thr is None: exit_now() content_json_thr = parse_history_page(content_html_thr) filename_thr_path = \ os.path.join(future_basis_history_downloaded_origin_path, future_code+'_'+FUTURES_BASIS[future_code]['name']+'_'+future_id+'_'+'基差率图.js') with open(filename_thr_path, 'w', encoding='utf-8') as f: f.write(json.dumps(content_json_thr)) # 交易日期,品种代码,现货价格,最近合约,主力合约,最近基差,主力基差,最近基差率,主力基差率 conbine_all_list = list() conbine_all_list.append(FUTURE_BASIS_CSV_TITLE) if FUTURES_BASIS[future_code]['name'] != '': one_trade_date_list = content_json_one['xAxis']['data'] one_0_list = content_json_one['series'][0]['data'] one_1_list = content_json_one['series'][1]['data'] one_2_list = content_json_one['series'][2]['data'] two_trade_date_list = content_json_two['xAxis']['data'] two_0_list = content_json_two['series'][0]['data'] two_1_list = content_json_two['series'][1]['data'] thr_trade_date_list = content_json_thr['xAxis']['data'] thr_0_list = content_json_thr['series'][0]['data'] thr_1_list = content_json_thr['series'][1]['data'] if len(one_trade_date_list) != len(two_trade_date_list) or len(one_trade_date_list) != len(thr_1_list)\ or len(one_trade_date_list) != len(thr_trade_date_list) \ or len(one_trade_date_list) != len(one_0_list) or len(one_trade_date_list) != len(one_1_list) \ or len(one_trade_date_list) != len(one_2_list) or len(one_trade_date_list) != len(two_0_list) \ or len(one_trade_date_list) != len(two_1_list) or len(one_trade_date_list) != len(thr_0_list): logger.error('数据格式不对请检查') exit_now() for i in range(len(one_trade_date_list)): conbine_all_list.append(','.join([one_trade_date_list[i], future_code, str(one_0_list[i]), str(one_1_list[i]), str(one_2_list[i]), str(two_0_list[i]), str(two_1_list[i]), str(thr_0_list[i]), str(thr_1_list[i])])) future_basis_file_path = get_future_basis_file_path(future_code) with open(future_basis_file_path, 'w', encoding='utf-8') as f: f.write('\n'.join(conbine_all_list))
def close(self): try: self.__conn.close() except MySQLdb.Error, e: logger.error('[-]....Mysql Error:%s' % e)
def parse_one(trade_date): trading_future_contract_name_list = list() future_price_downloaded_origin_file_path = \ get_future_price_downloaded_origin_file_path(trade_date, FuturesExchange.CZCE) if not os.path.exists(future_price_downloaded_origin_file_path): logger.error('%s 文件不存在!' % future_price_downloaded_origin_file_path) return False with open(future_price_downloaded_origin_file_path, 'r', encoding='utf-8') as f: _items = f.read().split('小计') for index, item in enumerate(_items[:-1]): lines = item.strip().replace('\n\n', '\n').replace('\t', '').replace( ' ', '').replace(',', '').split('\n') # logger.info('>' * 10+trade_date+'\t'+'\n'.join(lines)) if index == 0: if not lines[1].startswith('合约代码'): logger.error( '数据格式有变动请检查, future_price_downloaded_origin_file_path=' + future_price_downloaded_origin_file_path) exit_now() lines = lines[2:] lines.insert(0, '|') # 检查最后一行 if not lines[0].startswith('|'): logger.error( '数据格式有变动请检查, future_price_downloaded_origin_file_path=' + future_price_downloaded_origin_file_path) exit_now() lines = lines[1:] # logger.info('>>>>>>>>>> trade_date='+trade_date+'\n'+str(lines)) # 开始解析 for line in lines: # 交易日期,合约名称,昨结算,开盘价,最高价,最低价,收盘价,今结算,成交量,持仓量,增减量,成交额(万元),交割结算价 # 合约代码|昨结算|今开盘|最高价|最低价|今收盘|今结算|涨跌1|涨跌2|成交量|持仓量|增减量|成交额(万元)|交割结算价 line = line.split('|') czce_future_contract_name = line[0] # 提取期货品种代码 if len(czce_future_contract_name) < 4 or czce_future_contract_name[-1] not in __digit or \ czce_future_contract_name[-2] not in __digit or czce_future_contract_name[-3] not in __digit: logger.error('合约代码有问题, czce_future_contract_name: %s' % czce_future_contract_name) exit_now() future_variety_code = czce_future_contract_name[:-3] # 重新命名合约名称 future_contract_name = rename_czce_future_contract_name( trade_date, czce_future_contract_name, future_variety_code) logger.info(czce_future_contract_name + ' >>> ' + future_contract_name) # 交易日期,合约名称,昨结算,开盘价,最高价,最低价,收盘价,今结算,成交量,持仓量,增减量,成交额(万元),交割结算价 new_line = [ trade_date, future_contract_name, line[1], line[2], line[3], line[4], line[5], line[6], line[9], line[10], line[11], line[12], line[13] ] logger.info('new_line=' + str(new_line)) # 将该条数据插入文件 future_price_contract_file_path = \ get_future_price_contract_file_path(future_variety_code, future_contract_name) logger.info('future_price_contract_file_path=' + future_price_contract_file_path) insert_one_day_future_price_data_to_csv( future_price_contract_file_path, new_line) trading_future_contract_name_list.append(future_contract_name) # 正在上市交易的合约 record_trading_future_contract_name_file_path = \ get_record_trading_future_contract_name_file_path(FuturesExchange.CZCE) with open(record_trading_future_contract_name_file_path, 'w', encoding='utf-8') as f: f.write('\n'.join(trading_future_contract_name_list)) return True
def get(self): logger.error('here') return {'message': 'ok'}, 200
def parse_one(trade_date): """ python3 zipfile 解压文件时出现文件名为乱码情况的解决方案: 这个问题我在自己代码中跟中转换编码都不行,总出错! 介绍一种便捷方法,直接改源码: 直接找到zipfile.py文件 第一处: if zinfo.flag_bits & 0x800: # UTF-8 filename fname_str = fname.decode("utf-8") else: # fname_str = fname.decode("cp437") fname_str = fname.decode("gbk") 第二处: if flags & 0x800: # UTF-8 file names extension filename = filename.decode('utf-8') else: # Historical ZIP filename encoding # filename = filename.decode('cp437') filename = filename.decode('gbk') :param trade_date: :return: """ future_holding_downloaded_origin_file_path = \ get_future_holding_downloaded_origin_file_path(trade_date, FuturesExchange.DCE) if not os.path.exists(future_holding_downloaded_origin_file_path): logger.error('%s 文件不存在!' % future_holding_downloaded_origin_file_path) return False zip_file = zipfile.ZipFile(future_holding_downloaded_origin_file_path, "r") for file_name in zip_file.namelist(): try: content = zip_file.read(file_name).decode('utf-8').strip().replace('\r', '').replace('\n\n', '\n') except UnicodeDecodeError: logger.info('数据编码有变动请检查, future_holding_downloaded_origin_file_path=' + future_holding_downloaded_origin_file_path+file_name) content = zip_file.read(file_name).decode('gbk').strip().replace('\r', '').replace('\n\n', '\n') # 检查名次到总计的间隔 check_content_one = re.findall(r'名次.*总计', content, re.S) if not check_content_one or len(check_content_one) != 1: logger.error('数据名次到总计的间隔有变动请检查, future_holding_downloaded_origin_file_path=' + future_holding_downloaded_origin_file_path + file_name) return False check_content_one_list = check_content_one[0].split('\n') if len(check_content_one_list) == 6: logger.info(content) logger.info('名次到总计的间隔为6,空!!!') continue lines = content.split('\n') if len(lines) < 8: logger.error('数据不足8行有变动请检查, future_holding_downloaded_origin_file_path=' + future_holding_downloaded_origin_file_path+file_name) return False # 检查前两行 __repair_trade_date = None if '大连商品交易所' not in lines[0] or '合约代码:' not in lines[1] or 'Date:' not in lines[1]: tmp_title = file_name.split('_') if len(tmp_title) != 5 or len(tmp_title[1]) <= 4 or tmp_title[0] != trade_date: logger.error('数据格式有变动或日期对不上请检查, future_holding_downloaded_origin_file_path=' + future_holding_downloaded_origin_file_path+file_name) return False future_variety_code = tmp_title[1][:-4].upper() _tmp_delivery_month = tmp_title[1][-4:] else: tmp_title = lines[1].replace('合约代码:', '').replace('Date:', '').replace('-', '').strip().split() if trade_date == '20161115' and tmp_title[1] == '20161114': # 修复大连期货交易所提供的龙虎榜数据 logger.warn('repair_trade_date, future_holding_downloaded_origin_file_path=' + future_holding_downloaded_origin_file_path+file_name) __repair_trade_date = tmp_title[1] elif tmp_title[1] != trade_date: logger.info(str(tmp_title)+' | '+trade_date) logger.error('数据日期对不上请检查, future_holding_downloaded_origin_file_path=' + future_holding_downloaded_origin_file_path+file_name) return False if len(tmp_title) != 2 or len(tmp_title[0]) <= 4: logger.error('数据格式有变动请检查, future_holding_downloaded_origin_file_path=' + future_holding_downloaded_origin_file_path+file_name) return False future_variety_code = tmp_title[0][:-4].upper() if future_variety_code not in FUTURE_VARIETIES: logger.error('期货品种代码有问题请检查, future_holding_downloaded_origin_file_path=' + future_holding_downloaded_origin_file_path+file_name) return False _tmp_delivery_month = tmp_title[0][-4:] # 重新命名合约名称 future_contract_name = '%s%s%s' % (future_variety_code, trade_date[:2], _tmp_delivery_month) # 检查数据是否完整 content = '\n'.join(lines).strip() items = content.split('名次') if len(items) != 4: logger.error('数据格式有变动三个不全请检查, future_holding_downloaded_origin_file_path=' + future_holding_downloaded_origin_file_path+file_name) return False content_volume = items[1].strip() content_buy = items[2].strip() content_sell = items[3].strip() logger.info('%s, %s, %s | %s, %s' % (tmp_title, future_variety_code, future_contract_name, file_name, future_holding_downloaded_origin_file_path)) # 开始解析 future_holding_dict = get_future_holding_dict() future_holding_dict['code'] = future_contract_name future_holding_dict['date'] = trade_date if __repair_trade_date is not None: # 修复大连期货交易所提供的龙虎榜数据 future_holding_dict['date'] = __repair_trade_date # 解析成交量 lines_volume = content_volume.split('\n') if '成交量' not in lines_volume[0] or '总计' not in lines_volume[-1]: logger.error('成交量数据格式表头有变动请检查, future_holding_downloaded_origin_file_path=' + future_holding_downloaded_origin_file_path+file_name) return False if len(lines_volume) > 2: tmp_str_total = lines_volume[-1].replace('总计', '').strip().replace(',', '').split() if len(tmp_str_total) == 1: total_v, tmp_total_v = int(tmp_str_total[0]), 0 total_i_a_d, tmp_total_i_a_d = 0, 0 elif len(tmp_str_total) == 2: total_v, tmp_total_v = int(tmp_str_total[0]), 0 total_i_a_d, tmp_total_i_a_d = int(tmp_str_total[1]), 0 else: logger.error('成交量数据格式表尾有变动请检查, future_holding_downloaded_origin_file_path=' + future_holding_downloaded_origin_file_path+file_name) return False for line_volume in lines_volume[1:-1]: line_volume = line_volume.split() if len(line_volume) != 4: logger.error('成交量数据格式有变动请检查, future_holding_downloaded_origin_file_path=' + future_holding_downloaded_origin_file_path+file_name) return False v = int(line_volume[2].replace(',', '')) i_a_d = int(line_volume[3].replace(',', '')) future_holding_dict['volume'][int(line_volume[0])] = [line_volume[1], v, i_a_d] tmp_total_v += v tmp_total_i_a_d += i_a_d logger.info(line_volume) if total_v != tmp_total_v: logger.error('成交量各期货会员总持仓核对不上, future_holding_downloaded_origin_file_path=' + future_holding_downloaded_origin_file_path+file_name) return False if total_i_a_d > 0 and total_i_a_d != tmp_total_i_a_d: logger.error('成交量各期货会员总持仓核对不上, future_holding_downloaded_origin_file_path=' + future_holding_downloaded_origin_file_path+file_name) return False future_holding_dict['total_volume'] = [tmp_total_v, tmp_total_i_a_d] # 解析持买单量 lines_buy = content_buy.split('\n') if '持买单量' not in lines_buy[0] or '总计' not in lines_buy[-1]: logger.error('持买单量数据格式表头有变动请检查, future_holding_downloaded_origin_file_path=' + future_holding_downloaded_origin_file_path+file_name) return False if len(lines_buy) > 2: tmp_str_total = lines_buy[-1].replace('总计', '').strip().replace(',', '').split() if len(tmp_str_total) == 1: total_v, tmp_total_v = int(tmp_str_total[0]), 0 total_i_a_d, tmp_total_i_a_d = 0, 0 elif len(tmp_str_total) == 2: total_v, tmp_total_v = int(tmp_str_total[0]), 0 total_i_a_d, tmp_total_i_a_d = int(tmp_str_total[1]), 0 else: logger.error('成交量数据格式表尾有变动请检查, future_holding_downloaded_origin_file_path=' + future_holding_downloaded_origin_file_path+file_name) return False for line_buy in lines_buy[1:-1]: line_buy = line_buy.split() if len(line_buy) != 4: logger.error('持买单量数据格式有变动请检查, future_holding_downloaded_origin_file_path=' + future_holding_downloaded_origin_file_path+file_name) return False v = int(line_buy[2].replace(',', '')) i_a_d = int(line_buy[3].replace(',', '')) future_holding_dict['buy'][int(line_buy[0])] = [line_buy[1], v, i_a_d] tmp_total_v += v tmp_total_i_a_d += i_a_d logger.info(line_buy) if total_v != tmp_total_v: logger.error('持买单量各期货会员总持仓核对不上, future_holding_downloaded_origin_file_path=' + future_holding_downloaded_origin_file_path+file_name) return False if total_i_a_d > 0 and total_i_a_d != tmp_total_i_a_d: logger.error('成交量各期货会员总持仓核对不上, future_holding_downloaded_origin_file_path=' + future_holding_downloaded_origin_file_path+file_name) return False future_holding_dict['total_buy'] = [tmp_total_v, tmp_total_i_a_d] # 解析持卖单量 lines_sell = list() tmp_lines_sell = content_sell.split('\n') for tmp_line in tmp_lines_sell: lines_sell.append(tmp_line) if '总计' in tmp_line: break if '持卖单量' not in lines_sell[0] or '总计' not in lines_sell[-1]: logger.error('持卖单量数据格式表头有变动请检查, future_holding_downloaded_origin_file_path=' + future_holding_downloaded_origin_file_path+file_name) return False if len(lines_sell) > 2: tmp_str_total = lines_sell[-1].replace('总计', '').strip().replace(',', '').split() if len(tmp_str_total) == 1: total_v, tmp_total_v = int(tmp_str_total[0]), 0 total_i_a_d, tmp_total_i_a_d = 0, 0 elif len(tmp_str_total) == 2: total_v, tmp_total_v = int(tmp_str_total[0]), 0 total_i_a_d, tmp_total_i_a_d = int(tmp_str_total[1]), 0 else: logger.error('成交量数据格式表尾有变动请检查, future_holding_downloaded_origin_file_path=' + future_holding_downloaded_origin_file_path+file_name) return False for line_sell in lines_sell[1:-1]: line_sell = line_sell.split() if len(line_sell) != 4: logger.error('持卖单量数据格式有变动请检查, future_holding_downloaded_origin_file_path=' + future_holding_downloaded_origin_file_path+file_name) return False v = int(line_sell[2].replace(',', '')) i_a_d = int(line_sell[3].replace(',', '')) future_holding_dict['sell'][int(line_sell[0])] = [line_sell[1], v, i_a_d] tmp_total_v += v tmp_total_i_a_d += i_a_d logger.info(line_sell) if total_v != tmp_total_v: logger.error('持卖单量各期货会员总持仓核对不上, future_holding_downloaded_origin_file_path=' + future_holding_downloaded_origin_file_path+file_name) return False if total_i_a_d > 0 and total_i_a_d != tmp_total_i_a_d: logger.error('成交量各期货会员总持仓核对不上, future_holding_downloaded_origin_file_path=' + future_holding_downloaded_origin_file_path+file_name) return False future_holding_dict['total_sell'] = [tmp_total_v, tmp_total_i_a_d] # 写入文件 future_holding_contract_file_path = \ get_future_holding_contract_file_path(future_variety_code, future_contract_name) insert_one_day_future_holding_data_to_js(future_holding_contract_file_path, future_holding_dict) # break zip_file.close() return True
def parse_downloaded_origin_file_from_20100818_to_20150918( trade_date, future_holding_downloaded_origin_file_path): with open(future_holding_downloaded_origin_file_path, 'r', encoding='utf-8') as f: items = f.read().replace('合约', '品种').split('品种') for index, item in enumerate(items): lines = item.strip().replace('\n\n', '\n').replace(' ', '').replace('\t', '').split('\n') logger.info('>' * 10 + trade_date + '\t' + '\n'.join(lines)) # 检查最后一行,合计 if '合计' not in lines[-1]: if index == 0: continue else: logger.error('缺少合计, ' + str(lines[-1])) exit_now() # 检查第一行, :棉花日期:2010-08-18 :ER105日期:2010-08-18 __future_varietties_dict = { 'CF': '棉花', 'RI': '早籼', 'OI': '菜油', 'SR': '白糖', 'TA': 'PTA', 'WS': '强麦', 'WT': '硬麦', 'MA': '甲醇', 'PM': '普麦', 'FG': '玻璃', 'RM': '菜粕', 'RS': '菜籽', # 'ZC': '动力煤', 'TC': '煤', 'JR': '粳稻', 'LR': '晚籼', 'SF': '硅铁', 'SM': '锰硅', } check_res = 0 future_variety_code = '' for _future_variety_code in FUTURE_VARIETIES.keys(): if _future_variety_code in lines[0]: check_res += 1 future_variety_code = _future_variety_code for _future_variety_code, _future_variety_name in __future_varietties_dict.items( ): if _future_variety_name in lines[0]: check_res += 1 future_variety_code = _future_variety_code if check_res != 1 and future_variety_code not in ('TA', ): logger.error('合约代码对不上, lines[0]: ' + str(lines[0]) + '|future_variety_code=' + future_variety_code) exit_now() # 重新命名合约名称 (:棉花日期:2010-08-18 :ER105日期:2010-08-18) future_contract_name = None czce_future_contract_name, future_date_str = \ lines[0].strip().replace('-', '').replace(':', '').replace(':', '').split('日期') if czce_future_contract_name in FUTURE_VARIETIES[future_variety_code][ '品种全称']: future_contract_name = 'total' elif future_variety_code in czce_future_contract_name: future_contract_name = \ rename_czce_future_contract_name(trade_date, czce_future_contract_name, future_variety_code) logger.info(czce_future_contract_name + ' >>> ' + future_contract_name) else: logger.error('合约代码有问题, lines[0]: ' + str(lines[0]) + ' | 品种全称=' + FUTURE_VARIETIES[future_variety_code]['品种全称']) exit_now() # 检查日期 if future_date_str != trade_date or len(future_date_str) != 8: logger.error('日期对不上, lines[0]: ' + str(lines[0])) exit_now() # 解析数据 future_holding_dict = get_future_holding_dict() future_holding_dict['code'] = future_contract_name future_holding_dict['date'] = trade_date for line in lines[1:-1]: if '.0' not in line: logger.error('格式不对请检查, line=' + line) exit_now() line = line.replace('.0', '') line = line.strip().split(',') logger.info('line=' + str(line)) if line[1] != '': future_holding_dict['volume'][int( line[0])] = [line[1], int(line[2]), int(line[3])] if line[4] != '': future_holding_dict['buy'][int( line[0])] = [line[4], int(line[5]), int(line[6])] if line[7] != '': future_holding_dict['sell'][int( line[0])] = [line[7], int(line[8]), int(line[9])] line = lines[-1] if '.0' not in line: logger.error('格式不对请检查, line=' + line) exit_now() line = line.replace('.0', '') line = line.strip().split(',') if line[1] != '': line.insert(1, '') logger.info('合计 line=' + str(line)) future_holding_dict['total_volume'] = [int(line[2]), int(line[3])] future_holding_dict['total_buy'] = [int(line[5]), int(line[6])] future_holding_dict['total_sell'] = [int(line[8]), int(line[9])] # 写入文件 future_holding_contract_file_path = \ get_future_holding_contract_file_path(future_variety_code, future_contract_name) insert_one_day_future_holding_data_to_js( future_holding_contract_file_path, future_holding_dict)
if __name__ == "__main__": logger.info("Checking for updates...") os.system("git pull") logger.info( "If updates were done, restart this script by using CTRL-C to terminate it, and re run it." ) # Make connection to Discord try: RPC = Presence(client_id) # Initialize the Presence class RPC.connect() # Start the handshake loop except pypresence.exceptions.InvalidPipe: logger.error( "Could not connect to the discord pipe. Please ensure it's running." ) exit(1) except FileNotFoundError: logger.error( "Could not connect to the discord pipe. Please ensure it's running." ) exit(1) # Load our current config config = nso_functions.get_config_file() # We run this just to ensure the data collection is A-OK get_minutes_since()
def get(self): ret = db.session.query(Device).first() logger.error('test') return {'message': ret.name, 'status': 0}
def main(): logger.info("Checking for updates...") os.system("git pull") logger.info( "If updates were done, restart this script by using CTRL-C to terminate it, and re run it." ) # Make connection to Discord try: RPC = Presence(client_id) # Initialize the Presence class RPC.connect() # Start the handshake loop except pypresence.exceptions.InvalidPipe: logger.error( "Could not connect to the discord pipe. Please ensure it's running." ) exit(1) except FileNotFoundError: logger.error( "Could not connect to the discord pipe. Please ensure it's running." ) exit(1) # Load our current config config = nso_functions.get_config_file() logger.info("Check discord!") # get friend code from config, and add config option if does not exist try: friend_code = config['friend_code'] except KeyError: config['friend_code'] = 'Unset' config_f = open("config.txt", "w") config_f.write(json.dumps( config, sort_keys=True, indent=4, )) config_f.close() friend_code = config['friend_code'] while True: # The presence will stay on as long as the program is running for i in range(0, 4): minutes_since, last_match = get_minutes_since() # int is here so we don't have funky floating point madness seconds_since = int(minutes_since * 60) hours_since = int(minutes_since / 60) if minutes_since >= 60: details = "Last match: {} hour(s) ago".format(hours_since) elif minutes_since > 1: details = "Last match: {} minute(s) ago".format( math.floor(minutes_since)) else: details = "Last match: {} second(s) ago".format(seconds_since) # job_result is only present in salmon run JSON if last_match.get('job_result') is not None: gamemode_key = "salmon_run" if last_match['job_result']['is_clear']: outcome = "winning" else: outcome = "losing" large_text = "Last match was Salmon Run, {} with {} eggs".format( outcome, last_match['job_score']) if i == 0: state = "Grade: {}".format( (last_match["grade"])["long_name"]) elif i == 1: state = "Friend code: {}".format(friend_code) elif i == 2: state = "Difficulty: {}".format( str(last_match["danger_rate"])) elif i == 3: state = "Played on {}".format( last_match['schedule']['stage']['name']) else: large_text = "Last match was {}, {} on {}".format( last_match["game_mode"]["name"], last_match["rule"]["name"], last_match["stage"]["name"]) gamemode_key = last_match["rule"]["key"] if i == 0: state = "Friend code: {}".format(friend_code) elif i == 1: state = "K/D: {}/{}".format( last_match["player_result"]["kill_count"], last_match["player_result"]["death_count"]) elif i == 2: details = last_match["my_team_result"]["name"] try: state = "{}% vs {}%".format( last_match["my_team_percentage"], last_match["other_team_percentage"]) except KeyError: try: state = "{} vs {}".format( last_match["my_team_count"], last_match["other_team_count"]) except KeyError: state = "Gamemode not yet supported" elif i == 3: state = "{}p".format( last_match["player_result"]["game_paint_point"]) if show_weapon: details = "{}".format(last_match["player_result"] ["player"]["weapon"]["name"]) else: pass if minutes_since < timeout_minutes: RPC.update(details=details, state=state, large_image=gamemode_key, small_image="default", large_text=large_text) else: RPC.clear() logger.debug("RPC cleared, not in game long enough") time.sleep(time_interval)
def parse_one(trade_date): """ 说明: (1) 价格:自2019年12月02日起,纤维板报价单位由元/张改为元/立方米 (2) 价格:元/吨,鸡蛋为元/500千克,纤维板为元/立方米,胶合板为元/张 (3) 成交量、持仓量:手(按单边计算) (4) 成交额:万元(按单边计算) (5) 涨跌=收盘价-前结算价 (6) 涨跌1=今结算价-前结算价 (7) 合约系列:具有相同月份标的期货合约的所有期权合约的统称 (8) 隐含波动率:根据期权市场价格,利用期权定价模型计算的标的期货合约价格波动率 :param trade_date: :return: """ trading_future_contract_name_list = list() future_price_downloaded_origin_file_path = \ get_future_price_downloaded_origin_file_path(trade_date, FuturesExchange.DCE) if not os.path.exists(future_price_downloaded_origin_file_path): logger.error('%s 文件不存在!' % future_price_downloaded_origin_file_path) return False with open(future_price_downloaded_origin_file_path, 'r', encoding='utf-8') as f: content_html = f.read().strip().replace(' ', '').replace( '\t', '').replace(' ', '').replace('\n', '') table_obj = re.findall(r'<table.*?</table>', content_html, re.S) if not table_obj or len(table_obj) != 1: logger.error( 'html数据格式有变动请检查, future_price_downloaded_origin_file_path=' + future_price_downloaded_origin_file_path) return False table_obj = table_obj[0] tr_obj_list = re.findall(r'<tr>.*?</tr>', table_obj, re.S) if not tr_obj_list or len(tr_obj_list) <= 1: logger.error( 'html数据格式有变动请检查, future_price_downloaded_origin_file_path=' + future_price_downloaded_origin_file_path) return False for index, tr_obj in enumerate(tr_obj_list): if index == 0: if '收盘价' not in tr_obj or '<th>' not in tr_obj: logger.error( 'html数据格式有变动请检查, future_price_downloaded_origin_file_path=' + future_price_downloaded_origin_file_path) return False continue td_obj_list = re.findall(r'<td>(.*?)</td>', tr_obj, re.S) if len(td_obj_list) != 14: logger.error( 'html数据格式有变动请检查, future_price_downloaded_origin_file_path=' + future_price_downloaded_origin_file_path) return False if '小计' in td_obj_list[0] or '总计' in td_obj_list[0]: continue # 重新命名合约名称 if td_obj_list[0] not in MAPPING_DCE_FUTURE_VARIETIES_CODE: logger.error( 'MAPPING_DCE_FUTURE_VARIETIES_CODE ERROR: %s, %s' % (td_obj_list[0], future_price_downloaded_origin_file_path)) return False future_variety_code = MAPPING_DCE_FUTURE_VARIETIES_CODE[td_obj_list[0]] delivery_month = td_obj_list[1] if len(delivery_month) != 4: logger.error( '交割月份 ERROR: %s, %s' % (td_obj_list[1], future_price_downloaded_origin_file_path)) return False future_contract_name = '%s%s%s' % (future_variety_code, trade_date[:2], delivery_month) # 行中各字段数据解析 # 商品名称 交割月份 开盘价 最高价 最低价 收盘价 前结算价 结算价 涨跌 涨跌1 成交量 持仓量 持仓量变化 成交额 # 交易日期,合约名称,昨结算,开盘价,最高价,最低价,收盘价,今结算,成交量,持仓量,增减量,成交额(万元),交割结算价 new_line = [ trade_date, future_contract_name, td_obj_list[6].replace(',', '') if td_obj_list[6] != '-' else '', td_obj_list[2].replace(',', '') if td_obj_list[2] != '-' else '', td_obj_list[3].replace(',', '') if td_obj_list[3] != '-' else '', td_obj_list[4].replace(',', '') if td_obj_list[4] != '-' else '', td_obj_list[5].replace(',', '') if td_obj_list[5] != '-' else '', td_obj_list[7].replace(',', '') if td_obj_list[7] != '-' else '', td_obj_list[10].replace(',', '') if td_obj_list[10] != '-' else '', td_obj_list[11].replace(',', '') if td_obj_list[11] != '-' else '', td_obj_list[12].replace(',', '') if td_obj_list[12] != '-' else '', td_obj_list[13].replace(',', '') if td_obj_list[13] != '-' else '', '' ] logger.info('new_line=' + str(new_line)) # 将该条数据插入文件 future_price_contract_file_path = \ get_future_price_contract_file_path(future_variety_code, future_contract_name) logger.info('future_price_contract_file_path=' + future_price_contract_file_path) insert_one_day_future_price_data_to_csv( future_price_contract_file_path, new_line) trading_future_contract_name_list.append(future_contract_name) # 正在上市交易的合约 record_trading_future_contract_name_file_path = \ get_record_trading_future_contract_name_file_path(FuturesExchange.DCE) with open(record_trading_future_contract_name_file_path, 'w', encoding='utf-8') as f: f.write('\n'.join(trading_future_contract_name_list)) return True
def read_data(self): future_price_dict = self.get_future_price_contract_data_by_future_contract_name( self.future_variety_code, self.future_contract_name) future_holding_dict = self.get_future_holding_contract_data_by_future_contract_name( self.future_variety_code, self.future_contract_name) future_members_buy_holding_dict = self.get_future_members_holding_dict( len(future_price_dict.keys())) future_members_sell_holding_dict = self.get_future_members_holding_dict( len(future_price_dict.keys())) index = 0 for trade_date, line in future_price_dict.items(): if trade_date not in self.trade_calendar: logger.error('交易日历有问题,请检查!') exit_now() self.trade_date_list.append(trade_date) # 收盘价 if float(line[6]) == 0: self.price_list.append(None) else: self.price_list.append(float(line[6])) # 持仓量 total_holding = float(line[9]) self.total_holding_list.append(total_holding) # 警戒线 self.cordon_list.append(0.5) if trade_date not in future_holding_dict: self.buy_list.append(None) self.sell_list.append(None) self.individual_buy_list.append(None) self.individual_sell_list.append(None) else: buy_holding = float( future_holding_dict[trade_date]['total_buy'][0]) sell_holding = float( future_holding_dict[trade_date]['total_sell'][0]) self.buy_list.append( (buy_holding / total_holding) * 100 / 100.0) self.sell_list.append( (sell_holding / total_holding) * 100 / 100.0) self.individual_buy_list.append(1 - (buy_holding / total_holding) * 100 / 100.0) self.individual_sell_list.append( 1 - (sell_holding / total_holding) * 100 / 100.0) # 多单持仓 buy = future_holding_dict[trade_date]['buy'] for value in buy.values(): future_members_buy_holding_dict[value[0]][index] = value[1] # 空单持仓 sell = future_holding_dict[trade_date]['sell'] for value in sell.values(): future_members_sell_holding_dict[ value[0]][index] = value[1] index += 1 for future_member_name, future_member_holding_list in future_members_buy_holding_dict.items( ): total_num = 0 for v in future_member_holding_list: if v is None: continue total_num += v if total_num != 0: self.new_future_members_buy_holding_dict[ future_member_name] = future_member_holding_list for future_member_name, future_member_holding_list in future_members_sell_holding_dict.items( ): total_num = 0 for v in future_member_holding_list: if v is None: continue total_num += v if total_num != 0: self.new_future_members_sell_holding_dict[ future_member_name] = future_member_holding_list
# 交易日历 __trade_calendar_file_path = os.path.join(os.path.join(work_path, 'resources'), 'trade_calendar.csv') __trade_calendar = list() today = time.strftime("%Y%m%d", time.localtime()) logger.info('today=' + today) with open(__trade_calendar_file_path, 'r', encoding='utf-8') as f: _tmp = f.read().strip().split('\n') for item_date in _tmp: item_date = item_date.split(',')[0] if item_date > today: continue __trade_calendar.append(item_date) if len(__trade_calendar) == 0 or today > _tmp[-1]: logger.error('请检查交易日历!') exit(1) TRADE_CALENDAR = __trade_calendar # 期货交易数据表头 # 涨跌1=收盘价-昨结算 # 涨跌2=结算价-昨结算 # 持仓量=空盘量 # 价格:元 / 吨 # 成交量、持仓量:手(单边计算) # 成交额:万元(单边计算) FUTURE_PRICE_CSV_TITLE = '交易日期,合约名称,昨结算,开盘价,最高价,最低价,收盘价,今结算,成交量,持仓量,增减量,成交额(万元),交割结算价' FUTURE_BASIS_CSV_TITLE = '交易日期,品种代码,现货价格,最近合约,主力合约,最近基差,主力基差,最近基差率,主力基差率' # 期货会员, 共147个 FUTURE_MEMBERS = {
def main(): logger.info("Checking for updates...") os.system("git pull") logger.info( "If updates were done, restart this script by using CTRL-C to terminate it, and re run it." ) # Make connection to Discord try: RPC = Presence(client_id) # Initialize the Presence class RPC.connect() # Start the handshake loop except pypresence.exceptions.InvalidPipe: logger.error( "Could not connect to the discord pipe. Please ensure it's running." ) exit(1) except FileNotFoundError: logger.error( "Could not connect to the discord pipe. Please ensure it's running." ) exit(1) # Load our current config config = nso_functions.get_config_file() logger.info("Check discord!") # Get friend code from config, and add config option if does not exist try: friend_code = config['friend_code'] except KeyError: config['friend_code'] = 'Unset' config_f = open("config.txt", "w") config_f.write(json.dumps( config, sort_keys=True, indent=4, )) config_f.close() friend_code = config['friend_code'] while True: # The presence will stay on as long as the program is running for i in range(0, 5): minutes_since, last_match = get_minutes_since() # Calculating the secs/hours/days since Last Match/Run seconds_since = int(minutes_since * 60) hours_since = int(minutes_since / 60) days_since = int(minutes_since / 1440) # When Previous Match was Salmon Run # job_result is only present in salmon run JSON if last_match.get('job_result') is not None: # Sets Gamemode Key in order to change the Picture gamemode_key = "salmon_run" # Decides if last Run is shown in days, hours, minutes or seconds # In Days if minutes_since >= 1440: details = "Last Run: {} day{} ago".format( days_since, plural_logic(days_since)) # In Hours elif minutes_since >= 60: details = "Last Run: {} h{} ago".format( hours_since, plural_logic(hours_since)) # In Minutes elif minutes_since > 1: details = "Last Run: {} min{} ago".format( math.floor(minutes_since), plural_logic(math.floor(minutes_since))) # In Seconds else: details = "Last Run: {} sec{} ago".format( seconds_since, plural_logic(seconds_since)) # Deciding the Result if last_match['job_result']['is_clear']: outcome = "WON" else: outcome = "LOST" ### Checks how many waves were played on last Run # If all 3 Waves were played if last_match["wave_details"][2]: goldEgg = last_match["wave_details"][0]["golden_ikura_num"] + \ last_match["wave_details"][1]["golden_ikura_num"] + \ last_match["wave_details"][2]["golden_ikura_num"] powEgg = last_match["wave_details"][0]["ikura_num"] + \ last_match["wave_details"][1]["ikura_num"] + \ last_match["wave_details"][2]["ikura_num"] # If only 2 Waves were played elif not last_match["wave_details"][2] and last_match[ "wave_details"][1]: goldEgg = last_match["wave_details"][0][ "golden_ikura_num"] + last_match["wave_details"][1][ "golden_ikura_num"] powEgg = last_match["wave_details"][0][ "ikura_num"] + last_match["wave_details"][1][ "ikura_num"] # If only 1 Wave was played else: goldEgg = last_match["wave_details"][0]["golden_ikura_num"] powEgg = last_match["wave_details"][0]["ikura_num"] # When hovering on the Picture large_text = "Last match was Salmon Run on {}".format( last_match['schedule']['stage']['name']) # IGN and Salmon Run Rank if i == 0: details = "IGN: {}".format(last_match["my_result"]["name"]) state = "{} {}".format((last_match["grade"])["long_name"], last_match["grade_point"]) # Friend code elif i == 1: if not friend_code: state = "FC: Not Given" else: state = "FC: {}".format(friend_code) # Hazard Level elif i == 2: state = "Hazard Level: {}".format( str(last_match["danger_rate"]) + "%") # Result and Total Collected Golden Eggs / Power Eggs elif i == 3: details = "GoldEgg/PowEgg ({})".format(outcome) state = "{} / {}".format(goldEgg, powEgg) # Save / Death Ratio elif i == 4: state = "Save/Death Ratio: {}/{}".format( last_match["my_result"]["help_count"], last_match["my_result"]["dead_count"]) if minutes_since < timeout_minutes: RPC.update(details=details, state=state, large_image=gamemode_key, small_image="default", large_text=large_text) else: RPC.clear() logger.debug("RPC cleared, not in game long enough") time.sleep(time_interval) # When Previous Match was Turf, Ranked, League or Private else: # Decides if last Match is shown in days, hours, minutes or seconds # In Days if minutes_since >= 1440: details = "Last Match: {} day{} ago".format( days_since, plural_logic(days_since)) # In Hours elif minutes_since >= 60: details = "Last Match: {} h{} ago".format( hours_since, plural_logic(hours_since)) # In Minutes elif minutes_since > 1: details = "Last Match: {} min{} ago".format( math.floor(minutes_since), plural_logic(math.floor(minutes_since))) # In Seconds else: details = "Last Match: {} sec{} ago".format( seconds_since, plural_logic(seconds_since)) # When hovering on the Picture large_text = "Last match was {}, {} on {}".format( last_match["game_mode"]["name"], last_match["rule"]["name"], last_match["stage"]["name"]) # Gets Gamemode Key in order to change the Picture gamemode_key = last_match["rule"]["key"] # Gets Lobby Key lobby_key = last_match["game_mode"]["key"] # IGN and Level (+ Rank) if i == 0: details = "IGN: {}".format( last_match["player_result"]["player"]["nickname"]) # Checks if player has a Level Star # If player has no Level Star (yet XP) if not last_match["star_rank"]: # If last match was in a Regular Lobby (Turf War) or Private Lobby if lobby_key == "regular" or lobby_key == "private": state = "Level: {}".format( last_match["player_result"]["player"] ["player_rank"], ) # If last match was in a Ranked Solo Lobby elif lobby_key == "gachi": # If last match was Splat Zones if gamemode_key == "splat_zones": # If player has S+ Rank if last_match["udemae"]["name"] == "S+": state = "Lvl: {}/R(SZ): {}{}".format( last_match["player_result"]["player"] ["player_rank"], last_match["udemae"]["name"], last_match["udemae"]["s_plus_number"]) # If player has X Rank elif last_match["udemae"]["name"] == "X": # Checks if Player has any X Power # If Player has no X Power (yet XP) if not last_match["x_power"]: state = "Lvl: {}/R(SZ): X(TBD)".format( last_match["player_result"] ["player"]["player_rank"], ) # If Player has X Power else: state = "Lvl: {}/R(SZ): X({})".format( last_match["player_result"] ["player"]["player_rank"], last_match["x_power"]) # If player has other Ranks else: state = "Lvl: {}/R(SZ): {}".format( last_match["player_result"]["player"] ["player_rank"], last_match["udemae"]["name"]) # If last match was Tower Control elif gamemode_key == "tower_control": # If player has S+ Rank if last_match["udemae"]["name"] == "S+": state = "Lvl: {}/R(TC): {}{}".format( last_match["player_result"]["player"] ["player_rank"], last_match["udemae"]["name"], last_match["udemae"]["s_plus_number"]) # If player has X Rank elif last_match["udemae"]["name"] == "X": # Checks if Player has any X Power # If Player has no X Power (yet XP) if not last_match["x_power"]: state = "Lvl: {}/R(TC): X(TBD)".format( last_match["player_result"] ["player"]["player_rank"], ) # If Player has X Power else: state = "Lvl: {}/R(TC): X({})".format( last_match["player_result"] ["player"]["player_rank"], last_match["x_power"]) # If player has other Ranks else: state = "Lvl: {}/R(TC): {}".format( last_match["player_result"]["player"] ["player_rank"], last_match["udemae"]["name"]) # If last match was Rainmaker elif gamemode_key == "rainmaker": # If player has S+ Rank if last_match["udemae"]["name"] == "S+": state = "Lvl: {}/R(RM): {}{}".format( last_match["player_result"]["player"] ["player_rank"], last_match["udemae"]["name"], last_match["udemae"]["s_plus_number"]) # If player has X Rank elif last_match["udemae"]["name"] == "X": # Checks if Player has any X Power # If Player has no X Power (yet XP) if not last_match["x_power"]: state = "Lvl: {}/R(RM): X(TBD)".format( last_match["player_result"] ["player"]["player_rank"], ) # If Player has X Power else: state = "Lvl: {}/R(RM): X({})".format( last_match["player_result"] ["player"]["player_rank"], last_match["x_power"]) # If player has other Ranks else: state = "Lvl: {}/R(RM): {}".format( last_match["player_result"]["player"] ["player_rank"], last_match["udemae"]["name"]) # If last match was Clam Blitz else: # If player has S+ Rank if last_match["udemae"]["name"] == "S+": state = "Lvl: {}/R(CB): {}{}".format( last_match["player_result"]["player"] ["player_rank"], last_match["udemae"]["name"], last_match["udemae"]["s_plus_number"]) # If player has X Rank elif last_match["udemae"]["name"] == "X": # Checks if Player has any X Power # If Player has no X Power (yet XP) if not last_match["x_power"]: state = "Lvl: {}/R(CB): X(TBD)".format( last_match["player_result"] ["player"]["player_rank"], ) # If Player has X Power else: state = "Lvl: {}/R(CB): X({})".format( last_match["player_result"] ["player"]["player_rank"], last_match["x_power"]) # If player has other Ranks else: state = "Lvl: {}/R(CB): {}".format( last_match["player_result"]["player"] ["player_rank"], last_match["udemae"]["name"]) # If last match was in a League Pair/Team Lobby elif lobby_key == "league_pair" or lobby_key == "league_team": # Checks if Player has League Power # If Player has no League Power (yet XP) if not last_match["league_point"]: state = "Lvl: {}/Power: TBD".format( last_match["player_result"]["player"] ["player_rank"]) # If Player has League Power else: state = "Lvl: {}/Power: {}".format( last_match["player_result"]["player"] ["player_rank"], last_match["league_point"]) # If player has a Level Star else: # If last match was in a Regular Lobby (Turf War) or Private Lobby if lobby_key == "regular" or lobby_key == "private": state = "Level: {}☆{}".format( last_match["player_result"]["player"] ["player_rank"], last_match["player_result"]["player"] ["star_rank"], ) # If last match was in a Ranked Solo Lobby elif lobby_key == "gachi": # If last match was Splat Zones if gamemode_key == "splat_zones": # If player has S+ Rank if last_match["udemae"]["name"] == "S+": state = "Lvl: {}☆{}/R(SZ): {}{}".format( last_match["player_result"]["player"] ["player_rank"], last_match["player_result"]["player"] ["star_rank"], last_match["udemae"]["name"], last_match["udemae"]["s_plus_number"]) # If player has X Rank elif last_match["udemae"]["name"] == "X": # Checks if Player has any X Power # If Player has no X Power (yet XP) if not last_match["x_power"]: state = "Lvl: {}☆{}/R(SZ): X(TBD)".format( last_match["player_result"] ["player"]["player_rank"], last_match["player_result"] ["player"]["star_rank"], ) # If Player has X Power else: state = "Lvl: {}☆{}/R(SZ): X({})".format( last_match["player_result"] ["player"]["player_rank"], last_match["player_result"] ["player"]["star_rank"], last_match["x_power"]) # If player has other Ranks else: state = "Lvl: {}☆{}/R(SZ): {}".format( last_match["player_result"]["player"] ["player_rank"], last_match["player_result"]["player"] ["star_rank"], last_match["udemae"]["name"]) # If last match was Tower Control elif gamemode_key == "tower_control": # If player has S+ Rank if last_match["udemae"]["name"] == "S+": state = "Lvl: {}☆{}/R(TC): {}{}".format( last_match["player_result"]["player"] ["player_rank"], last_match["player_result"]["player"] ["star_rank"], last_match["udemae"]["name"], last_match["udemae"]["s_plus_number"]) # If player has X Rank elif last_match["udemae"]["name"] == "X": # Checks if Player has any X Power # If Player has no X Power (yet XP) if not last_match["x_power"]: state = "Lvl: {}☆{}/R(TC): X(TBD)".format( last_match["player_result"] ["player"]["player_rank"], last_match["player_result"] ["player"]["star_rank"], ) # If Player has X Power else: state = "Lvl: {}☆{}/R(TC): X({})".format( last_match["player_result"] ["player"]["player_rank"], last_match["player_result"] ["player"]["star_rank"], last_match["x_power"]) # If player has other Ranks else: state = "Lvl: {}☆{}/R(TC): {}".format( last_match["player_result"]["player"] ["player_rank"], last_match["player_result"]["player"] ["star_rank"], last_match["udemae"]["name"]) # If last match was Rainmaker elif gamemode_key == "rainmaker": # If player has S+ Rank if last_match["udemae"]["name"] == "S+": state = "Lvl: {}☆{}/R(RM): {}{}".format( last_match["player_result"]["player"] ["player_rank"], last_match["player_result"]["player"] ["star_rank"], last_match["udemae"]["name"], last_match["udemae"]["s_plus_number"]) # If player has X Rank elif last_match["udemae"]["name"] == "X": # Checks if Player has any X Power # If Player has no X Power (yet XP) if not last_match["x_power"]: state = "Lvl: {}☆{}/R(RM): X(TBD)".format( last_match["player_result"] ["player"]["player_rank"], last_match["player_result"] ["player"]["star_rank"], ) # If Player has X Power else: state = "Lvl: {}☆{}/R(RM): X({})".format( last_match["player_result"] ["player"]["player_rank"], last_match["player_result"] ["player"]["star_rank"], last_match["x_power"]) # If player has other Ranks else: state = "Lvl: {}☆{}/R(RM): {}".format( last_match["player_result"]["player"] ["player_rank"], last_match["player_result"]["player"] ["star_rank"], last_match["udemae"]["name"]) # If last match was Clam Blitz elif gamemode_key == "clam_blitz": # If player has S+ Rank if last_match["udemae"]["name"] == "S+": state = "Lvl: {}☆{}/R(CZ): {}{}".format( last_match["player_result"]["player"] ["player_rank"], last_match["player_result"]["player"] ["star_rank"], last_match["udemae"]["name"], last_match["udemae"]["s_plus_number"]) # If player has X Rank elif last_match["udemae"]["name"] == "X": # Checks if Player has any X Power # If Player has no X Power (yet XP) if not last_match["x_power"]: state = "Lvl: {}☆{}/R(CZ): X(TBD)".format( last_match["player_result"] ["player"]["player_rank"], last_match["player_result"] ["player"]["star_rank"], ) # If Player has X Power else: state = "Lvl: {}☆{}/R(CZ): X({})".format( last_match["player_result"] ["player"]["player_rank"], last_match["player_result"] ["player"]["star_rank"], last_match["x_power"]) # If player has other Ranks else: state = "Lvl: {}☆{}/R(CZ): {}".format( last_match["player_result"]["player"] ["player_rank"], last_match["player_result"]["player"] ["star_rank"], last_match["udemae"]["name"]) # If last match was in a League Pair/Team Lobby elif lobby_key == "league_pair" or lobby_key == "league_team": # Checks if Player has League Power # If Player has no League Power (yet XP) if not last_match["league_point"]: state = "Lvl: {}☆{}/Power: TBD".format( last_match["player_result"]["player"] ["player_rank"], last_match["player_result"]["player"] ["star_rank"], ) # If Player has League Power else: state = "Lvl: {}☆{}/Power: {}".format( last_match["player_result"]["player"] ["player_rank"], last_match["player_result"]["player"] ["star_rank"], last_match["league_point"]) # Friend Code elif i == 1: if not friend_code: state = "FC: Not Given" else: state = "FC: {}".format(friend_code) # Kill (Assist) / Death Ratio elif i == 2: state = "K(A)/D: {}({})/{}".format( last_match["player_result"]["kill_count"], last_match["player_result"]["assist_count"], last_match["player_result"]["death_count"]) # Result and Percentages elif i == 3: details = last_match["my_team_result"]["name"] try: state = "{}% vs {}%".format( last_match["my_team_percentage"], last_match["other_team_percentage"]) except KeyError: try: state = "{} vs {}".format( last_match["my_team_count"], last_match["other_team_count"]) except KeyError: state = "Gamemode not yet supported" # Used Weapon and Total Points elif i == 4: state = "{}p".format( last_match["player_result"]["game_paint_point"]) if show_weapon: details = "{}".format(last_match["player_result"] ["player"]["weapon"]["name"]) else: pass if minutes_since < timeout_minutes: RPC.update(details=details, state=state, large_image=gamemode_key, small_image="default", large_text=large_text) else: RPC.clear() logger.debug("RPC cleared, not in game long enough") time.sleep(time_interval)
def close(self): try: self.__conn.close() except MySQLdb.Error,e: logger.error('[-]....Mysql Error:%s' % e)
import os import sys import time from shutil import copyfile, rmtree from config.logger import logger quiet = True # set this to false if you're having issues installing to debug stuff if quiet: quiet = "-q" else: quiet = "" logger.info("Installing requirements...") response = os.system( '{} -m pip install -r requirements.txt {}'.format(sys.executable, quiet)) if response != 0: logger.error( "Failed to instal system wide (we recommend a venv if you're not!") logger.info("Attempting to install as a user package") response = os.system( '{} -m pip install -r requirements.txt {} --user'.format(sys.executable, quiet)) if response == 0: logger.info("Requirements installed! Go have some fun!") sys.exit(0) else: logger.info( "Something went wrong while installing requirements. Check your python setup and permissions.") sys.exit(1)
# @Time : 2021/5/29 8:00 # @Author : 胡远 # @Github : https://github.com/QuixoteHY # @Email : [email protected] # @Describe : from config.logger import logger from config.utils import exit_now from downloader.video.youtube import video_youtube_download_path, download_youtube_video_by_youtube_dl_in_cmd import os from bs4 import BeautifulSoup andy_lee_work_path = os.path.join(video_youtube_download_path, 'Andy_Lee') if not os.path.exists(andy_lee_work_path): logger.error('工作目录不存在!%s' % andy_lee_work_path) exit_now() andy_lee_video_path = os.path.join(andy_lee_work_path, 'video') if not os.path.exists(andy_lee_video_path): os.makedirs(andy_lee_video_path) andy_lee_all_video_ids_file_path = os.path.join(andy_lee_work_path, 'all_video_ids.txt') andy_lee_downloaded_record_file_path = os.path.join(andy_lee_work_path, 'downloaded_video_id.txt') def read_downloaded_video_ids(): downloaded_video_ids = list() for file_name in os.listdir(andy_lee_video_path): video_id = file_name[9:20] if video_id not in downloaded_video_ids: downloaded_video_ids.append(video_id) return downloaded_video_ids
def set_record_shfe_future_holding_parse_date(downloaded_date): if downloaded_date not in TRADE_CALENDAR: logger.error('set_record_shfe_future_holding_parse_date(), %s日期不在交易日里内请检查!' % downloaded_date) exit_now() with open(record_shfe_future_holding_parse_date_path, 'w', encoding='utf-8') as f: f.write(downloaded_date)
def parse_downloaded_origin_file_from_20150921( trade_date, future_holding_downloaded_origin_file_path): with open(future_holding_downloaded_origin_file_path, 'r', encoding='utf-8') as f: items = f.read().replace('合约', '品种').split('品种') for index, item in enumerate(items): lines = item.strip().replace('\n\n', '\n').replace(' ', '').replace(',', '').split('\n') logger.info('>' * 10 + trade_date + '\t' + '\n'.join(lines)) # 检查最后一行,合计 if '合计' not in lines[-1]: if index == 0: continue else: logger.error('缺少合计, ' + str(lines[-1])) exit_now() # 检查第二行表头 if '名次' not in lines[1]: logger.error('缺少表头, ' + str(lines[1])) exit_now() # 第一行提取期货品种代码, 棉花CF日期::2015-09-21 future_variety_code = None czce_future_contract_name = lines[0].replace( ':', '').split('日期')[0].strip() check_res = 0 for __code, __item in FUTURE_VARIETIES.items(): if czce_future_contract_name == __item['品种全称']: check_res += 1 future_variety_code = __code if check_res > 1: logger.error('合约代码与品种全称匹配数大于1, czce_future_contract_name: %s, %s' % (czce_future_contract_name, lines[0])) exit_now() if future_variety_code is None: if len(czce_future_contract_name) < 4 or czce_future_contract_name[-1] not in __digit or \ czce_future_contract_name[-2] not in __digit or czce_future_contract_name[-3] not in __digit: logger.error('合约代码有问题, czce_future_contract_name: %s, %s' % (czce_future_contract_name, lines[0])) exit_now() future_variety_code = czce_future_contract_name[:-3] # 重新命名合约名称 future_contract_name = None czce_future_contract_name, future_date_str = \ lines[0].strip().replace('-', '').replace(':', '').replace(':', '').split('日期') if czce_future_contract_name == FUTURE_VARIETIES[future_variety_code][ '品种全称']: future_contract_name = 'total' elif future_variety_code in czce_future_contract_name: future_contract_name = \ rename_czce_future_contract_name(trade_date, czce_future_contract_name, future_variety_code) logger.info(czce_future_contract_name + ' >>> ' + future_contract_name) else: logger.error('合约代码有问题, lines[0]: ' + str(lines[0])) exit_now() # 检查日期 if future_date_str != trade_date or len(future_date_str) != 8: logger.error('日期对不上, lines[0]: ' + str(lines[0])) exit_now() # 解析数据 future_holding_dict = get_future_holding_dict() future_holding_dict['code'] = future_contract_name future_holding_dict['date'] = trade_date for line in lines[2:-1]: line = line.strip().split('|') if line[1] != '-': future_holding_dict['volume'][int( line[0])] = [line[1], int(line[2]), int(line[3])] if line[4] != '-': future_holding_dict['buy'][int( line[0])] = [line[4], int(line[5]), int(line[6])] if line[7] != '-': future_holding_dict['sell'][int( line[0])] = [line[7], int(line[8]), int(line[9])] line = lines[-1].strip().split('|') future_holding_dict['total_volume'] = [int(line[2]), int(line[3])] future_holding_dict['total_buy'] = [int(line[5]), int(line[6])] future_holding_dict['total_sell'] = [int(line[8]), int(line[9])] # 写入文件 future_holding_contract_file_path = \ get_future_holding_contract_file_path(future_variety_code, future_contract_name) insert_one_day_future_holding_data_to_js( future_holding_contract_file_path, future_holding_dict)