def testSubObj(self): num = 1.0 print(type(abs)) # 异常检查判断 self.assertRaises(Exception, CommonUtil.subObj, num, 0, 1) self.assertEqual([1, 3, 5], CommonUtil.subObj([1, 2, 3, 4, 5, 6], 0, 6, 2)) self.assertEqual((1, 3, 5), CommonUtil.subObj((1, 2, 3, 4, 5, 6), 0, 6, 2))
def getIndexStockByType(dir_path, index_type): heads = ['StockCode','StockName','Price',\ 'Trade','PB','PE_S','PE_D','EarningsPerShare','NetProfitDes','ROE','GrossProfitRate','NetAssetValuePerShare','CapitalStock',\ 'ScaleShareType','FinanceAnalize'] browser = webdriver.Chrome() browser.get('http://data.eastmoney.com/other/index/hs300.html') parent = browser.find_element_by_id("mk_type") boards = parent.find_elements_by_xpath('.//*') workbook = xlwt.Workbook() for i, board in enumerate(boards): board_name = board.text if board_name == index_type: sheet = addHeaders(workbook, board_name, heads) board.click() time.sleep(2) print("正在获取:" + board_name + "...") total_page = int( browser.find_element_by_xpath( '//*[@id="miniPageNav"]/b[4]/span').text) for page in range(1, total_page + 1): for j in range(50): cu.printProgress("第" + str(page) + "页-" + "第" + str(j) + "条...") company = browser.find_element_by_xpath( '//*[@id="dt_1"]/tbody/tr[' + str(j + 1) + ']/td[3]/a') code = browser.find_element_by_xpath( '//*[@id="dt_1"]/tbody/tr[' + str(j + 1) + ']/td[2]/a') price = browser.find_element_by_xpath( '//*[@id="dt_1"]/tbody/tr[' + str(j + 1) + ']/td[4]/span') row = (page - 1) * 50 + 1 + j sheet.write(row, 0, code.text) sheet.write(row, 1, company.text) sheet.write(row, 2, price.text) getBaseDataFromF10(code.text, row, 3, sheet) page_parent = browser.find_element_by_id("PageCont") page_btns = page_parent.find_elements_by_xpath('.//*') subscript = 0 if (len(page_btns) > 0) & (page != total_page + 1): for page_btn in page_btns: if page_btn.text == "下一页": subscript = page_btns.index(page_btn) page_btns[subscript].click() time.sleep(2) file_path = dir_path + board_name + '.xls' workbook.save(file_path) print('数据下载完毕,已保存到' + file_path) browser.close() browser.quit()
def isEmpty(string: str) -> bool: """ 判断空对象 :param string: :return: True False """ return CommonUtil.isEmpty(string)
def subList(ls: list, start: int, end: int, step: int = 1) -> list: """ 分割list :param ls: list :param start: 起始数 :param end: 结束数 :param step: 步长 :return: """ return CommonUtil.subObj(ls, start, end, step)
def subString(string: str, start: int, end: int, step: int = 1) -> str: """ 字符串切分 :param string: 需要切分的字符串 :param start: 开始 :param end: :param step: 步长 :return: """ return CommonUtil.subObj(string, start, end, step)
def __init__(self, position, radius, tag=False, scale=Vector2(1.0, 1.0)): import entity.EntityManager as EntityManager self.entity_id = CommonUtil.genID() EntityManager.register_entity(self) self.state_machine = StateMachine(self) self._position = position self._bounding_radius = radius self._tag = tag self.scale = scale
def __init__(self, configFile='redis.properties'): """ 初始化 默认: redis.properties :param configFile: 文件名 """ config = FileUtil.Properties(configFile) password = None if not CommonUtil.isEmpty(config.get('pass')): password = config.get('pass') super(RedisClient, self).__init__(host=config.get('host'), port=config.get('port', 6379), password=password)
def sendMail(self, mailTo: list, subject: str, content: str, cc: list = [], _charset: str = "UTF-8", mimeType: MimeType = MimeType.PLAIN ): """ 发送邮件的方法 :param mailTo: toAddr 发送到的邮箱地址 需要是数组 :param subject: 标题 :param content: 内容 :param cc 抄送 :param _charset 编码 :param mimeType 邮件格式 """ if not isinstance(mailTo, list): logging.error("mailTo: TypeError: must be list") return if cc is not None and not isinstance(cc, list): logging.error("cc: TypeError: must be list") return mail = MIMEMultipart() msg = MIMEText(content, _subtype=mimeType.value, _charset=_charset) mail.attach(msg) mail['To'] = self.__formatAddr("<%s>" % StringUtil.arrayToStr(mailTo, ';')) if not CommonUtil.isEmpty(cc): mail['Cc'] = StringUtil.arrayToStr(cc, ';') mailTo = mailTo + cc mail['Subject'] = Header(subject, 'UTF-8').encode() mail['From'] = self.__formatAddr(self.__fromAlias + ' <%s>' % self.__mailFrom) try: server = smtplib.SMTP(self.__smtpServer, self.__smtpPort) server.set_debuglevel(1) server.login(self.__mailFrom, self.__password) server.sendmail(self.__mailFrom, mailTo, mail.as_string()) except smtplib.SMTPException: logging.error("邮件发送失败") finally: server.quit()
def test_is_empty(self): self.assertTrue(CommonUtil.isEmpty("")) self.assertTrue(CommonUtil.isEmpty(None)) self.assertFalse(CommonUtil.isEmpty("me"))
def testAttr(self): print(CommonUtil.listAllAttr('123'))
def testFilter(self): nums = [1, 2, 3, 4, 5] # 使用了匿名函数 ds = CommonUtil.filterList(lambda x: x % 2 == 1, nums) self.assertEqual([1, 3, 5], ds)
def testMap(self): nums = [1, 2, 3, 4, 5] ds = CommonUtil.mapList(doubles, nums) self.assertEqual([2, 4, 6, 8, 10], ds)
def getStockDataByType(dir_path, stock_type=STOCK_TYPE_HSA, save_to_db=False): # StockCode:股票代码 # StockName:股票简称 # Price:当前股价 '-------每日浮动数据-------' # PriceLimit:涨跌幅 # QuantityRelativeR:量比 # TurnoverRate:换手率 '-------相对固定数据-------' # Trade:行业 # PB:市净率 # PE_S:静态市盈率 # PE_D:动态市盈率 # EarningsPerShare:每股收益 # NetProfitDes:净利润描述(包括净利润和增长比率) # ROE:净资产收益率 # GrossProfitRate:毛利率 # NetAssetValuePerShare:每股净资产 # CapitalStock:股本 # ScaleShareType:类型(大盘股、小盘股) # FinanceAnalize:财务分析 # MoneyFlowPerShare:每股现金流量 '-------备用字段-------' # Proceeds:营业收入 # ProceedsYOY:营业收入-同比增长 # ProceedsQOQ:营业收入-环比增长 # NetProfit:净利润 # NetProfitYOY:净利润-同比增长 # NetProfitQOQ:净利润-环比增长 # GrossProfitRate:销售毛利率 heads = ['StockCode','StockName','Price','PriceLimit','QuantityRelativeR','TurnoverRate',\ 'Trade','PB','PE_S','PE_D','EarningsPerShare','NetProfitDes','ROE','GrossProfitRate','NetAssetValuePerShare','CapitalStock',\ 'ScaleShareType','FinanceAnalize','MoneyFlowPerShare'] # 初始化webdriver browser = webdriver.Chrome() browser.get("http://quote.eastmoney.com/center/gridlist.html#hs_a_board") parent = browser.find_element_by_xpath('//*[@id="tab"]/ul') boards = parent.find_elements_by_xpath('.//*') # 初始化xl对象 workbook = xlwt.Workbook() #连接数据库 engine = create_engine( 'mysql+pymysql://root:root@localhost:3306/listed_company') try: for i, board in enumerate(boards): board_name = board.text if board_name == stock_type: sheet = addHeaders(workbook, board_name, heads) board.click() time.sleep(2) print("正在获取:" + board_name + "..." + '请勿关闭浏览器') page_parent = browser.find_element_by_class_name( 'paginate_page') pages = page_parent.find_elements_by_xpath('.//*') total_page = int(pages[len(pages) - 1].text) # total_page = 2 #统计每页缺失数据的数量 missing_count = 0 row_index = 0 for page in range(1, total_page + 1): for j in range(20): cu.printProgress("第" + str(page) + "页-" + "第" + str(j) + "条...") company = browser.find_element_by_xpath( '//*[@id="table_wrapper-table"]/tbody/tr[' + str(j + 1) + ']/td[3]/a') code = browser.find_element_by_xpath( '//*[@id="table_wrapper-table"]/tbody/tr[' + str(j + 1) + ']/td[2]/a') price = browser.find_element_by_xpath( '//*[@id="table_wrapper-table"]/tbody/tr[' + str(j + 1) + ']/td[5]/span') pl = browser.find_element_by_xpath( '//*[@id="table_wrapper-table"]/tbody/tr[' + str(j + 1) + ']/td[6]/span') qrr = browser.find_element_by_xpath( '//*[@id="table_wrapper-table"]/tbody/tr[' + str(j + 1) + ']/td[15]') tr = browser.find_element_by_xpath( '//*[@id="table_wrapper-table"]/tbody/tr[' + str(j + 1) + ']/td[16]') # 排除无价格、ST和退市的股票 if ((price.text == '-') | sdu.isST(company.text) | sdu.isDelist(company.text)): missing_count += 1 continue else: row_index += 1 if save_to_db: sql = 'INSERT INTO t_tonghua (stock_code,stock_name) VALUES ("{}","{}")'.format( code.text, company.text) engine.execute(sql) else: sheet.write(row_index, 0, code.text) sheet.write(row_index, 1, company.text) sheet.write(row_index, 2, price.text) sheet.write(row_index, 3, pl.text) sheet.write(row_index, 4, qrr.text) sheet.write(row_index, 5, tr.text) getBaseDataFromF10(code.text, row_index, 6, sheet) page_btn = browser.find_element_by_xpath( '//*[@id="main-table_paginate"]/a[2]') if page < total_page: page_btn.click() time.sleep(2) except Exception: traceback.print_exc() if save_to_db == False: #保存为xls文件 file_path = dir_path + board_name + '.xls' workbook.save(file_path) print('数据下载完毕,已保存到' + file_path) browser.close() browser.quit()
def getAnnualReports(dir_path, df, year): workbook = xlwt.Workbook() # stock_code:股票代码 # stock_name:股票简称 '-------财务数据-------' # total_revenue:营业收入 # total_revenue_gr:营业收入增长率 # net_profit_atsopc:净利润 # basic_eps:每股收益 # basic_eps_gr:每股收益增长率 # np_per_share:每股净资产 # operate_cash_flow_ps:现金流 # avg_roe:净资产收益率 # asset_liab_ratio:资产负债率 heads = [ 'stock_code', 'stock_name', 'total_revenue', 'total_revenue_gr', 'net_profit_atsopc', 'net_profit_atsopc_gr', 'basic_eps', 'basic_eps_gr', 'np_per_share', 'np_per_share_gr', 'operate_cash_flow_ps', 'operate_cash_flow_ps_gr', 'avg_roe', 'asset_liab_ratio' ] #添加表头 sheet = workbook.add_sheet(year + "年报") for h in range(len(heads)): sheet.write(0, h, heads[h]) row_index = 0 for index, row in df.iterrows(): stock_code = row['StockCode'] dict_ar = getAnnualReportByStockCode(stock_code, years[year]) if len(dict_ar) == 0: continue if year not in dict_ar['report_name']: continue row_index = row_index + 1 cu.printProgress('获取第' + str(row_index) + '条') sheet.write(row_index, 0, stock_code) sheet.write(row_index, 1, row['StockName']) sheet.write(row_index, 2, dict_ar['total_revenue'][0]) sheet.write(row_index, 3, dict_ar['total_revenue'][1]) sheet.write(row_index, 4, dict_ar['net_profit_atsopc'][0]) sheet.write(row_index, 5, dict_ar['net_profit_atsopc'][1]) sheet.write(row_index, 6, dict_ar['basic_eps'][0]) sheet.write(row_index, 7, dict_ar['basic_eps'][1]) sheet.write(row_index, 8, dict_ar['np_per_share'][0]) sheet.write(row_index, 9, dict_ar['np_per_share'][1]) sheet.write(row_index, 10, dict_ar['operate_cash_flow_ps'][0]) sheet.write(row_index, 11, dict_ar['operate_cash_flow_ps'][1]) sheet.write( row_index, 12, float(0 if dict_ar['avg_roe'][0] is None else dict_ar['avg_roe'][0] ) / 100) sheet.write(row_index, 13, float(dict_ar['asset_liab_ratio'][0]) / 100) # if index == 10: # break file_path = dir_path + year + '年报.xls' workbook.save(file_path) print('数据下载完毕,已保存到' + file_path)