def run(self): http_session = get_http_session(self.mode) # 抓取沪市 ETF 列表 url = 'http://query.sse.com.cn/commonQuery.do?sqlId=COMMON_SSE_ZQPZ_ETFLB_L_NEW' text = sync_get(http_session, url, headers=DEFAULT_SH_ETF_LIST_HEADER, return_type='text') if text is None: return response_dict = demjson.decode(text) df = pd.DataFrame(response_dict.get('result', [])) self.persist_etf_list(df, exchange='sh') self.logger.info('沪市 ETF 列表抓取完成...') # 抓取沪市 ETF 成分股 self.download_sh_etf_component(df, http_session) self.logger.info('沪市 ETF 成分股抓取完成...') # 抓取深市 ETF 列表 url = 'http://www.szse.cn/api/report/ShowReport?SHOWTYPE=xlsx&CATALOGID=1945' content = sync_get(http_session, url, return_type='content') if content is None: return df = pd.read_excel(io.BytesIO(content), dtype=str) self.persist_etf_list(df, exchange='sz') self.logger.info('深市 ETF 列表抓取完成...') # 抓取深市 ETF 成分股 self.download_sz_etf_component(df, http_session) self.logger.info('深市 ETF 成分股抓取完成...')
def run(self): http_session = get_http_session(self.mode) # 上证、中证 self.fetch_csi_index(http_session) # 深证 self.fetch_szse_index(http_session)
async def async_process(self, entities, prog_count): """Create http_session to concurrently fetch tickers.""" logger.info(f'{self.name} http_session for {len(entities)} entities') http_session = get_http_session(self.mode) tasks = [] for entity in entities: task = asyncio.create_task(self.func(entity, http_session)) task.add_done_callback( functools.partial(self.async_callback, entity, self.result_queue, prog_count)) tasks.append(task) await asyncio.wait(tasks) await http_session.close()
def run_sync(self): http_session = get_http_session(self.mode) # Get all tasks while True: # logger.info(f'{self.name} running, task empty: {self.task_queue.empty()}') if self.task_queue.empty(): # logger.info(f'{self.name} break loop') return entity = self.task_queue.get() self.task_queue.task_done() try: self.func(entity, http_session) except Exception as e: logger.error(f'{self.func.__name__} entity:{entity} error {e}') # symbol = entity if isinstance(entity, str) else entity.code # self.result_queue.put(f'{symbol} done') self.prog_count.value += 1
}, "keyword3": { "value": '{:.2%}'.format(change_pct), "color": "#173177" }, "remark": { "value": "会所嫩模 Or 下海干活?", "color": "#173177" } } } return the_json if __name__ == '__main__': email_action = EmailInformer() email_action.send_message(["*****@*****.**", '*****@*****.**'], 'helo', 'just a test', sub_size=20) http_session = get_http_session() weixin_action = WechatInformer(http_session) weixin_action.send_price_notification(http_session, to_user='******', security_name='BTC/USDT', current_price=1000, change_pct='0.5%') # the __all__ is generated __all__ = ['Informer', 'EmailInformer', 'WechatInformer']