def _get_global_country_name_url() -> dict: """ 可获得指数数据国家对应的 URL :return: URL :rtype: dict """ url = "https://cn.investing.com/indices/" res = session.post(url, headers=short_headers) soup = BeautifulSoup(res.text, "lxml") name_url_option_list = soup.find("select", attrs={ "name": "country" }).find_all("option")[1:] # 去掉-所有国家及地区 url_list = [item["value"] for item in name_url_option_list] name_list = [item.get_text() for item in name_url_option_list] name_code_map_dict = {} name_code_map_dict.update(zip(name_list, url_list)) return name_code_map_dict
def index_investing_global_country_name_url(country: str = "中国") -> dict: """ 参考网页: https://cn.investing.com/indices/ 获取选择国家对应的: 主要指数, 主要行业, 附加指数, 其他指数 :param country: str 中文国家名称, 对应 get_global_country_name_url 函数返回的国家名称 :return: dict """ pd.set_option("mode.chained_assignment", None) name_url_dict = _get_global_country_name_url() name_code_dict = _get_global_index_country_name_url() url = f"https://cn.investing.com{name_url_dict[country]}?&majorIndices=on&primarySectors=on&additionalIndices=on&otherIndices=on" res = session.post(url, headers=short_headers) soup = BeautifulSoup(res.text, "lxml") url_list = [ item.find("a")["href"] for item in soup.find_all(attrs={"class": "plusIconTd"}) ] name_list = [ item.find("a").get_text() for item in soup.find_all(attrs={"class": "plusIconTd"}) ] name_code_map_dict = {} name_code_map_dict.update(zip(name_list, url_list)) url = "https://cn.investing.com/indices/global-indices" params = { "majorIndices": "on", "primarySectors": "on", "bonds": "on", "additionalIndices": "on", "otherIndices": "on", "c_id": name_code_dict[country], } r = session.get(url, params=params, headers=short_headers) data_text = r.text soup = BeautifulSoup(data_text, "lxml") soup_list = soup.find("table", attrs={"id": "cr_12"}).find_all("a") global_index_url = [item["href"] for item in soup_list] global_index_name = [item["title"] for item in soup_list] name_code_map_dict.update(zip(global_index_name, global_index_url)) return name_code_map_dict
def index_investing_global_from_url( url: str = "https://www.investing.com/indices/ftse-epra-nareit-eurozone", period: str = "每日", start_date: str = "20000101", end_date: str = "20210909", ) -> pd.DataFrame: """ 获得具体指数的从 start_date 到 end_date 期间的数据 https://www.investing.com/indices/ftse-epra-nareit-eurozone :param url: 具体数据链接 :type url: str :param period: choice of {"每日", "每周", "每月"} :type period: str :param start_date: '2000-01-01', 注意格式 :type start_date: str :param end_date: '2019-10-17', 注意格式 :type end_date: str :return: 指定参数的数据 :rtype: pandas.DataFrame """ start_date = "/".join([start_date[:4], start_date[4:6], start_date[6:]]) end_date = "/".join([end_date[:4], end_date[4:6], end_date[6:]]) period_map = {"每日": "Daily", "每周": "Weekly", "每月": "Monthly"} url_name = url.split("/")[-1] temp_url = f"https://cn.investing.com/indices/{url_name}-historical-data" res = session.post(temp_url, headers=short_headers) soup = BeautifulSoup(res.text, "lxml") title = soup.find("h2", attrs={"class": "float_lang_base_1"}).get_text() res = session.post(temp_url, headers=short_headers) soup = BeautifulSoup(res.text, "lxml") data = soup.find_all( text=re.compile("window.histDataExcessInfo"))[0].strip() para_data = re.findall(r"\d+", data) payload = { "curr_id": para_data[0], "smlID": para_data[1], "header": title, "st_date": start_date, "end_date": end_date, "interval_sec": period_map[period], "sort_col": "date", "sort_ord": "DESC", "action": "historical_data", } url = "https://cn.investing.com/instruments/HistoricalDataAjax" res = session.post(url, data=payload, headers=long_headers) df_data = pd.read_html(res.text)[0] if period == "每月": df_data.index = pd.to_datetime(df_data["日期"], format="%Y年%m月") else: df_data.index = pd.to_datetime(df_data["日期"], format="%Y年%m月%d日") if any(df_data["交易量"].astype(str).str.contains("-")): df_data["交易量"][df_data["交易量"].str.contains("-")] = df_data["交易量"][ df_data["交易量"].str.contains("-")].replace("-", 0) if any(df_data["交易量"].astype(str).str.contains("B")): df_data["交易量"][df_data["交易量"].str.contains("B").fillna(False)] = ( df_data["交易量"][df_data["交易量"].str.contains("B").fillna(False)].str. replace("B", "").str.replace(",", "").astype(float) * 1000000000) if any(df_data["交易量"].astype(str).str.contains("M")): df_data["交易量"][df_data["交易量"].str.contains("M").fillna(False)] = ( df_data["交易量"][df_data["交易量"].str.contains("M").fillna(False)].str. replace("M", "").str.replace(",", "").astype(float) * 1000000) if any(df_data["交易量"].astype(str).str.contains("K")): df_data["交易量"][df_data["交易量"].str.contains("K").fillna(False)] = ( df_data["交易量"][df_data["交易量"].str.contains("K").fillna(False)].str. replace("K", "").str.replace(",", "").astype(float) * 1000) df_data["交易量"] = df_data["交易量"].astype(float) df_data = df_data[["收盘", "开盘", "高", "低", "交易量"]] df_data = df_data.astype(float) df_data.sort_index(inplace=True) df_data.reset_index(inplace=True) df_data["日期"] = pd.to_datetime(df_data["日期"]).dt.date return df_data
def bond_investing_global( country: str = "中国", index_name: str = "中国1年期国债", period: str = "每日", start_date: str = "20000101", end_date: str = "20191017", ) -> pd.DataFrame: """ 具体国家的具体指数的从 start_date 到 end_date 期间的数据 https://cn.investing.com/rates-bonds/ :param country: 对应函数中的国家名称 :type country: str :param index_name: 对应函数中的指数名称 :type index_name: str :param period: choice of {"每日", "每周", "每月"} :type period: str :param start_date: '2000-01-01', 注意格式 :type start_date: str :param end_date: '2019-10-17', 注意格式 :type end_date: str :return: 指定参数的数据 :rtype: pandas.DataFrame """ start_date = "/".join([start_date[:4], start_date[4:6], start_date[6:]]) end_date = "/".join([end_date[:4], end_date[4:6], end_date[6:]]) period_map = {"每日": "Daily", "每周": "Weekly", "每月": "Monthly"} name_code_dict = bond_investing_global_country_name_url(country) temp_url = f"https://cn.investing.com/{name_code_dict[index_name]}-historical-data" res = session.get(temp_url, headers=short_headers, timeout=30) soup = BeautifulSoup(res.text, "lxml") title = soup.find("h2", attrs={"class": "float_lang_base_1"}).get_text() data = soup.find_all( text=re.compile("window.histDataExcessInfo"))[0].strip() para_data = re.findall(r"\d+", data) payload = { "curr_id": para_data[0], "smlID": para_data[1], "header": title, "st_date": start_date, "end_date": end_date, "interval_sec": period_map[period], "sort_col": "date", "sort_ord": "DESC", "action": "historical_data", } url = "https://cn.investing.com/instruments/HistoricalDataAjax" res = session.post(url, data=payload, headers=long_headers, timeout=60) df_data = pd.read_html(res.text)[0] df_data.columns = [ '日期', '收盘', '开盘', '高', '低', '涨跌幅', ] if period == "每月": df_data.index = pd.to_datetime(df_data["日期"], format="%Y年%m月") else: df_data.index = pd.to_datetime(df_data["日期"], format="%Y年%m月%d日") df_data = df_data[["收盘", "开盘", "高", "低", "涨跌幅"]] df_data["涨跌幅"] = df_data["涨跌幅"].str.replace("%", "") df_data["涨跌幅"] = df_data["涨跌幅"].str.replace(",", "") df_data = df_data.astype(float) return df_data