def ino(site): """ live forex rates scraped from ino.com """ uri = "https://assets.ino.com/data/history/" try: data = {} symbols = ["CNY", "RUB", "EUR", "JPY", "KRW", "GBP"] for symbol in symbols: query = f"?s=FOREX_USD{symbol}&b=&f=json" url = uri + query ret = requests.get(url, timeout=(15, 15)).json()[-1][-2] data["USD:" + symbol] = float(ret) symbols = ["XAG", "XAU"] for symbol in symbols: query = f"?s=FOREX_{symbol}USDO&b=&f=json" url = uri + query ret = requests.get(url, timeout=(15, 15)).json()[-1][-2] data["USD:" + symbol] = 1 / float(ret) data = refine_data(data) print(site, data) race_write(f"{site}_forex.txt", json_dumps(data)) except: print(f"{site} failed to load")
def aastock(site): """ live forex rates scraped from aastock.com (morningstar backdoor) """ uri = "http://www.aastocks.com/en/resources/datafeed/getrtcurconvert.ashx?curr=" symbols = "USDCNY,USDEUR,USDGBP,USDKRW,USDJPY,USDXAU,USDXAG" url = uri + symbols try: raw = requests.get(url).json() data = {} for item in raw: if item["to"] == "USD": if item["from"] in ["XAU", "XAG"]: data[item["to"] + ":" + item["from"]] = 1 / float(item["price"]) else: data[item["symbol"].replace("USD", "USD:")] = float(item["price"]) data = refine_data(data) print(site, data) race_write(f"{site}_forex.txt", json_dumps(data)) except: print(f"{site} failed to load")
def forextime(site): """ live forex rates scraped from forextime.com """ def parse(values): """ extract price from raw json """ price = (float(values["ask"]) + float(values["bid"])) / 2 if values["name"][-3:] == "USD": price = 1 / price return price uri = "https://www.forextime.com/informers/rates/symbols?symbols=" symbols = "EURUSD,GBPUSD,USDRUB,USDJPY,XAUUSD,XAGUSD" url = uri + symbols try: raw = requests.get(url, timeout=(15, 15)).json() data = {} for pair, values in raw.items(): price = parse(values) symbol = "USD:" + pair.replace("USD", "") data[symbol] = price data = refine_data(data) print(site, data) race_write(f"{site}_forex.txt", json_dumps(data)) except: print(f"{site} failed to load")
def forexrates(site): """ live forex rates scraped from forexrates.net """ def parse(raw): """ remove xml tags, return list of rates split with = """ lines = [i.split(";")[0] for i in raw.split("Values")] lines.pop(0) return [ i.replace('"', "").replace("[", "").replace("]", "").replace(" ", "") for i in lines ] url = "https://www.forexrates.net/widget/FR-FRW-2.php?" symbols = "c1=USD/EUR&c2=USD/GBP&c3=USD/RUB&c4=USD/JPY&c5=USD/CNY" url += symbols try: raw = requests.get(url, timeout=20).text rates = parse(raw) data = {} for rate in rates: symbol = rate.split("=")[0].replace("USD", "USD:") price = float(rate.split("=")[1]) data[symbol] = price data = refine_data(data) print(site, data) race_write(f"{site}_forex.txt", json_dumps(data)) except: print(f"{site} failed to load")
def barchart(site): """ https://ondemand.websol.barchart.com/getQuote.json?apikey=key&symbols=AAPL%2CGOOG limit 400 per day """ key = config_apikeys()[site] url = "https://marketdata.websol.barchart.com/getQuote.json" symbols = "^USDEUR,^USDJPY,^USDGBP,^USDCNY,^USDKRW,^USDRUB,^XAGUSD,^XAUUSD" params = {"symbols": symbols, "apikey": key} try: ret = requests.get(url, params=params, timeout=(15, 15)).json()["results"] data = {} for item in ret: try: data[item["symbol"].replace("^USD", "USD:")] = float(item["lastPrice"]) except: pass data["USD:XAG"] = 1 / data.pop("^XAGUSD") data["USD:XAU"] = 1 / data.pop("^XAUUSD") data = refine_data(data) print(site, data) race_write(f"{site}_forex.txt", json_dumps(data)) except: print(f"{site} failed to load")
def wocu(site): """ live forex rates scraped from wocu.com XAU XAG are not precise """ def parse(raw, symbol): """ attempt to extract a float value from the html matrix """ return float(raw.split(symbol)[1].split("</td>")[2].split(">")[1]) url = "http://54.154.247.217/wocutab.php" symbols = ["EUR", "RUB", "GBP", "KRW", "CNY", "JPY"] try: raw = requests.get(url, timeout=(15, 15)).text data = {} for symbol in symbols: try: data["USD:" + symbol] = parse(raw, symbol) except: pass data = refine_data(data) print(site, data) race_write(f"{site}_forex.txt", json_dumps(data)) except: print(f"{site} failed to load")
def fscapi(site): """ https://fcsapi.com/document/forex-api https://fcsapi.com/api/forex/latest?symbol=USD/JPY&access_key=key limit 10 per minute """ key = config_apikeys()[site] url = "https://fcsapi.com/api/forex/latest" symbols = "USD/EUR,USD/JPY,USD/GBP,USD/CNY,USD/KRW,USD/RUB,XAU/USD,XAG/USD" params = {"symbol": symbols, "access_key": key} try: ret = requests.get(url, params=params, timeout=(15, 15)).json()["response"] data = {} for item in ret: try: data[item["symbol"].replace("/", ":")] = float( item["price"].replace(",", "")) except: pass data["USD:XAG"] = 1 / data.pop("XAG:USD") data["USD:XAU"] = 1 / data.pop("XAU:USD") data = refine_data(data) print(site, data) race_write(f"{site}_forex.txt", json_dumps(data)) except: print(f"{site} failed to load")
def duckduckgo(site): """ live forex rates scraped from XE via duckduckgo.com """ uri = "https://duckduckgo.com/js/spice/currency/1/usd/" try: data = {} currencies = ["XAU", "XAG", "KRW", "JPY", "RUB"] # AUD,CAD,COP,EUR,GBP,INR,MXN,MYR,ZAR containted in topConversions by default for currency in currencies: if currency in ["XAU", "XAG"]: url = uri.replace("usd/", "") + currency + "/usd" else: url = uri + currency raw = requests.get(url, timeout=(15, 15)).text raw = (raw.replace("\n", "").replace(" ", "").replace("ddg_spice_currency(", "").replace(");", "")) ret = json_loads(raw) if currency in ["XAU", "XAG"]: data["USD:" + currency] = 1 / float( ret["conversion"]["converted-amount"]) else: data["USD:" + currency] = float( ret["conversion"]["converted-amount"]) time.sleep(1) for item in ret["topConversions"]: data["USD:" + item["to-currency-symbol"]] = float( item["converted-amount"]) data = refine_data(data) print(site, data) race_write(f"{site}_forex.txt", json_dumps(data)) except: print(f"{site} failed to load")
def fixerio(site): """ http://data.fixer.io/api/latest?access_key=key&base=USD&symbols=AUD,CAD limit 1000 per month (hourly updates) NOTE: XAU and XAG are NOT ACCURATE (SUPPORT TICKET OPEN) """ key = config_apikeys()[site] url = "http://data.fixer.io/api/latest" symbols = "USD,RUB,GBP,CNY,KRW,JPY" params = {"symbols": symbols, "access_key": key} try: ret = requests.get(url, params=params, timeout=(15, 15)).json()["rates"] eurusd = float(ret["USD"]) usdeur = 1 / eurusd data = { "USD:EUR": usdeur, "USD:RUB": float(ret["RUB"]) * usdeur, "USD:GBP": float(ret["GBP"]) * usdeur, "USD:CNY": float(ret["CNY"]) * usdeur, "USD:KRW": float(ret["KRW"]) * usdeur, "USD:JPY": float(ret["JPY"]) * usdeur, } data = refine_data(data) print(site, data) race_write(f"{site}_forex.txt", json_dumps(data)) except: print(f"{site} failed to load")
def oanda(site): """ make external request, decode, decrypt, reformat to dict """ key = "aaf6cb4f0ced8a211c2728328597268509ade33040233a11af" url = "https://www1.oanda.com/lfr/rates_lrrr?tstamp=" def hex_decode(raw): """ latin-1 from hexidecimal """ return bytes.fromhex("0" + raw if len(raw) % 2 else raw).decode("latin-1") def rc4(cypher, key): """ decryption of rc4 stream cypher from latin-1 """ idx1 = 0 output = [] r256 = [*range(256)] for idx2 in range(256): idx1 = (idx1 + r256[idx2] + ord(cypher[idx2 % len(cypher)])) % 256 r256[idx2], r256[idx1] = r256[idx1], r256[idx2] idx1, idx2 = 0, 0 for _, item in enumerate(key): idx2 = (idx2 + 1) % 256 idx1 = (idx1 + r256[idx2]) % 256 r256[idx2], r256[idx1] = r256[idx1], r256[idx2] output.append( chr(ord(item) ^ r256[(r256[idx2] + r256[idx1]) % 256])) return ("").join(output) try: while True: try: millies = str(int(round(time.time() * 1000))) raw = requests.get(url + millies, timeout=(15, 15)).text hex_decoded = hex_decode(raw) decrypted = rc4(key, hex_decoded) break except: time.sleep(5) content = decrypted.split("\n") parsed = { raw.split("=")[0]: (float(raw.split("=")[1]) + float(raw.split("=")[2])) / 2 for raw in content } data = {} for pair, price in parsed.items(): data[pair.replace("/", ":")] = float(price) data = refine_data(data) print(site, data) race_write(f"{site}_forex.txt", json_dumps(data)) except: print(f"{site} failed to load")
def bitcoinaverage(site): """ live forex rates scraped from bitcoinaverage.com """ url = "https://apiv2.bitcoinaverage.com/frontend/constants/exchangerates/local" try: session = requests.Session() cfscrape_requests = cfscrape.create_scraper(sess=session) ret = cfscrape_requests.get(url, timeout=(15, 15)).json()["rates"] data = {} for key, val in ret.items(): data["USD:" + key] = float(val["rate"]) data = refine_data(data) print(site, data) race_write(f"{site}_forex.txt", json_dumps(data)) except: print(f"{site} failed to load")
def freeforex(site): """ live forex rates scraped from freeforexapi.com """ url = "https://www.freeforexapi.com/api/live?pairs=" currencies = "EURUSD,GBPUSD,USDJPY" url += currencies try: ret = requests.get(url, timeout=(15, 15)).json()["rates"] data = {} for key, val in ret.items(): symbol = key[:3] + ":" + key[-3:] data[symbol] = float(val["rate"]) data = refine_data(data) print(site, data) race_write(f"{site}_forex.txt", json_dumps(data)) except: print(f"{site} failed to load")
def reuters(site): """ live forex rates scraped from reuters.com (refinitiv) """ uri = "https://www.reuters.com/companies/api/getFetchQuotes/" symbols = "USDCNY,USDJPY,USDKRW,USDRUB,USDGBP,USDEUR" url = uri + symbols try: raw = requests.get(url, timeout=(15, 15)).json() ret = raw["market_data"]["currencypairs"] data = {} for item in ret: price = (float(item["bid"]) + float(item["ask"])) / 2 data[item["symbol"].replace("USD", "USD:")] = price data = refine_data(data) print(site, data) race_write(f"{site}_forex.txt", json_dumps(data)) except: print(f"{site} failed to load")
def wsj(site): """ live forex rates scraped from wsj.com """ uri = "https://api.wsj.net/api/deltoro-mw/marketwatchsite/quote/currency/convert" try: currencies = ["EUR", "CNY", "RUB", "KRW", "JPY"] data = {} for currency in currencies: endpoint = f"?from=USD&to={currency}USD&amount=1.00" url = uri + endpoint raw = requests.get(url, timeout=(15, 15)).text data["USD:" + currency] = float(raw) time.sleep(1) data = refine_data(data) print(site, data) race_write(f"{site}_forex.txt", json_dumps(data)) except: print(f"{site} failed to load")
def investing(site): """ live forex rates scraped from investing.com https://www.investing.com/webmaster-tools/live-currency-cross-rates """ url = ("https://www.widgets.investing.com/live-currency-cross-rates?" + "theme=darkTheme&cols=last&pairs=3,2111,2124,2126,650,962711,69,68") headers = { "href": ("https://www.investing.com?utm_source=WMT&utm_medium=referral&" + "utm_campaign=LIVE_CURRENCY_X_RATES&utm_content=Footer%20Link"), "target": "_blank", "rel": "nofollow", } try: session = requests.Session() session.headers = headers cfscrape_requests = cfscrape.create_scraper(sess=session) ret = cfscrape_requests.get(url, headers=headers, timeout=(15, 15)).text lines = ret.split('target="_blank"') lines = [i.replace(" ", "").replace(",", "") for i in lines] lines = [i for i in lines if "askpid" in i] lines = [i.split("hiddenFour")[0] for i in lines] data = {} for item in lines: data[item.split("</a>")[0].replace(">", "")] = item.split('last">')[1] data = { k.replace("/", ":"): v.split("</div>")[0] for k, v in data.items() } data = {k: float(v) for k, v in data.items()} data["USD:XAG"] = 1 / data.pop("XAG:USD") data["USD:XAU"] = 1 / data.pop("XAU:USD") data = refine_data(data) print(site, data) race_write(f"{site}_forex.txt", json_dumps(data)) except: print(f"{site} failed to load")
def exchangeratewidget(site): """ live forex rates scraped from exchangeratewidget.com """ url = "https://www.exchangeratewidget.com/converter.php?v=11&t=" symbols = ["USDEUR", "USDGBP", "USDJPY", "USDCNY", "USDRUB", "USDKRW"] for symbol in symbols: url += symbol + "," try: data = {} raw = requests.get(url, timeout=20).text for symbol in symbols: currency = symbol.replace("USD", "") price = raw.split(currency)[1].split("</span>")[1].split(">")[1] data["USD:" + currency] = float(price) data = refine_data(data) print(site, data) race_write(f"{site}_forex.txt", json_dumps(data)) except: print(f"{site} failed to load")
def fxmarket(site): """ https://fixer.io/documentation https://fxmarketapi.com/apilive?api_key=key¤cy=EURUSD,GBPUSD 1000 / month """ key = config_apikeys()[site] url = "https://fxmarketapi.com/apilive" symbols = "USDEUR,USDGBP,USDCNY,USDKRW,USDRUB,USDJPY,XAUUSD,XAGUSD" params = {"currency": symbols, "api_key": key} try: ret = requests.get(url, params=params, timeout=(15, 15)).json()["price"] data = {k.replace("USD", "USD:"): float(v) for k, v in ret.items()} data["USD:XAG"] = 1 / data.pop("XAGUSD:") data["USD:XAU"] = 1 / data.pop("XAUUSD:") data = refine_data(data) print(site, data) race_write(f"{site}_forex.txt", json_dumps(data)) except: print(f"{site} failed to load")
def fxrate(site): """ live forex rates scraped from fx-rate.net """ def parse(raw): """ extract data from xml """ ret = raw.split('title="American Dollar to')[1] ret = ret.split("per")[0].split(";")[-1].replace(" ", "") return float(ret) def parse_metal(raw): """ extract data from xml """ ret = raw.split('to American Dollar Rates')[1] ret = ret.split("per")[0].split(";")[-1].replace(" ", "") return 1 / float(ret) try: data = {} uri = "https://fx-rate.net/conversion.php?currency=USD¤cy_pair=" symbols = ["EUR", "RUB", "GBP", "KRW", "CNY", "JPY"] for symbol in symbols: url = uri + symbol raw = requests.get(url, timeout=(15, 15)).text data["USD:" + symbol] = parse(raw) symbols = ["XAG", "XAU"] uri = "https://fx-rate.net/conversion.php?" for symbol in symbols: endpoint = f"currency={symbol}¤cy_pair=USD" url = uri + endpoint raw = requests.get(url, timeout=(15, 15)).text data["USD:" + symbol] = parse_metal(raw) data = refine_data(data) print(site, data) race_write(f"{site}_forex.txt", json_dumps(data)) except: print(f"{site} failed to load")
def openexchangerates(site): """ https://docs.openexchangerates.org/ https://openexchangerates.org/api/latest.json?app_id=key limit 1000 per month (hourly updates) """ key = config_apikeys()[site] url = "https://openexchangerates.org/api/latest.json" params = {"app_id": key} symbols = ["EUR", "RUB", "GBP", "KRW", "CNY", "JPY", "XAU", "XAG"] try: ret = requests.get(url, params=params, timeout=(15, 15)).json()["rates"] data = {} for key, val in ret.items(): if key in symbols: data["USD:" + key] = float(val) data = refine_data(data) print(site, data) race_write(f"{site}_forex.txt", json_dumps(data)) except: print(f"{site} failed to load")
def liveusd(site): """ live forex rates scraped from liveusd.com """ url = "http://liveusd.com/veri/refresh/total.php" try: ret = requests.get(url, timeout=(15, 15)).text ret = ret.replace(" ", "").split("\n") data = {} for item in ret: if item: try: pair = item.split(":")[0].replace("USD", "USD:") price = item.split(":")[1] data[pair] = float(price) except: pass data = refine_data(data) print(site, data) race_write(f"{site}_forex.txt", json_dumps(data)) except: print(f"{site} failed to load")
def currencyconverter(site): """ https://free.currconv.com/api/v7/convert?q=USD_PHP&compact=ultra&apikey=key 100/hour two pairs per request """ key = config_apikeys()[site] url = "https://free.currconv.com/api/v7/convert" symbols = ["USD_EUR,USD_GBP", "USD_CNY,USD_KRW", "USD_JPY,USD_RUB"] try: data = {} for symbol in symbols: try: params = {"compact": "y", "apikey": key, "q": symbol} ret = requests.get(url, params=params, timeout=(15, 15)).json() for key, val in ret.items(): data[key.replace("_", ":")] = float(val["val"]) except: pass data = refine_data(data) print(site, data) race_write(f"{site}_forex.txt", json_dumps(data)) except: print(f"{site} failed to load")
def fxempire2(site): """ live forex rates scraped from fxempire (tradingview backdoor) ?symbol=USD-RUB&resolution=1&from=158016000&to=1580162240" """ uri = "https://tradingview.fxempire.com/api/history" symbols = [ "USD-CNY", "USD-RUB", "USD-EUR", "USD-GBP", "USD-KRW", "USD-JPY" ] try: data = {} for symbol in symbols: now = int(time.time()) then = int((now - 200) / 10) # note weird /10 here params = f"?symbol={symbol}&resolution=1&from={then}&to={now}" url = uri + params raw = requests.get(url).json()["c"][-1] data[symbol.replace("-", ":")] = float(raw) data = refine_data(data) print(site, data) race_write(f"{site}_forex.txt", json_dumps(data)) except: print(f"{site} failed to load")
def currencyme(site): """ live forex rates scraped from currency.me.uk """ symbols = ["EUR", "RUB", "GBP", "KRW", "CNY", "JPY"] url = "https://www.currency.me.uk/remote/ER-CCCS2-AJAX.php" try: data = {} for symbol in symbols: try: params = { "ConvertTo": symbol, "ConvertFrom": "USD", "amount": 1 } raw = requests.get(url, params=params, timeout=(15, 15)).text data["USD:" + symbol] = float(raw.replace(" ", "")) except: pass data = refine_data(data) print(site, data) race_write(f"{site}_forex.txt", json_dumps(data)) except: print(f"{site} failed to load")
def finviz(site): """ live forex rates scraped from finviz.com """ url = "https://finviz.com/api/forex_all.ashx?timeframe=m5" try: ret = requests.get(url, timeout=(15, 15)).json() data = {} data["AUD:USD"] = float(ret["AUDUSD"]["last"]) data["EUR:GBP"] = float(ret["EURGBP"]["last"]) data["EUR:USD"] = float(ret["EURUSD"]["last"]) data["GBP:JPY"] = float(ret["GBPJPY"]["last"]) data["GBP:USD"] = float(ret["GBPUSD"]["last"]) data["USD:CAD"] = float(ret["USDCAD"]["last"]) data["NZD:USD"] = float(ret["NZDUSD"]["last"]) data["USD:CHF"] = float(ret["USDCHF"]["last"]) data["USD:JPY"] = float(ret["USDJPY"]["last"]) data["XAG:USD"] = float(ret["SI"]["last"]) data["XAU:USD"] = float(ret["GC"]["last"]) data = refine_data(data) print(site, data) race_write(f"{site}_forex.txt", json_dumps(data)) except: print(f"{site} failed to load")
def yahoo(site): """ live forex rates scraped from yahoo finance XAU and XAG are stale CG=F is a gold 30 day future SI=F is silver 60 day future """ uri = "https://query1.finance.yahoo.com/v7/finance/spark?symbols=USD" try: currencies = ["EUR", "CNY", "RUB", "KRW", "JPY"] data = {} for currency in currencies: endpoint = f"{currency}%3DX&range=1m&interval=1m" url = uri + endpoint raw = requests.get(url, timeout=(15, 15)).json() ret = raw["spark"]["result"][0]["response"][0]["meta"][ "regularMarketPrice"] data["USD:" + currency] = float(ret) time.sleep(1) data = refine_data(data) print(site, data) race_write(f"{site}_forex.txt", json_dumps(data)) except: print(f"{site} failed to load")
def fxempire1(site): """ live forex rates scraped from fxempire.com (backdoor to xignite) """ url = "https://www.fxempire.com/api/v1/en/markets/list" headers = { "authority": "www.fxempire.com", "method": "GET", "path": "/api/v1/en/markets/list", "scheme": "https", "accept": ("text/html,application/xhtml+xml,application/xml;q=0.9,image/webp," + "image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9"), "accept-encoding": "gzip, deflate, br", "accept-language": "en-US,en;q=0.9", "cache-control": "max-age=0", "dnt": "1", "sec-fetch-mode": "navigate", "sec-fetch-site": "none", "sec-fetch-user": "******", "upgrade-insecure-requests": "1", "user-agent": ("Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36" + " (KHTML, like Gecko) Chrome/79.0.3945.79 Safari/537.36 OPR/66.0.3515.27" ), } try: session = requests.Session() session.headers = headers cfscrape_requests = cfscrape.create_scraper(sess=session) ret = cfscrape_requests.get(url, timeout=(15, 15)).json() data = {} for item in ret["forex"]: if item: try: pair = item["name"].replace("/", ":") price = item["value"] data[pair] = float(price) except: pass for item in ret["commodities"]: try: if item["symbol"] in ["XAUUSD", "XAGUSD"]: pair = "USD:" + item["symbol"].replace("USD", "") price = 1 / float(item["value"]) data[pair] = price except: pass data = {k: v for k, v in data.items() if "RUB" not in k} # RUBLE is stale data = refine_data(data) print(site, data) race_write(f"{site}_forex.txt", json_dumps(data)) except: print(f"{site} failed to load")
def fxcm(site): """ live forex rates scraped from fxcm.com """ timestamp = int(time.time() * 1000) - 1000 url = f"https://ratesjson.fxcm.com/DataDisplayer?t={timestamp}" headers = { "authority": "www.fxcm.com", "method": "GET", "path": "/api/v1/en/markets/list", "scheme": "https", "accept": ("text/html,application/xhtml+xml,application/xml;q=0.9,image/webp," + "image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9"), "accept-encoding": "gzip, deflate, br", "accept-language": "en-US,en;q=0.9", "cache-control": "max-age=0", "dnt": "1", "sec-fetch-mode": "navigate", "sec-fetch-site": "none", "sec-fetch-user": "******", "upgrade-insecure-requests": "1", "user-agent": ("Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, " + "like Gecko) Chrome/79.0.3945.79 Safari/537.36 OPR/66.0.3515.27"), } try: # fails during some hours of day session = requests.Session() session.headers = headers cfscrape_requests = cfscrape.create_scraper(sess=session) ret = cfscrape_requests.get(url, timeout=(15, 15)).text # print (ret) data = (ret.replace(" ", "").replace('null({"Rates":', "").replace(",}]});", "}]").replace(",}", "}")) # print(data) # {"Symbol":"CHFJPY","Bid":"1.1","Ask":"1.2","Spread":"0.1","ProductType":"1",} raw = json_loads(data) data = {} for item in raw: symbol = item["Symbol"] if symbol.isupper() and (len(symbol) == 6): symbol = symbol[:3] + ":" + symbol[-3:] data[symbol] = (float(item["Ask"]) + float(item["Bid"])) / 2 data = refine_data(data) print(site, data) race_write(f"{site}_forex.txt", json_dumps(data)) except: print(f"{site} failed to load")
def bloomberg(site): """ live forex rates scraped from bloomberg.com """ uri = "https://www.bloomberg.com/markets/api/bulk-time-series/price/" endpoint = "USDCNY%3ACUR,USDRUB%3ACUR,USDJPY%3ACUR,USDEUR%3ACUR,USDKRW%3ACUR,XAUUSD%3ACUR,XAGUSD%3ACUR" url = uri + endpoint headers = { "authority": "www.bloomberg.com", "method": "GET", "path": ("/markets/api/comparison/data?securities=" + "USDCNY%3ACUR,USDRUB%3ACUR,USDJPY%3ACUR,USDEUR%3ACUR,USDKRW%3ACUR,XAUUSD%3ACUR,XAGUSD%3ACUR" + "&securityType=CURRENCY&locale=en"), "scheme": "https", "accept": ("text/html,application/xhtml+xml,application/xml;q=0.9,image/" + "webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9"), "accept-encoding": "gzip, deflate, br", "accept-language": "en-US,en;q=0.9", "cache-control": "max-age=0", "cookie": ("bbAbVisits=1; _pxhd=e24b47c64d37711c147cfb3c4b35c845563d2f9831b" + "03d9189f8cd761bc2be4f:d78eeb01-34c9-11ea-8f86-51d2aad9afb3; _px" + "vid=d78eeb01-34c9-11ea-8f86-51d2aad9afb3; _reg-csrf=s%3Ab0pWvbcs" + "UtrjYeJ0T2GrTaaD.8kaQlvHchJ1D%2FZZMaQWQiTizJTxrqqyzzuEZHEvlQNw;" + " agent_id=7989385a-d6d9-4446-b7aa-3c937407862b;" + " session_id=5702901e-d5fe-41e7-b259-df46322015e0;" + " session_key=3179869387f4c4ec4385e0d16222f0e59f48c47f;" + " _user-status=anonymous; _is-ip-whitelisted=false;" + " _user-ip=91.132.137.116; trc_cookie_storage=taboola%2520global%253" + "Auser-id%3D2f4acdc6-7c3c-412c-8766-d9c80dcffc38-tuct513df3e;" + " bdfpc=004.0586371899.1578785723722;" + " _reg-csrf-token=4ZxUa9q8-fkNXQkoHHXhnobWne1sDlIVcKEQ"), "dnt": "1", "if-none-match": 'W/"lZU52eQYxjadyNKGCyftEg=="', "sec-fetch-mode": "navigate", "sec-fetch-site": "none", "sec-fetch-user": "******", "upgrade-insecure-requests": "1", "user-agent": ("Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36" + " (KHTML, like Gecko) Chrome/79.0.3945.79 Safari/537.36 OPR/66.0.3515.27" ), } try: session = requests.Session() session.headers = headers cfscrape_requests = cfscrape.create_scraper(sess=session) ret = cfscrape_requests.get(url, timeout=(15, 15)).json() data = {} for item in ret: symbol = item["id"].replace(":CUR", "") symbol = symbol[:3] + ":" + symbol[-3:] data[symbol] = float(item["lastPrice"]) data["USD:XAG"] = 1 / data.pop("XAG:USD") data["USD:XAU"] = 1 / data.pop("XAU:USD") data = refine_data(data) print(site, data) race_write(f"{site}_forex.txt", json_dumps(data)) except: print(f"{site} failed to load")