def test_coinapi(self): source = CoinAPI() root = os.path.join(self._root(), "coinapi") for symbol in os.listdir(root): if len(os.listdir(os.path.join(root, symbol))) == 0: continue df = source.read(self._start, self._end, symbol, DAILY) self.assertEqual(df.index.nunique(), len(df.index)) self.assertFalse(df.isna().any(axis=1).any()) opens = df.loc[:, "open"] highs = df.loc[:, "high"] lows = df.loc[:, "low"] closes = df.loc[:, "close"] rows = df.loc[(opens <= 0) | (highs <= 0) | (lows <= 0) | (closes <= 0)] if len(rows) > 0: pretty.color_print( colors.PAPER_RED_400, f"{symbol.upper()} contains 0 in open, high, low, or close", ) self.assertEqual( len(rows), 0, )
def _urls(self) -> Iterable[str]: months: CONTRACT_MONTHS month_table = { "cl": ALL_CONTRACT_MONTHS, "gc": EVEN_CONTRACT_MONTHS, } for symbol in self._symbols: months = month_table.get(symbol, FINANCIAL_CONTRACT_MONTHS) # if symbol == "cl": # months = ALL_CONTRACT_MONTHS # elif symbol == "gc": # months = EVEN_CONTRACT_MONTHS # else: # months = FINANCIAL_CONTRACT_MONTHS for y in range(self._start, self._end + 1): for m in months: code = f"{symbol}{m}{y % 100:02}" pretty.color_print(colors.PAPER_CYAN_300, f"downloading: {code}") if self._page == HISTORICAL_PAGE: yield f"https://www.barchart.com/futures/quotes/{code}/historical-download" elif self._page == INTERACTIVE_PAGE: yield f"https://www.barchart.com/futures/quotes/{code}/interactive-chart" else: raise ValueError("unknown barchart page") input()
def _write(self, database: str, collection: str) -> None: path = self._path(database, collection) pretty.color_print(colors.PAPER_YELLOW_400, f"writing database to {path}") with open(path, "w") as f: json.dump(self._database, f, indent=2)
def _read_data( self, start: datetime, end: datetime, symbol: str, frequency: FREQUENCY ) -> pd.DataFrame: df = pd.read_csv( self._localfile(self._url(start, end, symbol, DAILY)), header=1 ) df = df.drop("unix", axis=1) df = df.drop("symbol", axis=1) if "vwap" in df.columns: df = df.drop("vwap", axis=1) if "tradecount" in df.columns: df.loc[:, "tradecount"].fillna(0, inplace=True) opens = df.loc[:, "open"] highs = df.loc[:, "high"] lows = df.loc[:, "low"] closes = df.loc[:, "close"] rows = df.loc[(opens <= 0) | (highs <= 0) | (lows <= 0) | (closes <= 0)].index if len(rows) > 0: pretty.color_print( colors.PAPER_AMBER_300, f"CryptoData {symbol.upper()} contains 0 in open, high, low, or close. Dropping {len(rows)} rows", ) df = df.drop(rows) return df
def _testing(self, src_exts, tar_ext, root): for f in os.listdir(root): path = os.path.join(root, f) if os.path.isdir(path): if os.path.basename(path).startswith(".") or os.path.basename( path).startswith("_"): for fs in os.listdir(path): pretty.color_print( colors.PAPER_AMBER_300, f"testing: {os.path.join(path, fs)}") self.assertTrue(tar_ext not in fs) else: self._testing(src_exts, tar_ext, path) else: ext = os.path.splitext(f)[1] if ext in src_exts: path = os.path.join(root, f.replace(ext, tar_ext)) pretty.color_print(colors.PAPER_AMBER_300, f"testing: {path}") if os.path.basename(path).startswith( ".") or os.path.basename(path).startswith("_"): self.assertFalse(os.path.exists(path)) else: self.assertTrue(os.path.exists(path))
def check(self) -> None: for symbol in self._symbols_table.keys(): tar = os.path.join(self._tar, "barchart", symbol, f"{symbol}.csv") if not os.path.exists(tar): pretty.color_print(colors.PAPER_PINK_300, f"missing files: {tar}")
def _get_charts(self): for i, symbol in enumerate(self._symbols): path = os.path.join( config.ROOT, symbol, self._frequency, f"{self._year}_{symbol}_{self._frequency}.png", ) if not os.path.exists(path): pretty.color_print( colors.PAPER_RED_400, f"invalid symbol {symbol.upper()} with frequency {self._frequency.upper()}", ) continue chart = cv2.imread(path, cv2.IMREAD_COLOR) chart = cv2.resize(chart, config.SIZE) self._charts.append(Chart(symbol, self._year, self._frequency, path, chart)) if len(self._charts) == 0: raise ValueError("empty charts") self._studying = self._charts[0]
def args_parse() -> Dict[str, Any]: parser = argparse.ArgumentParser() parser.add_argument( "--operations", metavar="", nargs="*", default=["download", "rename"], choices=["download", "rename", "check"], help="operations", ) parser.add_argument( "--years", metavar="", nargs="?", type=str, help="years", ) args = vars(parser.parse_args()) assert args.get("operations") is not None pretty.color_print( colors.PAPER_ORANGE_300, f"operations input: {', '.join(cast(List[str], args.get('operations')))}", ) pretty.color_print(colors.PAPER_ORANGE_300, f"years input: {args.get('years')}") return args
def _cache_refresh(self, src: str) -> None: assert os.path.exists(src) assert src in self._cache_store() h = self._cache_hash(src) if self._cache_store()[src] != h: pretty.color_print(colors.PAPER_TEAL_300, f"cache: {src}") self._cache_store()[src] = h
def __init__( self, start_year: Optional[int] = None, end_year: Optional[int] = None, # start: Optional[int] = None, # end: Optional[int] = None, page: BARCHART_PAGE = HISTORICAL_PAGE, ) -> None: super().__init__() assert page in (HISTORICAL_PAGE, INTERACTIVE_PAGE) self._page = page if start_year is None or end_year is None: # if start is None or end is None: self._start = datetime.now().year if datetime.now().month > 10: self._end = datetime.now().year + 1 else: self._end = datetime.now().year else: self._start = start_year self._end = end_year # self._start = start # self._end = end pretty.color_print( colors.PAPER_BROWN_300, f"Barchart Processor\nstart year: {self._start}, end year: {self._end}", ) self._symbols = [ "es", "nq", "qr", "ym", # "nl", "np", # "no", "fx", "zn", "ge", "tj", "gg", "dx", "j6", "e6", "b6", "a6", "n6", "d6", "s6", "gc", "cl", ]
def _check_root(self, root): if not os.path.exists(root): os.makedirs(root) else: for f in os.listdir(root): pretty.color_print(colors.PAPER_RED_500, f"removing file {os.path.join(root, f)}") os.remove(os.path.join(root, f)) self.assertTrue(os.path.exists(root))
def forward(self) -> bool: if self._eindex == len(self._quotes) - 1: pretty.color_print(colors.PAPER_AMBER_300, "cache is at the last quote") return False self._sindex += 1 self._eindex += 1 return True
def trade_statistic(): try: return TradeHandler().response_statistic() except (ValueError, IndexError, NotImplementedError, AssertionError) as err: info = sys.exc_info() traceback.print_tb(info[2]) pretty.color_print( colors.PAPER_RED_500, f"{type(info[1]).__name__}: {str(info[1])}" ) return {"error": f"{type(err)}: {err}"}
def backward(self) -> bool: if self._sindex == 0: pretty.color_print(colors.PAPER_AMBER_300, "cache is at the first quote") return False self._sindex -= 1 self._eindex -= 1 return True
def __init__( self, symbol: str, quotes: pd.DataFrame, frequency: FREQUENCY, candlesticks_body_width: float = 0.6, color_entry: str = colors.PAPER_BROWN_700, color_close: str = colors.PAPER_GREEN_700, color_warning: str = colors.PAPER_LIME_900, alpha: float = 0.3, ) -> None: assert quotes is not None root = os.path.join( os.getenv("HOME"), "Documents", "TRADING_NOTES", "study_zone" ) self._symbol = symbol self._quotes = quotes self._frequency = frequency self._candlesticks_body_width = candlesticks_body_width self._color_entry = color_entry self._color_close = color_close self._color_warning = color_warning self._alpha = alpha self._studies = None for r in os.listdir(root): pattern = r"[&_,|]" targets = re.split(pattern, r) targets = list(map(lambda x: x.strip(), targets)) if self._symbol.lower() in targets: path = os.path.join(root, r) assert os.path.exists(path) for f in os.listdir(path): with open(os.path.join(path, f), "r") as src: try: if self._studies is None: self._studies = json.load(src) else: self._studies.extend(json.load(src)) except json.JSONDecodeError: pretty.color_print( colors.PAPER_RED_400, f"invalid json file for trading study zone: {path}", ) self._studies = None break
def _urls(self) -> Iterable[str]: months: CONTRACT_MONTHS for symbol in self._symbols_table.keys(): pretty.color_print(colors.PAPER_CYAN_300, f"downloading: {symbol}") if self._page == HISTORICAL_PAGE: yield f"https://www.barchart.com/futures/quotes/{symbol}/historical-download" elif self._page == INTERACTIVE_PAGE: yield f"https://www.barchart.com/futures/quotes/{symbol}/interactive-chart" else: raise ValueError("unknown barchart page")
def _urls(self) -> Iterable[str]: for symbol, time in self._symbols.items(): dtime = datetime.strptime(time, "%Y%m%d").replace(tzinfo=timezone.utc) pretty.color_print(colors.PAPER_CYAN_300, f"downloading: {symbol}") yield ( f"https://finance.yahoo.com/quote/{requests.utils.quote(symbol)}/history?" + f"period1={int(dtime.timestamp())}&" + f"period2={int((datetime.utcnow() + timedelta(days=2)).timestamp())}&" + f"interval=1d&filter=history&frequency=1d")
def _clean_root(self, root): for f in os.listdir(root): pretty.color_print(colors.PAPER_RED_500, f"removing file {os.path.join(root, f)}") os.remove(os.path.join(root, f)) self.assertFalse(os.path.exists(os.path.join(root, f))) pretty.color_print(colors.PAPER_RED_500, f"removing dir {root}") os.rmdir(root) self.assertFalse(os.path.exists(root))
def __init__( self, page: BARCHART_PAGE = HISTORICAL_PAGE, ) -> None: super().__init__() assert page in (HISTORICAL_PAGE, INTERACTIVE_PAGE) self._page = page pretty.color_print( colors.PAPER_BROWN_300, "Barchart Stocks Processor", ) self._symbols_table = { # "$iqx": "spxew", # "$slew": "smlew", # "$sdew": "midew", # "$topx": "topix", # "$addn": "addn", # "$addq": "addq", # "$avdn": "avdn", # "$avdq": "avdq", # "$addt": "addt", # "$avdt": "avdt", "^btcusd": "btcusd", "^ethusd": "ethusd", "^ltcusd": "ltcusd", "^xrpusd": "xrpusd", "$dxy": "dxy", "^eurusd": "eurusd", "^usdjpy": "usdjpy", # "^jpyusd": "jpyusd", "^gbpusd": "gbpusd", "^audusd": "audusd", "^usdcad": "usdcad", # "^cadusd": "cadusd", "^usdchf": "usdchf", "^nzdusd": "nzdusd", # "^chfusd": "chfusd", "^eurjpy": "eurjpy", "^eurgbp": "eurgbp", "^euraud": "euraud", "^eurcad": "eurcad", "^eurchf": "eurchf", "^gbpjpy": "gbpjpy", "^audjpy": "audjpy", "^cadjpy": "cadjpy", # "^nzdjpy": "nzdjpy", }
def cp(src: str, tar: str) -> None: global SAFEGUARD assert src != "" and tar != "" assert SAFEGUARD in tar pretty.color_print(colors.PAPER_LIGHT_BLUE_300, f"cp: {src} -> {tar}") cmd = subprocess.run(["cp", "-rp", src, tar], capture_output=True, encoding="utf-8") if cmd.returncode != 0: raise subprocess.SubprocessError(cmd.stderr)
def rm(tar: str) -> None: global SAFEGUARD assert tar != "" assert SAFEGUARD in tar pretty.color_print(colors.PAPER_RED_500, f"rm: {tar}") cmd = subprocess.run(["rm", "-rf", tar], capture_output=True, encoding="utf-8") if cmd.returncode != 0: raise subprocess.SubprocessError(cmd.stderr)
def _rolling_date(self, front: Contract, back: Contract) -> datetime: fdf = front.dataframe() bdf = back.dataframe() volume = ((bdf.loc[bdf.index.isin(fdf.index), "volume"] >= fdf.loc[fdf.index.isin(bdf.index), "volume"]) & (bdf.loc[bdf.index.isin(fdf.index), "volume"] != 0) & (fdf.loc[fdf.index.isin(bdf.index), "volume"] != 0)) interest = ((bdf.loc[bdf.index.isin(fdf.index), "open interest"] >= fdf.loc[fdf.index.isin(bdf.index), "open interest"]) & (bdf.loc[bdf.index.isin(fdf.index), "open interest"] != 0) & (fdf.loc[fdf.index.isin(bdf.index), "open interest"] != 0)) union = None if volume.any() and interest.any(): union = volume & interest elif volume.any() and not interest.any(): union = volume elif not volume.any() and interest.any(): union = interest else: pretty.color_print( colors.PAPER_AMBER_300, f"empty volume and open interest in contracts {front.code().upper()} and {back.code().upper()}" ", use backup rolling method instead", ) return cast(datetime, self._backup.rolling_date(front, back)) assert union is not None selector = (fdf.index.isin(bdf.index)) & ( (fdf.index[-1] - fdf.index).days < 90) cross = fdf.loc[selector].loc[union] if len(cross) == 0: pretty.color_print( colors.PAPER_AMBER_300, f"no valid intersection in contracts {front.code().upper()} and {back.code().upper()}" ", use backup rolling method instead", ) return cast(datetime, self._backup.rolling_date(front, back)) return cast( datetime, cross.index[0].to_pydatetime(), )
def args_parse() -> Dict[str, Any]: parser = argparse.ArgumentParser() parser.add_argument("--input", metavar="", type=str, help="the input directory") parser.add_argument( "--operation", metavar="", nargs="*", default=["scss", "dart"], choices=["scss", "dart", "ts"], help="processor", ) parser.add_argument( "--optimized", metavar="", nargs="?", const=True, default=False, type=bool, help="optimize the output file", ) parser.add_argument( "--interval", metavar="", nargs="?", const=3, default=3, type=int, help="sleep interval (seconds)", ) args = vars(parser.parse_args()) assert args.get("input") is not None assert args.get("operation") is not None assert args.get("optimized") is not None assert args.get("interval") is not None pretty.color_print(colors.PAPER_INDIGO_300, f"input: {args.get('input')}") pretty.color_print( colors.PAPER_INDIGO_300, f"operation: {', '.join(cast(List[str], args.get('operation')))}", ) pretty.color_print(colors.PAPER_INDIGO_300, f"optimized: {args.get('optimized')}") pretty.color_print(colors.PAPER_INDIGO_300, f"interval: {args.get('interval')} seconds") return args
def _urls(self) -> Iterable[str]: for url in [ r"https://www.investing.com/indices/stoxx-50-volatility-vstoxx-eur-historical-data", r"https://www.investing.com/indices/jpx-nikkei-400-historical-data", r"https://www.investing.com/indices/nikkei-volatility-historical-data", r"https://www.investing.com/indices/hsi-volatility-historical-data", # r"https://www.investing.com/indices/cboe-china-etf-volatility-historical-data", ]: pretty.color_print( colors.PAPER_CYAN_300, f"downloading: {url.split('/')[-1].split('-')[0]}", ) yield url
def wrapper_testing(*args, **kargs): count = 0 print() for book in sets: pretty.color_print( colors.PAPER_LIGHT_GREEN_300, f"running parameterized test of index {count}", ) func(*args, **book) count += 1 pretty.color_print( colors.PAPER_LIGHT_BLUE_300, f"\nfinished running {count} parameterized tests", )
def _compile(self, src: str, dst: str) -> None: pretty.color_print(colors.PAPER_LIGHT_BLUE_300, f"process: {src} -> {dst}") cmd: subprocess.CompletedProcess if self._optimized: cmd = subprocess.run(self._optimized_command(src, dst), capture_output=True, encoding="utf-8") else: cmd = subprocess.run(self._command(src, dst), capture_output=True, encoding="utf-8") self._cache_refresh(src) if cmd.returncode != 0: pretty.color_print( colors.PAPER_RED_400, f"\nmessage: {cmd.stdout}\nerror: {cmd.stderr}")
def check(self) -> None: months: CONTRACT_MONTHS for symbol in self._symbols: if symbol == "cl": months = ALL_CONTRACT_MONTHS elif symbol == "gc": months = EVEN_CONTRACT_MONTHS else: months = FINANCIAL_CONTRACT_MONTHS for y in range(self._start, self._end + 1): for m in months: code = f"{symbol}{m}{y % 100:02}" tar = os.path.join(self._tar, "continuous", code[:2], f"{code}.csv") if not os.path.exists(tar): pretty.color_print(colors.PAPER_PINK_300, f"missing files: {tar}")
def contract_list( start: datetime, end: datetime, symbol: str, months: CONTRACT_MONTHS, fmt: CODE_FORMAT, read_data: bool = True, src: DataSource = BarchartContract(), frequency: FREQUENCY = DAILY, ) -> List[Contract]: try: cur = Contract.front_month( symbol=symbol, months=months, fmt=fmt, time=end, read_data=read_data, src=src, frequency=frequency, ) except FileNotFoundError: msg = "empty contract list" pretty.color_print(colors.PAPER_AMBER_300, msg) raise ValueError(msg) contracts = [cur] while not ( (cur.year() * 10000 + cur.month() * 100) < (start.year * 10000 + start.month * 100) ): try: cur = cur.previous_contract(read_data=read_data) contracts.append(cur) except FileNotFoundError as err: pretty.color_print(colors.PAPER_AMBER_300, str(err)) break assert len(contracts) != 0 return contracts
def rename(self) -> None: for fs in os.listdir(self._src): match = re.match( # r"^(\$[\w]+)_([^_-]+)(?:-[^_-]+)*_[^_-]+-[^_-]+-\d{2}-\d{2}-\d{4}.csv$", r"^([\$\^][\w]+)_([^_-]+)(?:-[^_-]+)*_[^_-]+-[^_-]+-\d{2}-\d{2}-\d{4}.csv$", fs, ) if match is None: continue else: symbol = match.group(1).lower() symbol = self._symbols_table.get(symbol, None) if symbol is None: continue src = os.path.join(self._src, fs) tar = os.path.join(self._tar, "barchart", f"{symbol}.csv") assert os.path.exists(os.path.dirname(tar)) pretty.color_print(colors.PAPER_DEEP_PURPLE_300, f"move file: {src} => {tar}") os.rename(src, tar) self._rename_count += 1 pretty.color_print(colors.PAPER_LIGHT_GREEN_A200, f"rename {self._rename_count} files") if self._download_count != self._rename_count: pretty.color_print(colors.PAPER_RED_400, "rename operation miss some downloaded files")
def rename(self) -> None: table = { "STOXX 50 Volatility VSTOXX EUR Historical Data.csv": "vstx", "JPX-Nikkei 400 Historical Data.csv": "nk400", "Nikkei Volatility Historical Data.csv": "jniv", "HSI Volatility Historical Data.csv": "vhsi", # "CBOE China Etf Volatility Historical Data.csv": "vxfxi", } for fs in os.listdir(self._src): symbol = table.get(fs, None) if symbol is None: continue src = os.path.join(self._src, fs) tar = os.path.join(self._tar, "investing.com", f"{symbol}.csv") assert os.path.exists(os.path.dirname(tar)) pretty.color_print(colors.PAPER_DEEP_PURPLE_200, f"move file: {src} => {tar}") os.rename(src, tar) self._rename_count += 1 pretty.color_print(colors.PAPER_LIGHT_GREEN_A200, f"rename {self._rename_count} files") if self._download_count != self._rename_count: pretty.color_print(colors.PAPER_RED_400, "rename operation miss some downloaded files")