def test_coinapi(self):
source = CoinAPI()
root = os.path.join(self._root(), "coinapi")
for symbol in os.listdir(root):
if len(os.listdir(os.path.join(root, symbol))) == 0:
continue
df = source.read(self._start, self._end, symbol, DAILY)
self.assertEqual(df.index.nunique(), len(df.index))
self.assertFalse(df.isna().any(axis=1).any())
opens = df.loc[:, "open"]
highs = df.loc[:, "high"]
lows = df.loc[:, "low"]
closes = df.loc[:, "close"]
rows = df.loc[(opens <= 0) | (highs <= 0) | (lows <= 0) |
(closes <= 0)]
if len(rows) > 0:
pretty.color_print(
colors.PAPER_RED_400,
f"{symbol.upper()} contains 0 in open, high, low, or close",
)
self.assertEqual(
len(rows),
0,
)
def _urls(self) -> Iterable[str]:
months: CONTRACT_MONTHS
month_table = {
"cl": ALL_CONTRACT_MONTHS,
"gc": EVEN_CONTRACT_MONTHS,
}
for symbol in self._symbols:
months = month_table.get(symbol, FINANCIAL_CONTRACT_MONTHS)
# if symbol == "cl":
# months = ALL_CONTRACT_MONTHS
# elif symbol == "gc":
# months = EVEN_CONTRACT_MONTHS
# else:
# months = FINANCIAL_CONTRACT_MONTHS
for y in range(self._start, self._end + 1):
for m in months:
code = f"{symbol}{m}{y % 100:02}"
pretty.color_print(colors.PAPER_CYAN_300,
f"downloading: {code}")
if self._page == HISTORICAL_PAGE:
yield f"https://www.barchart.com/futures/quotes/{code}/historical-download"
elif self._page == INTERACTIVE_PAGE:
yield f"https://www.barchart.com/futures/quotes/{code}/interactive-chart"
else:
raise ValueError("unknown barchart page")
input()
def _write(self, database: str, collection: str) -> None:
path = self._path(database, collection)
pretty.color_print(colors.PAPER_YELLOW_400, f"writing database to {path}")
with open(path, "w") as f:
json.dump(self._database, f, indent=2)
def _read_data(
self, start: datetime, end: datetime, symbol: str, frequency: FREQUENCY
) -> pd.DataFrame:
df = pd.read_csv(
self._localfile(self._url(start, end, symbol, DAILY)), header=1
)
df = df.drop("unix", axis=1)
df = df.drop("symbol", axis=1)
if "vwap" in df.columns:
df = df.drop("vwap", axis=1)
if "tradecount" in df.columns:
df.loc[:, "tradecount"].fillna(0, inplace=True)
opens = df.loc[:, "open"]
highs = df.loc[:, "high"]
lows = df.loc[:, "low"]
closes = df.loc[:, "close"]
rows = df.loc[(opens <= 0) | (highs <= 0) | (lows <= 0) | (closes <= 0)].index
if len(rows) > 0:
pretty.color_print(
colors.PAPER_AMBER_300,
f"CryptoData {symbol.upper()} contains 0 in open, high, low, or close. Dropping {len(rows)} rows",
)
df = df.drop(rows)
return df
def _testing(self, src_exts, tar_ext, root):
for f in os.listdir(root):
path = os.path.join(root, f)
if os.path.isdir(path):
if os.path.basename(path).startswith(".") or os.path.basename(
path).startswith("_"):
for fs in os.listdir(path):
pretty.color_print(
colors.PAPER_AMBER_300,
f"testing: {os.path.join(path, fs)}")
self.assertTrue(tar_ext not in fs)
else:
self._testing(src_exts, tar_ext, path)
else:
ext = os.path.splitext(f)[1]
if ext in src_exts:
path = os.path.join(root, f.replace(ext, tar_ext))
pretty.color_print(colors.PAPER_AMBER_300,
f"testing: {path}")
if os.path.basename(path).startswith(
".") or os.path.basename(path).startswith("_"):
self.assertFalse(os.path.exists(path))
else:
self.assertTrue(os.path.exists(path))
def check(self) -> None:
for symbol in self._symbols_table.keys():
tar = os.path.join(self._tar, "barchart", symbol, f"{symbol}.csv")
if not os.path.exists(tar):
pretty.color_print(colors.PAPER_PINK_300,
f"missing files: {tar}")
def _get_charts(self):
for i, symbol in enumerate(self._symbols):
path = os.path.join(
config.ROOT,
symbol,
self._frequency,
f"{self._year}_{symbol}_{self._frequency}.png",
)
if not os.path.exists(path):
pretty.color_print(
colors.PAPER_RED_400,
f"invalid symbol {symbol.upper()} with frequency {self._frequency.upper()}",
)
continue
chart = cv2.imread(path, cv2.IMREAD_COLOR)
chart = cv2.resize(chart, config.SIZE)
self._charts.append(Chart(symbol, self._year, self._frequency, path, chart))
if len(self._charts) == 0:
raise ValueError("empty charts")
self._studying = self._charts[0]
def args_parse() -> Dict[str, Any]:
parser = argparse.ArgumentParser()
parser.add_argument(
"--operations",
metavar="",
nargs="*",
default=["download", "rename"],
choices=["download", "rename", "check"],
help="operations",
)
parser.add_argument(
"--years",
metavar="",
nargs="?",
type=str,
help="years",
)
args = vars(parser.parse_args())
assert args.get("operations") is not None
pretty.color_print(
colors.PAPER_ORANGE_300,
f"operations input: {', '.join(cast(List[str], args.get('operations')))}",
)
pretty.color_print(colors.PAPER_ORANGE_300,
f"years input: {args.get('years')}")
return args
def _cache_refresh(self, src: str) -> None:
assert os.path.exists(src)
assert src in self._cache_store()
h = self._cache_hash(src)
if self._cache_store()[src] != h:
pretty.color_print(colors.PAPER_TEAL_300, f"cache: {src}")
self._cache_store()[src] = h
def __init__(
self,
start_year: Optional[int] = None,
end_year: Optional[int] = None,
# start: Optional[int] = None,
# end: Optional[int] = None,
page: BARCHART_PAGE = HISTORICAL_PAGE,
) -> None:
super().__init__()
assert page in (HISTORICAL_PAGE, INTERACTIVE_PAGE)
self._page = page
if start_year is None or end_year is None:
# if start is None or end is None:
self._start = datetime.now().year
if datetime.now().month > 10:
self._end = datetime.now().year + 1
else:
self._end = datetime.now().year
else:
self._start = start_year
self._end = end_year
# self._start = start
# self._end = end
pretty.color_print(
colors.PAPER_BROWN_300,
f"Barchart Processor\nstart year: {self._start}, end year: {self._end}",
)
self._symbols = [
"es",
"nq",
"qr",
"ym",
# "nl",
"np",
# "no",
"fx",
"zn",
"ge",
"tj",
"gg",
"dx",
"j6",
"e6",
"b6",
"a6",
"n6",
"d6",
"s6",
"gc",
"cl",
]
def _check_root(self, root):
if not os.path.exists(root):
os.makedirs(root)
else:
for f in os.listdir(root):
pretty.color_print(colors.PAPER_RED_500,
f"removing file {os.path.join(root, f)}")
os.remove(os.path.join(root, f))
self.assertTrue(os.path.exists(root))
def forward(self) -> bool:
if self._eindex == len(self._quotes) - 1:
pretty.color_print(colors.PAPER_AMBER_300,
"cache is at the last quote")
return False
self._sindex += 1
self._eindex += 1
return True
def trade_statistic():
try:
return TradeHandler().response_statistic()
except (ValueError, IndexError, NotImplementedError, AssertionError) as err:
info = sys.exc_info()
traceback.print_tb(info[2])
pretty.color_print(
colors.PAPER_RED_500, f"{type(info[1]).__name__}: {str(info[1])}"
)
return {"error": f"{type(err)}: {err}"}
def backward(self) -> bool:
if self._sindex == 0:
pretty.color_print(colors.PAPER_AMBER_300,
"cache is at the first quote")
return False
self._sindex -= 1
self._eindex -= 1
return True
def __init__(
self,
symbol: str,
quotes: pd.DataFrame,
frequency: FREQUENCY,
candlesticks_body_width: float = 0.6,
color_entry: str = colors.PAPER_BROWN_700,
color_close: str = colors.PAPER_GREEN_700,
color_warning: str = colors.PAPER_LIME_900,
alpha: float = 0.3,
) -> None:
assert quotes is not None
root = os.path.join(
os.getenv("HOME"), "Documents", "TRADING_NOTES", "study_zone"
)
self._symbol = symbol
self._quotes = quotes
self._frequency = frequency
self._candlesticks_body_width = candlesticks_body_width
self._color_entry = color_entry
self._color_close = color_close
self._color_warning = color_warning
self._alpha = alpha
self._studies = None
for r in os.listdir(root):
pattern = r"[&_,|]"
targets = re.split(pattern, r)
targets = list(map(lambda x: x.strip(), targets))
if self._symbol.lower() in targets:
path = os.path.join(root, r)
assert os.path.exists(path)
for f in os.listdir(path):
with open(os.path.join(path, f), "r") as src:
try:
if self._studies is None:
self._studies = json.load(src)
else:
self._studies.extend(json.load(src))
except json.JSONDecodeError:
pretty.color_print(
colors.PAPER_RED_400,
f"invalid json file for trading study zone: {path}",
)
self._studies = None
break
def _urls(self) -> Iterable[str]:
months: CONTRACT_MONTHS
for symbol in self._symbols_table.keys():
pretty.color_print(colors.PAPER_CYAN_300, f"downloading: {symbol}")
if self._page == HISTORICAL_PAGE:
yield f"https://www.barchart.com/futures/quotes/{symbol}/historical-download"
elif self._page == INTERACTIVE_PAGE:
yield f"https://www.barchart.com/futures/quotes/{symbol}/interactive-chart"
else:
raise ValueError("unknown barchart page")
def _urls(self) -> Iterable[str]:
for symbol, time in self._symbols.items():
dtime = datetime.strptime(time,
"%Y%m%d").replace(tzinfo=timezone.utc)
pretty.color_print(colors.PAPER_CYAN_300, f"downloading: {symbol}")
yield (
f"https://finance.yahoo.com/quote/{requests.utils.quote(symbol)}/history?"
+ f"period1={int(dtime.timestamp())}&" +
f"period2={int((datetime.utcnow() + timedelta(days=2)).timestamp())}&"
+ f"interval=1d&filter=history&frequency=1d")
def _clean_root(self, root):
for f in os.listdir(root):
pretty.color_print(colors.PAPER_RED_500,
f"removing file {os.path.join(root, f)}")
os.remove(os.path.join(root, f))
self.assertFalse(os.path.exists(os.path.join(root, f)))
pretty.color_print(colors.PAPER_RED_500, f"removing dir {root}")
os.rmdir(root)
self.assertFalse(os.path.exists(root))
def __init__(
self,
page: BARCHART_PAGE = HISTORICAL_PAGE,
) -> None:
super().__init__()
assert page in (HISTORICAL_PAGE, INTERACTIVE_PAGE)
self._page = page
pretty.color_print(
colors.PAPER_BROWN_300,
"Barchart Stocks Processor",
)
self._symbols_table = {
# "$iqx": "spxew",
# "$slew": "smlew",
# "$sdew": "midew",
# "$topx": "topix",
# "$addn": "addn",
# "$addq": "addq",
# "$avdn": "avdn",
# "$avdq": "avdq",
# "$addt": "addt",
# "$avdt": "avdt",
"^btcusd": "btcusd",
"^ethusd": "ethusd",
"^ltcusd": "ltcusd",
"^xrpusd": "xrpusd",
"$dxy": "dxy",
"^eurusd": "eurusd",
"^usdjpy": "usdjpy",
# "^jpyusd": "jpyusd",
"^gbpusd": "gbpusd",
"^audusd": "audusd",
"^usdcad": "usdcad",
# "^cadusd": "cadusd",
"^usdchf": "usdchf",
"^nzdusd": "nzdusd",
# "^chfusd": "chfusd",
"^eurjpy": "eurjpy",
"^eurgbp": "eurgbp",
"^euraud": "euraud",
"^eurcad": "eurcad",
"^eurchf": "eurchf",
"^gbpjpy": "gbpjpy",
"^audjpy": "audjpy",
"^cadjpy": "cadjpy",
# "^nzdjpy": "nzdjpy",
}
def cp(src: str, tar: str) -> None:
global SAFEGUARD
assert src != "" and tar != ""
assert SAFEGUARD in tar
pretty.color_print(colors.PAPER_LIGHT_BLUE_300, f"cp: {src} -> {tar}")
cmd = subprocess.run(["cp", "-rp", src, tar],
capture_output=True,
encoding="utf-8")
if cmd.returncode != 0:
raise subprocess.SubprocessError(cmd.stderr)
def rm(tar: str) -> None:
global SAFEGUARD
assert tar != ""
assert SAFEGUARD in tar
pretty.color_print(colors.PAPER_RED_500, f"rm: {tar}")
cmd = subprocess.run(["rm", "-rf", tar],
capture_output=True,
encoding="utf-8")
if cmd.returncode != 0:
raise subprocess.SubprocessError(cmd.stderr)
def _rolling_date(self, front: Contract, back: Contract) -> datetime:
fdf = front.dataframe()
bdf = back.dataframe()
volume = ((bdf.loc[bdf.index.isin(fdf.index), "volume"] >=
fdf.loc[fdf.index.isin(bdf.index), "volume"])
& (bdf.loc[bdf.index.isin(fdf.index), "volume"] != 0)
& (fdf.loc[fdf.index.isin(bdf.index), "volume"] != 0))
interest = ((bdf.loc[bdf.index.isin(fdf.index), "open interest"] >=
fdf.loc[fdf.index.isin(bdf.index), "open interest"])
&
(bdf.loc[bdf.index.isin(fdf.index), "open interest"] != 0)
&
(fdf.loc[fdf.index.isin(bdf.index), "open interest"] != 0))
union = None
if volume.any() and interest.any():
union = volume & interest
elif volume.any() and not interest.any():
union = volume
elif not volume.any() and interest.any():
union = interest
else:
pretty.color_print(
colors.PAPER_AMBER_300,
f"empty volume and open interest in contracts {front.code().upper()} and {back.code().upper()}"
", use backup rolling method instead",
)
return cast(datetime, self._backup.rolling_date(front, back))
assert union is not None
selector = (fdf.index.isin(bdf.index)) & (
(fdf.index[-1] - fdf.index).days < 90)
cross = fdf.loc[selector].loc[union]
if len(cross) == 0:
pretty.color_print(
colors.PAPER_AMBER_300,
f"no valid intersection in contracts {front.code().upper()} and {back.code().upper()}"
", use backup rolling method instead",
)
return cast(datetime, self._backup.rolling_date(front, back))
return cast(
datetime,
cross.index[0].to_pydatetime(),
)
def args_parse() -> Dict[str, Any]:
parser = argparse.ArgumentParser()
parser.add_argument("--input",
metavar="",
type=str,
help="the input directory")
parser.add_argument(
"--operation",
metavar="",
nargs="*",
default=["scss", "dart"],
choices=["scss", "dart", "ts"],
help="processor",
)
parser.add_argument(
"--optimized",
metavar="",
nargs="?",
const=True,
default=False,
type=bool,
help="optimize the output file",
)
parser.add_argument(
"--interval",
metavar="",
nargs="?",
const=3,
default=3,
type=int,
help="sleep interval (seconds)",
)
args = vars(parser.parse_args())
assert args.get("input") is not None
assert args.get("operation") is not None
assert args.get("optimized") is not None
assert args.get("interval") is not None
pretty.color_print(colors.PAPER_INDIGO_300, f"input: {args.get('input')}")
pretty.color_print(
colors.PAPER_INDIGO_300,
f"operation: {', '.join(cast(List[str], args.get('operation')))}",
)
pretty.color_print(colors.PAPER_INDIGO_300,
f"optimized: {args.get('optimized')}")
pretty.color_print(colors.PAPER_INDIGO_300,
f"interval: {args.get('interval')} seconds")
return args
def _urls(self) -> Iterable[str]:
for url in [
r"https://www.investing.com/indices/stoxx-50-volatility-vstoxx-eur-historical-data",
r"https://www.investing.com/indices/jpx-nikkei-400-historical-data",
r"https://www.investing.com/indices/nikkei-volatility-historical-data",
r"https://www.investing.com/indices/hsi-volatility-historical-data",
# r"https://www.investing.com/indices/cboe-china-etf-volatility-historical-data",
]:
pretty.color_print(
colors.PAPER_CYAN_300,
f"downloading: {url.split('/')[-1].split('-')[0]}",
)
yield url
def wrapper_testing(*args, **kargs):
count = 0
print()
for book in sets:
pretty.color_print(
colors.PAPER_LIGHT_GREEN_300,
f"running parameterized test of index {count}",
)
func(*args, **book)
count += 1
pretty.color_print(
colors.PAPER_LIGHT_BLUE_300,
f"\nfinished running {count} parameterized tests",
)
def _compile(self, src: str, dst: str) -> None:
pretty.color_print(colors.PAPER_LIGHT_BLUE_300,
f"process: {src} -> {dst}")
cmd: subprocess.CompletedProcess
if self._optimized:
cmd = subprocess.run(self._optimized_command(src, dst),
capture_output=True,
encoding="utf-8")
else:
cmd = subprocess.run(self._command(src, dst),
capture_output=True,
encoding="utf-8")
self._cache_refresh(src)
if cmd.returncode != 0:
pretty.color_print(
colors.PAPER_RED_400,
f"\nmessage: {cmd.stdout}\nerror: {cmd.stderr}")
def check(self) -> None:
months: CONTRACT_MONTHS
for symbol in self._symbols:
if symbol == "cl":
months = ALL_CONTRACT_MONTHS
elif symbol == "gc":
months = EVEN_CONTRACT_MONTHS
else:
months = FINANCIAL_CONTRACT_MONTHS
for y in range(self._start, self._end + 1):
for m in months:
code = f"{symbol}{m}{y % 100:02}"
tar = os.path.join(self._tar, "continuous", code[:2],
f"{code}.csv")
if not os.path.exists(tar):
pretty.color_print(colors.PAPER_PINK_300,
f"missing files: {tar}")
def contract_list(
start: datetime,
end: datetime,
symbol: str,
months: CONTRACT_MONTHS,
fmt: CODE_FORMAT,
read_data: bool = True,
src: DataSource = BarchartContract(),
frequency: FREQUENCY = DAILY,
) -> List[Contract]:
try:
cur = Contract.front_month(
symbol=symbol,
months=months,
fmt=fmt,
time=end,
read_data=read_data,
src=src,
frequency=frequency,
)
except FileNotFoundError:
msg = "empty contract list"
pretty.color_print(colors.PAPER_AMBER_300, msg)
raise ValueError(msg)
contracts = [cur]
while not (
(cur.year() * 10000 + cur.month() * 100)
< (start.year * 10000 + start.month * 100)
):
try:
cur = cur.previous_contract(read_data=read_data)
contracts.append(cur)
except FileNotFoundError as err:
pretty.color_print(colors.PAPER_AMBER_300, str(err))
break
assert len(contracts) != 0
return contracts
def rename(self) -> None:
for fs in os.listdir(self._src):
match = re.match(
# r"^(\$[\w]+)_([^_-]+)(?:-[^_-]+)*_[^_-]+-[^_-]+-\d{2}-\d{2}-\d{4}.csv$",
r"^([\$\^][\w]+)_([^_-]+)(?:-[^_-]+)*_[^_-]+-[^_-]+-\d{2}-\d{2}-\d{4}.csv$",
fs,
)
if match is None:
continue
else:
symbol = match.group(1).lower()
symbol = self._symbols_table.get(symbol, None)
if symbol is None:
continue
src = os.path.join(self._src, fs)
tar = os.path.join(self._tar, "barchart", f"{symbol}.csv")
assert os.path.exists(os.path.dirname(tar))
pretty.color_print(colors.PAPER_DEEP_PURPLE_300,
f"move file: {src} => {tar}")
os.rename(src, tar)
self._rename_count += 1
pretty.color_print(colors.PAPER_LIGHT_GREEN_A200,
f"rename {self._rename_count} files")
if self._download_count != self._rename_count:
pretty.color_print(colors.PAPER_RED_400,
"rename operation miss some downloaded files")
def rename(self) -> None:
table = {
"STOXX 50 Volatility VSTOXX EUR Historical Data.csv": "vstx",
"JPX-Nikkei 400 Historical Data.csv": "nk400",
"Nikkei Volatility Historical Data.csv": "jniv",
"HSI Volatility Historical Data.csv": "vhsi",
# "CBOE China Etf Volatility Historical Data.csv": "vxfxi",
}
for fs in os.listdir(self._src):
symbol = table.get(fs, None)
if symbol is None:
continue
src = os.path.join(self._src, fs)
tar = os.path.join(self._tar, "investing.com", f"{symbol}.csv")
assert os.path.exists(os.path.dirname(tar))
pretty.color_print(colors.PAPER_DEEP_PURPLE_200,
f"move file: {src} => {tar}")
os.rename(src, tar)
self._rename_count += 1
pretty.color_print(colors.PAPER_LIGHT_GREEN_A200,
f"rename {self._rename_count} files")
if self._download_count != self._rename_count:
pretty.color_print(colors.PAPER_RED_400,
"rename operation miss some downloaded files")
