def read_daily_option_quote(
browser: ChromeDriver,
symbol: str,
option_type: str,
strike_price: float,
exp_date: int,
use_barchart: bool = False,
suppress_log: bool = False,
) -> DailyOptionQuote:
option_quote = DailyOptionQuote(symbol, option_type, strike_price,
exp_date, int(get_date_str()))
if use_barchart:
url = "https://www.barchart.com/stocks/quotes/"
url += f"{symbol}/options?expiration={str(exp_date)}&moneyness=allRows"
eclass = "bc-options-quotes"
web_data = browser.download_data(
url=url,
wait_base=1,
element_class=eclass,
suppress_log=suppress_log,
)
else:
datetime_diff = get_date(str(exp_date)) - get_datetime_start()
date_url = int(datetime_diff.total_seconds())
url = f"https://finance.yahoo.com/quote/{symbol}/options?date={date_url}"
eid = 'Col1-1-OptionContracts-Proxy'
web_data = browser.download_data(
url=url,
wait_base=1,
element_id=eid,
suppress_log=suppress_log,
)
parse_daily_option_quote(option_quote, web_data.splitlines(), symbol,
option_type, strike_price, exp_date, use_barchart)
return option_quote
def read_daily_option_info(
browser: ChromeDriver,
symbol: str,
use_barchart: bool = True,
suppress_all_log: bool = False,
suppress_sub_log: bool = False,
) -> DailyOptionInfo:
option_info = DailyOptionInfo(symbol, int(get_date_str()))
if use_barchart:
all_exp_dates = read_exp_dates(
browser,
symbol,
suppress_log=suppress_all_log,
)
eclass = 'bc-futures-options-quotes-totals'
pre_eclass = 'bc-datatable'
for exp_date in all_exp_dates:
url = "https://www.barchart.com/stocks/quotes/"
url += f"{symbol}/options?expiration={str(exp_date)}"
web_data = browser.download_data(
url=url,
wait_base=1,
pre_element_class=pre_eclass,
element_class=eclass,
suppress_log=suppress_sub_log,
)
parse_daily_option_info(option_info, web_data.splitlines())
else:
logger.error('other data sources not implemented')
raise ValueError(f"use_barchart={use_barchart}")
return option_info
def read_info(
symbol_queue: mp.Queue,
info_queue: mp.Queue,
):
pname = mp.current_process().name
logger.info(f"History builder worker-{pname} starting")
num_items = 0
with ChromeDriver() as browser:
while True:
try:
symbol = symbol_queue.get(block=True, timeout=600)
except Exception as e:
logger.warning(f"Unexpected expcetion: {str(e)}")
break
if symbol is None:
symbol_queue.put(None, block=True, timeout=60)
break
num_items += 1
try:
daily_option_info = read_daily_option_info(
browser,
symbol,
suppress_all_log=False,
suppress_sub_log=True,
)
except Exception as e:
logger.info(f"Encountered exception ({str(e)}) on {symbol}")
continue
else:
info_queue.put(daily_option_info)
info_queue.put(None)
logger.info(f"History builder worker-{pname} processed {num_items} items")
def evaluate_option_activity(
in_queue: mp.Queue,
out_queue: mp.Queue,
cache_queue: mp.Queue,
):
pname = mp.current_process().name
logger.info(f"OA evaluation worker-{pname} starting")
cache = {}
num_items = 0
with ChromeDriver() as browser:
while True:
try:
raw_data = in_queue.get(block=True, timeout=600)
except Exception as e:
logger.warning(f"Unexpected exception: {str(e)}")
break
if raw_data is None:
break
num_items += 1
option_activity: OptionActivity = raw_data[0]
avg_option_info: AvgOptionInfo = raw_data[1]
# filtering
if filter(option_activity, avg_option_info, browser, cache):
out_queue.put(option_activity)
out_queue.put(None)
cache_queue.put(cache)
logger.info(f"OA evaluation worker-{pname} processed {num_items} items")
def read_effect(
oa_queue: mp.Queue,
effect_queue: mp.Queue,
):
with ChromeDriver() as browser:
while True:
try:
option_activity = oa_queue.get(block=True, timeout=600)
except Exception as e:
logger.warning(f"Unexpected expcetion on get: {str(e)}")
break
if option_activity is None:
oa_queue.put(None, block=True, timeout=60)
break
option_quote = read_daily_option_quote(
browser,
option_activity.get("symbol"),
option_activity.get("option_type"),
option_activity.get("strike_price"),
option_activity.get("exp_date"),
use_barchart=True,
suppress_log=False,
)
stock_quote = read_stock_quote(
browser,
option_activity.get("symbol"),
suppress_log=False,
)
option_effect = OptionEffect(daily_option_quote=option_quote,
daily_stock_quote=stock_quote)
effect_queue.put(option_effect, block=True, timeout=600)
effect_queue.put(None, block=True, timeout=600)
def read_option_activity(browser: ChromeDriver,
save_file=False,
folder='logs') -> list:
retry_timeout = 4
url = 'https://www.barchart.com/options/unusual-activity/stocks?page=all'
eid = 'main-content-column'
#buttons = ['a.show-all']
buttons = None
num_retry = 0
while num_retry < retry_timeout:
num_retry += 1
try:
web_data = browser.download_data(url=url,
wait_base=num_retry,
button_css=buttons,
element_id=eid)
except Exception as e:
logger.error(f'error {str(e)} (retry={num_retry}/{retry_timeout})')
time.sleep(num_retry)
continue
else:
option_activity_list = parse_option_activity(web_data.splitlines())
if len(option_activity_list) > 0:
logger.info(
'retrieved option activity list: # items={:d}'.format(
len(option_activity_list)))
break
else:
logger.warning(
'option activity list empty? retry={}/{}'.format(
num_retry, retry_timeout))
time.sleep(num_retry)
continue
if num_retry >= retry_timeout:
return []
# save a copy
if save_file:
# try to remove duplicates if folder exists
if os.path.exists(folder):
today_str = get_date_str()
for item in os.listdir(folder):
if item.startswith('OA_%s' % (today_str)):
os.remove(os.path.join(folder, item))
else:
# otherwise create folder
os.makedirs(folder)
# write new one
filename = os.path.join(folder, f'OA_{get_datetime_str()}.txt.gz')
with openw(filename, 'wt') as fout:
fout.write('\n'.join(option_activity_list))
logger.info(f'{get_time_log()} save option activity to {filename}')
return option_activity_list
def read_stock_quote(
browser: ChromeDriver,
symbol: str,
suppress_log: bool = False,
) -> DailyStockQuote:
stock_quote = DailyStockQuote(symbol, int(get_date_str()))
url = f"https://finance.yahoo.com/quote/{symbol}"
eid = "quote-summary"
web_data = browser.download_data(
url=url,
element_id=eid,
suppress_log=suppress_log,
)
parse_stock_quote(stock_quote, web_data.splitlines())
return stock_quote
if daily_stock_quote:
for k in DailyStockQuote.fields.keys():
if k in OptionEffect.fields:
self.__set(k, daily_stock_quote.get(k))
def set(self, key, value):
raise RuntimeError("Should not be used")
def __set(self, key, value):
if key not in OptionEffect.fields:
raise KeyError(f"Invalid key={key}")
self._values[key] = value
if __name__ == '__main__':
metadata_dir = setup_metadata_dir()
setup_logger(__file__)
logger.setLevel(logging.DEBUG)
logging.getLogger("web_chrome_driver").setLevel(logging.DEBUG)
# test from online reading
with ChromeDriver() as browser:
option_quote = read_daily_option_quote(browser,
"AMD",
"Call",
60.0,
20220121,
use_barchart=True)
stock_quote = read_stock_quote(browser, "AMD")
option_effect = OptionEffect(daily_option_quote=option_quote,
daily_stock_quote=stock_quote)
print(json.dumps(option_effect.__dict__, indent=4))
def hunt_option_activity(
max_oa: int = 0,
option_activity_file: str = None,
) -> typing.Mapping[str, DailyOptionInfo]:
logger.info("================================================")
logger.info(f"{get_time_log()} Hunt or to be hunted !!!")
# synchronized queues
raw_oa_queues = [mp.Queue(maxsize=1000) for _ in range(mp.cpu_count())]
filtered_oa_queue = mp.Queue(maxsize=1000)
info_cache_queue = mp.Queue(maxsize=100)
# pull option activity list
option_activity_list = []
if option_activity_file is None:
with ChromeDriver() as browser:
option_activity_list = read_option_activity(
browser,
save_file=True,
folder="records",
)
else:
with openw(option_activity_file, "rt") as fp:
option_activity_list = fp.readlines()
if max_oa > 0 and max_oa < len(option_activity_list):
option_activity_list = option_activity_list[0:max_oa]
# launch down-stream workers
worker_procs = [
mp.Process(
target=evaluate_option_activity,
args=(
raw_oa_queues[idx],
filtered_oa_queue,
info_cache_queue,
),
name=f"ProcWorker{idx}",
) for idx in range(len(raw_oa_queues))
]
_ = [p.start() for p in worker_procs]
# scan UOA list and enqueue candidates
with DBMS(get_db_file()) as db:
for line in option_activity_list:
option_activity = OptionActivity()
option_activity.from_activity_str(line)
# lookup volume info
symbol = option_activity.get('symbol')
avg_option_info = query_avg_option_info(db, symbol)
if avg_option_info.get("count") == 0:
# add to tracking list
add_to_symbol_table(db, symbol)
# add to queue for processing
worker_idx = hash_symbol(symbol) % len(raw_oa_queues)
raw_oa_queues[worker_idx].put(
[option_activity, avg_option_info],
block=True,
)
_ = [x.put(None) for x in raw_oa_queues]
_ = [p.join() for p in worker_procs]
logger.info("Hunting done !!!")
return process_unusual_option_activity(filtered_oa_queue,
info_cache_queue,
len(raw_oa_queues), db)
return {}