def loop_listen(self) -> None:
while True:
data = self.client.recv(self.buffer_size).decode('utf-8')
if data != '':
Logger.console_log(message="Received {} of type {}".format(
data, type(data)),
status=Logger.LogStatus.SUCCESS)
def __init__(self,
data_refresh_rate: int = 1800,
relative_path_correction: str = "") -> None:
"""
Constructor.
:param data_refresh_rate:
:param relative_path_correction:
"""
Logger.console_log("Initializing Stock Scraper.",
status=Logger.LogStatus.COMMUNICATION)
self.scrape = True
self.available_cpus = cpu_count() - 1
self.threads = {}
self.mode_lock = Lock()
self.executive_lock = Lock()
self.mode = Scraper.ServerModes.INVALID
self.stock_ticker_letters_to_survey = list(ascii_uppercase)
self.relative_path_correction = relative_path_correction
# Initialize Threads
# Build worker threads for available number of cpus
self.threads[Scraper.WorkerThread] = []
for available_cpu_index in range(self.available_cpus):
self.threads[Scraper.WorkerThread].append(
Scraper.WorkerThread(thread_id=available_cpu_index,
scraper_server=self))
# Build Executive Thread
self.threads[Scraper.ExecutiveThread] = Scraper.ExecutiveThread(
thread_id=0,
scraper_server=self,
worker_threads=self.threads[Scraper.WorkerThread])
def survey_market():
from yahoo_fin.stock_info import get_data
Logger.console_log("Beginning survey of all possible stock symbols", 1)
def concatenate_letters(one, two, three, four, five):
if letter_four is None:
return letter_one + letter_two + letter_three
elif five is None:
return letter_one + letter_two + letter_three + letter_four
else:
return letter_one + letter_two + letter_three + letter_four + letter_five
def log_valid_stock_symbol(symbol):
log_location = os.getcwd(
) + os.path.sep + "Data" + os.path.sep + "valid_stocks.txt"
with open(log_location, 'a+') as stock_log:
stock_log.write(symbol + '\n')
def survey_symbol(symbol):
try:
try:
data = get_data(symbol)
# Logger.console_log("Stock for symbol {} exists.".format(symbol), 1)
log_valid_stock_symbol(symbol)
except KeyError:
# Logger.console_log("Stock for symbol {} produced KeyError on adjclose.".format(symbol), 4)
pass
except ValueError:
Logger.console_log(
"ValueError when attempting to retrieve data for stock symbol {}. Retrying..."
.format(symbol), Logger.LogStatus.FAIL)
survey_symbol(symbol)
except AssertionError:
# Logger.console_log("Stock for symbol {} does not exist.".format(symbol), 2)
pass
ascii_uppercase_and_none = list(ascii_uppercase)
ascii_uppercase_and_none.append(None)
for letter_one in ascii_uppercase[2:]:
for letter_two in ascii_uppercase:
for letter_three in ascii_uppercase:
for letter_four in ascii_uppercase_and_none:
if letter_four is None:
three_letter_symbol = concatenate_letters(
letter_one, letter_two, letter_three, letter_four,
None)
survey_symbol(three_letter_symbol)
else:
for letter_five in ascii_uppercase_and_none:
four_or_five_letter_symbol = concatenate_letters(
letter_one, letter_two, letter_three,
letter_four, letter_five)
survey_symbol(four_or_five_letter_symbol)
def connection_test(self) -> bool:
try:
test_connection = self.create_connection()
Logger.console_log(message="Connection test with " +
self.database + " located at " + self.host +
" was a success",
status=Logger.LogStatus.SUCCESS)
return True
except self.sql_type.value.Error as err:
Logger.console_log(
message="Unable to establish connection with database " +
self.database + ". Error [" + str(err) + "] was returned",
status=Logger.LogStatus.FAIL)
return False
def survey_symbol(symbol):
try:
try:
data = get_data(symbol)
# Logger.console_log("Stock for symbol {} exists.".format(symbol), 1)
log_valid_stock_symbol(symbol)
except KeyError:
# Logger.console_log("Stock for symbol {} produced KeyError on adjclose.".format(symbol), 4)
pass
except ValueError:
Logger.console_log(
"ValueError when attempting to retrieve data for stock symbol {}. Retrying..."
.format(symbol), Logger.LogStatus.FAIL)
survey_symbol(symbol)
except AssertionError:
# Logger.console_log("Stock for symbol {} does not exist.".format(symbol), 2)
pass
def survey_ticker(thread: Thread, ticker: str) -> bool:
"""
Uses Scraper.get_current_ticker_data to survey the given ticker string. The result is returned.
:param thread: A Scraper.WorkerThread responsible for carrying out the survey.
:param ticker: Stock ticker.
:return: Boolean denoting rather or not the ticker was successfully scraped during the survey.
"""
try:
data = Scraper.WebInterface.get_current_ticker_data(ticker=ticker)
if None in data:
return False
else:
Logger.console_log(
"Successful scraping of ticker " + ticker + " from " +
str(thread), Logger.LogStatus.SUCCESS)
return True
except Exception as e:
return False
def menu(self) -> None:
Logger.console_log(message="==== MENU ====",
status=Logger.LogStatus.COMMUNICATION)
Logger.console_log(message="1) send a message",
status=Logger.LogStatus.COMMUNICATION)
Logger.console_log(message="0) end connection",
status=Logger.LogStatus.COMMUNICATION)
response = int(input("What would you like to do: "))
if response == 1:
self.prompt_for_message()
elif response == 0:
self.end_connection()
else:
Logger.console_log(
message="Invalid menu selection. Please try again...",
status=Logger.LogStatus.FAIL)
return response
def connect(self, ip_address: str, port: int) -> None:
port_description = (ip_address, port)
Logger.console_log(
message="Attempting to bind to port with description: " +
str(port_description),
status=Logger.LogStatus.EMPHASIS)
# Bind to the port description
try:
self.sock.connect(port_description)
Logger.console_log(
message=
"Successfully created connection with server @ address {} through port {}"
.format(ip_address, port),
status=Logger.LogStatus.SUCCESS)
except Exception as err:
Logger.console_log(
message="Unable to send message due to error: " + str(err),
status=Logger.LogStatus.FAIL)
def connect(self, ip_address: str, port: int) -> None:
port_description = (ip_address, port)
try:
# Bind to the port description
self.sock.bind(port_description)
# Sets up and start TCP Listener
self.sock.listen(2)
# Wait for TCP connection to arrive [Returns client]
Logger.console_log(message="Waiting for client...",
status=Logger.LogStatus.EMPHASIS)
self.client, client_address = self.sock.accept()
Logger.console_log(
message="Creating connection with client from " +
str(client_address),
status=Logger.LogStatus.SUCCESS)
except Exception as err:
Logger.console_log(
message="Unable to send message due to error: " + str(err),
status=Logger.LogStatus.FAIL)
def engineer_additional_features(stock_data: pd.DataFrame) -> None:
intervals = range(6, 21)
Logger.console_log(
"Calculating Relative Stock Index over intervals " + str(intervals),
Logger.LogStatus.EMPHASIS)
RSI(stock_data, 'close', intervals)
Logger.console_log(
"Calculating Williams Percent Range over intervals " + str(intervals),
Logger.LogStatus.EMPHASIS)
WilliamsR(stock_data, intervals)
Logger.console_log(
"Calculating Weighted Moving Average over intervals " + str(intervals),
Logger.LogStatus.EMPHASIS)
WMA(stock_data, 'close', intervals)
Logger.console_log(
"Calculating Exponential Moving Average over intervals " +
str(intervals), Logger.LogStatus.EMPHASIS)
EMA(stock_data, 'close', intervals)
Logger.console_log(
"Calculating Simple Moving Average over intervals " + str(intervals),
Logger.LogStatus.EMPHASIS)
SMA(stock_data, 'close', intervals)
Logger.console_log(
"Calculating Hull Moving Average over intervals " + str(intervals),
Logger.LogStatus.EMPHASIS)
HMA(stock_data, 'close', intervals)
Logger.console_log(
"Calculating Triple Exponentially Smoothed Moving Average over intervals "
+ str(intervals), Logger.LogStatus.EMPHASIS)
TRIX(stock_data, 'close', intervals)
Logger.console_log(
"Calculating Commodity Channel Index over intervals " + str(intervals),
Logger.LogStatus.EMPHASIS)
CCI(stock_data, intervals)
Logger.console_log(
"Calculating Chande Momentum Oscillator over intervals " +
str(intervals), Logger.LogStatus.EMPHASIS)
CMO(stock_data, 'close', intervals)
Logger.console_log(
"Calculating Moving Average Convergence/Divergence over intervals " +
str(intervals), Logger.LogStatus.EMPHASIS)
MACD(stock_data, 'close', intervals)
Logger.console_log(
"Calculating Detrended Price Oscillator over intervals " +
str(intervals), Logger.LogStatus.EMPHASIS)
DPO(stock_data, 'close', intervals)
Logger.console_log(
"Calculating Rate of Change over intervals " + str(intervals),
Logger.LogStatus.EMPHASIS)
ROC(stock_data, 'close', intervals)
Logger.console_log(
"Calculating Chaikin Money Flow over intervals " + str(intervals),
Logger.LogStatus.EMPHASIS)
CMF(stock_data, intervals)
Logger.console_log(
"Calculating Average Directional Index over intervals " +
str(intervals), Logger.LogStatus.EMPHASIS)
ADX(stock_data, intervals)
Logger.console_log(
"Calculating Force Index over intervals " + str(intervals),
Logger.LogStatus.EMPHASIS)
ForceIndex(stock_data, intervals)
stock_data = stock_data.drop(stock_data.index[range(intervals[-1])])
def engineer_data_over_single_interval(df: pd.DataFrame,
indicators: list,
ticker: str = "",
rsi_n: int = 14,
cmo_n: int = 7,
macd_fast: int = 12,
macd_slow: int = 26,
macd_sign: int = 9,
roc_n: int = 12,
cci_n: int = 20,
dpo_n: int = 20,
cmf_n: int = 20,
adx_n: int = 14,
mass_index_low: int = 9,
mass_index_high: int = 25,
trix_n: int = 15,
stochastic_oscillator_n: int = 14,
stochastic_oscillator_sma_n: int = 3,
ultimate_oscillator_short_n: int = 7,
ultimate_oscillator_medium_n: int = 14,
ultimate_oscillator_long_n: int = 28,
ao_short_n: int = 5,
ao_long_n: int = 34,
kama_n: int = 10,
tsi_high_n: int = 25,
tsi_low_n: int = 13,
eom_n: int = 14,
force_index_n: int = 13,
ichimoku_low_n: int = 9,
ichimoku_medium_n: int = 26):
from ta.momentum import rsi, wr, roc, ao, stoch, uo, kama, tsi
from ta.trend import macd, macd_signal, cci, dpo, adx, mass_index, trix, ichimoku_a
from ta.volume import chaikin_money_flow, acc_dist_index, ease_of_movement, force_index
# Momentum Indicators
if Indicators.RELATIVE_STOCK_INDEX in indicators:
Logger.console_log(message="Calculating " +
Indicators.RELATIVE_STOCK_INDEX.value +
" for stock " + ticker,
status=Logger.LogStatus.EMPHASIS)
df[Indicators.RELATIVE_STOCK_INDEX.value] = rsi(close=df['close'],
n=rsi_n)
if Indicators.WILLIAMS_PERCENT_RANGE in indicators:
Logger.console_log(message="Calculating " +
Indicators.WILLIAMS_PERCENT_RANGE.value +
" for stock " + ticker,
status=Logger.LogStatus.EMPHASIS)
df[Indicators.WILLIAMS_PERCENT_RANGE.value] = wr(
df['high'], df['low'], df['close'])
if Indicators.CHANDE_MOMENTUM_OSCILLATOR in indicators:
Logger.console_log(message="Calculating " +
Indicators.CHANDE_MOMENTUM_OSCILLATOR.value +
" for stock " + ticker,
status=Logger.LogStatus.EMPHASIS)
df[Indicators.CHANDE_MOMENTUM_OSCILLATOR.
value] = chande_momentum_oscillator(close_data=df['close'],
period=cmo_n)
if Indicators.RATE_OF_CHANGE in indicators:
Logger.console_log(message="Calculating " +
Indicators.RATE_OF_CHANGE.value + " for stock " +
ticker,
status=Logger.LogStatus.EMPHASIS)
df[Indicators.RATE_OF_CHANGE.value] = roc(close=df['close'], n=roc_n)
if Indicators.STOCHASTIC_OSCILLATOR in indicators:
Logger.console_log(message="Calculating " +
Indicators.STOCHASTIC_OSCILLATOR.value +
" for stock " + ticker,
status=Logger.LogStatus.EMPHASIS)
df[Indicators.STOCHASTIC_OSCILLATOR.value] = stoch(
high=df['high'],
low=df['low'],
close=df['close'],
n=stochastic_oscillator_n,
d_n=stochastic_oscillator_sma_n)
if Indicators.ULTIMATE_OSCILLATOR in indicators:
Logger.console_log(message="Calculating " +
Indicators.ULTIMATE_OSCILLATOR.value +
" for stock " + ticker,
status=Logger.LogStatus.EMPHASIS)
df[Indicators.ULTIMATE_OSCILLATOR.value] = uo(
high=df['high'],
low=df['low'],
close=df['close'],
s=ultimate_oscillator_short_n,
m=ultimate_oscillator_medium_n,
len=ultimate_oscillator_long_n)
if Indicators.AWESOME_OSCILLATOR in indicators:
Logger.console_log(message="Calculating " +
Indicators.AWESOME_OSCILLATOR.value +
" for stock " + ticker,
status=Logger.LogStatus.EMPHASIS)
df[Indicators.AWESOME_OSCILLATOR.value] = ao(high=df['high'],
low=df['low'],
s=ao_short_n,
len=ao_long_n)
if Indicators.KAUFMAN_ADAPTIVE_MOVING_AVERAGE in indicators:
Logger.console_log(message="Calculating " +
Indicators.KAUFMAN_ADAPTIVE_MOVING_AVERAGE.value +
" for stock " + ticker,
status=Logger.LogStatus.EMPHASIS)
df[Indicators.KAUFMAN_ADAPTIVE_MOVING_AVERAGE.value] = kama(
close=df['close'], n=kama_n)
if Indicators.TRUE_STRENGTH_INDEX in indicators:
Logger.console_log(message="Calculating " +
Indicators.TRUE_STRENGTH_INDEX.value +
" for stock " + ticker,
status=Logger.LogStatus.EMPHASIS)
df[Indicators.TRUE_STRENGTH_INDEX.value] = tsi(close=df['close'],
r=tsi_high_n,
s=tsi_low_n)
# Trend Indicator
if Indicators.MOVING_AVERAGE_CONVERGENCE_DIVERGENCE in indicators:
Logger.console_log(
message="Calculating " +
Indicators.MOVING_AVERAGE_CONVERGENCE_DIVERGENCE.value +
" for stock " + ticker,
status=Logger.LogStatus.EMPHASIS)
df[Indicators.MOVING_AVERAGE_CONVERGENCE_DIVERGENCE.value] = macd(close=df['close'],
n_slow=macd_slow,
n_fast=macd_fast) - \
macd_signal(close=df['close'],
n_slow=macd_slow,
n_fast=macd_fast,
n_sign=macd_sign)
if Indicators.COMMODITY_CHANNEL_INDEX in indicators:
Logger.console_log(message="Calculating " +
Indicators.COMMODITY_CHANNEL_INDEX.value +
" for stock " + ticker,
status=Logger.LogStatus.EMPHASIS)
df[Indicators.COMMODITY_CHANNEL_INDEX.value] = cci(high=df['high'],
low=df['low'],
close=df['close'],
n=cci_n)
if Indicators.DETRENDED_PRICE_OSCILLATOR in indicators:
Logger.console_log(message="Calculating " +
Indicators.DETRENDED_PRICE_OSCILLATOR.value +
" for stock " + ticker,
status=Logger.LogStatus.EMPHASIS)
df[Indicators.DETRENDED_PRICE_OSCILLATOR.value] = dpo(
close=df['close'], n=dpo_n)
if Indicators.AVERAGE_DIRECTIONAL_INDEX in indicators:
Logger.console_log(message="Calculating " +
Indicators.AVERAGE_DIRECTIONAL_INDEX.value +
" for stock " + ticker,
status=Logger.LogStatus.EMPHASIS)
df[Indicators.AVERAGE_DIRECTIONAL_INDEX.value] = adx(high=df['high'],
low=df['low'],
close=df['close'],
n=adx_n)
if Indicators.MASS_INDEX in indicators:
Logger.console_log(message="Calculating " +
Indicators.MASS_INDEX.value + " for stock " +
ticker,
status=Logger.LogStatus.EMPHASIS)
df[Indicators.MASS_INDEX.value] = mass_index(high=df['high'],
low=df['low'],
n=mass_index_low,
n2=mass_index_high)
if Indicators.TRIPLE_EXPONENTIALLY_SMOOTHED_MOVING_AVERAGE in indicators:
Logger.console_log(
message="Calculating " +
Indicators.TRIPLE_EXPONENTIALLY_SMOOTHED_MOVING_AVERAGE.value +
" for stock " + ticker,
status=Logger.LogStatus.EMPHASIS)
df[Indicators.TRIPLE_EXPONENTIALLY_SMOOTHED_MOVING_AVERAGE.
value] = trix(close=df['close'], n=trix_n)
if Indicators.ICHIMOKU_A in indicators:
Logger.console_log(message="Calculating " +
Indicators.ICHIMOKU_A.value + " for stock " +
ticker,
status=Logger.LogStatus.EMPHASIS)
df[Indicators.ICHIMOKU_A.value] = ichimoku_a(high=df['high'],
low=df['low'],
n1=ichimoku_low_n,
n2=ichimoku_medium_n)
# Volume Indicator
if Indicators.CHAIKIN_MONEY_FLOW in indicators:
Logger.console_log(message="Calculating " +
Indicators.CHAIKIN_MONEY_FLOW.value +
" for stock " + ticker,
status=Logger.LogStatus.EMPHASIS)
df[Indicators.CHAIKIN_MONEY_FLOW.value] = chaikin_money_flow(
high=df['high'],
low=df['low'],
close=df['close'],
volume=df['volume'],
n=cmf_n)
if Indicators.ACCUMULATION_DISTRIBUTION_INDEX in indicators:
Logger.console_log(message="Calculating " +
Indicators.ACCUMULATION_DISTRIBUTION_INDEX.value +
" for stock " + ticker,
status=Logger.LogStatus.EMPHASIS)
df[Indicators.ACCUMULATION_DISTRIBUTION_INDEX.value] = acc_dist_index(
high=df['high'],
low=df['low'],
close=df['close'],
volume=df['volume'])
if Indicators.EASE_OF_MOVEMENT in indicators:
Logger.console_log(message="Calculating " +
Indicators.EASE_OF_MOVEMENT.value + " for stock " +
ticker,
status=Logger.LogStatus.EMPHASIS)
df[Indicators.EASE_OF_MOVEMENT.value] = ease_of_movement(
high=df['high'], low=df['low'], volume=df['volume'], n=eom_n)
if Indicators.FORCE_INDEX in indicators:
Logger.console_log(message="Calculating " +
Indicators.FORCE_INDEX.value + " for stock " +
ticker,
status=Logger.LogStatus.EMPHASIS)
df[Indicators.FORCE_INDEX.value] = force_index(close=df['close'],
volume=df['volume'],
n=force_index_n)
def run(self) -> None:
"""
"""
try:
last_polled_server_mode = Scraper.ServerModes.INVALID
ticker_survey_list = self.ticker_survey_list
ticker_monitor_list = self.ticker_monitor_list
while last_polled_server_mode != Scraper.ServerModes.SHUT_DOWN:
if last_polled_server_mode == Scraper.ServerModes.SURVEY_MARKET:
# Survey the market, it must be closed.
if len(ticker_survey_list) == 0:
# No data yet... sleep
sleep(5)
if self.scraper_server.executive_lock.acquire(
blocking=False):
ticker_survey_list = self.ticker_survey_list
self.scraper_server.executive_lock.release()
else:
# Get ticker to survey.
ticker = self.ticker_survey_list[0]
# Log Result.
if Scraper.survey_ticker(thread=self,
ticker=ticker):
self.scraper_server.executive_lock.acquire()
Scraper.FileHandler.add_ticker_to_tickers(
ticker=ticker,
relative_path_correction=self.
scraper_server.relative_path_correction)
self.scraper_server.executive_lock.release()
# remove from survey list.
self.ticker_survey_list.remove(ticker)
elif last_polled_server_mode == Scraper.ServerModes.RETRIEVE_DATA:
# Stock market is open schedule
if len(ticker_monitor_list) == 0:
# The ticker monitor list is currently empty. Create non blocking check for tickers to monitor.
if self.scraper_server.executive_lock.acquire(
blocking=False):
ticker_monitor_list = self.ticker_monitor_list
self.scraper_server.executive_lock.release()
else:
# Survey the current ticker_monitor_list
for ticker in ticker_monitor_list:
try:
ticker_data = Scraper.WebInterface.get_current_ticker_data(
ticker=ticker)
Scraper.FileHandler.store_ticker_data(
ticker=ticker,
data=ticker_data,
relative_path_correction=self.
scraper_server.relative_path_correction
)
Logger.console_log(
message=str(self) +
" was able to retrieve data for ticker: "
+ ticker + ".",
status=Logger.LogStatus.SUCCESS)
except Exception as e:
Logger.console_log(
message=str(self) +
" was unable to retrieve data for ticker: "
+ ticker + " due to exception [" +
str(e) +
"] trying again in 3 seconds.",
status=Logger.LogStatus.FAIL)
sleep(3)
try:
ticker_data = Scraper.WebInterface.get_current_ticker_data(
ticker=ticker)
Scraper.FileHandler.store_ticker_data(
ticker=ticker,
data=ticker_data,
relative_path_correction=self.
scraper_server.
relative_path_correction)
Logger.console_log(
message=str(self) +
" was able to retrieve data for ticker: "
+ ticker + ".",
status=Logger.LogStatus.SUCCESS)
except ValueError:
Logger.console_log(
message=str(self) +
" was unable to retrieve data for ticker again: "
+ ticker +
" due to ValueError. Will try again later.",
status=Logger.LogStatus.FAIL)
except TypeError:
Logger.console_log(
message=str(self) +
" was unable to retrieve data for ticker again: "
+ ticker +
" due to TypeError. It is likely not a valid ticker. Removing"
+ "from list of valid tickers.",
status=Logger.LogStatus.FAIL)
Scraper.FileHandler.remove_invalid_ticker(
ticker=ticker,
relative_path_correction=self.
scraper_server.
relative_path_correction)
except Exception as e:
Logger.console_log(
message=str(self) +
" was unable to retrieve data for ticker again: "
+ ticker + " due to exception " +
str(e) + ".",
status=Logger.LogStatus.FAIL)
# Update the server mode for the next loop
if last_polled_server_mode != self.scraper_server.mode:
Logger.console_log(
str(self) +
" has noticed a change in scraper server mode to "
+ str(self.scraper_server.mode) + " the time is " +
str(datetime.now()),
status=Logger.LogStatus.COMMUNICATION)
# server_mode change. Time to retrieve processing data.
if self.scraper_server.mode == Scraper.ServerModes.SURVEY_MARKET:
# Survey the market, it must be closed.
self.scraper_server.executive_lock.acquire()
ticker_survey_list = self.ticker_survey_list
self.scraper_server.executive_lock.release()
elif self.scraper_server.mode == Scraper.ServerModes.RETRIEVE_DATA:
# Stock market is open schedule
self.scraper_server.executive_lock.acquire()
ticker_monitor_list = self.ticker_monitor_list
self.scraper_server.executive_lock.release()
# Update the server mode
last_polled_server_mode = self.scraper_server.mode
except (KeyboardInterrupt, SystemExit):
self.scraper_server.mode_lock.acquire()
self.scraper_server.mode = Scraper.ServerModes.SHUT_DOWN
self.scraper_server.mode_lock.release()
def run(self) -> None:
"""
"""
try:
last_polled_server_mode = Scraper.ServerModes.INVALID
while last_polled_server_mode != Scraper.ServerModes.SHUT_DOWN:
# Set the mode of the scraper server
self.scraper_server.mode_lock.acquire()
self.scraper_server.mode = Scraper.ServerModes.set_mode()
# The mode has changed. It is time to update data for the workers.
if last_polled_server_mode != self.scraper_server.mode:
Logger.console_log(
str(self) +
" has updated the scraper server mode to " +
str(self.scraper_server.mode) + " the time is " +
str(datetime.now()),
status=Logger.LogStatus.COMMUNICATION)
# The server mode is updated. Time to update the data
if self.scraper_server.mode == Scraper.ServerModes.SURVEY_MARKET:
# Give workers a list of stocks to monitor Get letters to be surveyed by workers
worker_starter_letters = self.scraper_server.stock_ticker_letters_to_survey[:
self
.
scraper_server
.
available_cpus]
# Remove worker starter letters from survey list
for worker_starter_letter in worker_starter_letters:
self.scraper_server.stock_ticker_letters_to_survey.remove(
worker_starter_letter)
for worker_enum, worker_starter_letter in enumerate(
worker_starter_letters):
Logger.console_log(
str(self) + " is asking " +
str(self.worker_threads[worker_enum]) +
" to survey letter " +
worker_starter_letter,
Logger.LogStatus.COMMUNICATION)
# Get last surveyed ticker
previous_ticker_list = Scraper.FileHandler.load_tickers(
ticker_starting_letter=
worker_starter_letter,
relative_path_correction=self.
scraper_server.relative_path_correction)
if len(previous_ticker_list) > 0:
last_surveyed_ticker = previous_ticker_list[
-1]
else:
last_surveyed_ticker = None
# Get new survey list.
ticker_survey_list = Scraper.get_ticker_survey_list(
starting_letter=worker_starter_letter,
last_ticker_surveyed=last_surveyed_ticker)
# Give the list to the worker
self.scraper_server.executive_lock.acquire()
self.worker_threads[
worker_enum].ticker_survey_list = ticker_survey_list
self.scraper_server.executive_lock.release()
elif self.scraper_server.mode == Scraper.ServerModes.RETRIEVE_DATA:
# Get ticker data
ticker_list = Scraper.FileHandler.load_all_tickers(
relative_path_correction=self.scraper_server.
relative_path_correction)
ticker_sub_lists = np.array_split(
ticker_list,
self.scraper_server.available_cpus)
# Give the ticker sub lists to workers to monitor for the day.
self.scraper_server.executive_lock.acquire()
for worker_enum, worker in enumerate(
self.worker_threads):
Logger.console_log(
str(self) + " is asking " + str(worker) +
" to monitor tickers: " +
str(ticker_sub_lists[worker_enum]),
Logger.LogStatus.COMMUNICATION)
worker.ticker_monitor_list = ticker_sub_lists[
worker_enum]
self.scraper_server.executive_lock.release()
# Update the mode
last_polled_server_mode = self.scraper_server.mode
self.scraper_server.mode_lock.release()
else:
self.scraper_server.mode_lock.release()
# Mode has not changed. Make sure the workers have work to do.
if last_polled_server_mode == Scraper.ServerModes.SURVEY_MARKET:
# Check to see if any of the workers have exhausted their lists
for worker in self.worker_threads:
if len(worker.ticker_survey_list) == 0:
# wait 2 seconds to make sure the worker is finished
sleep(2)
# Get a new letter
new_letter = self.scraper_server.stock_ticker_letters_to_survey.pop(
0)
# Get a new ticker survey list for the worker thread
self.scraper_server.executive_lock.acquire(
)
Logger.console_log(
message=str(worker) +
" has finished their survey. " +
str(self) +
" is asking it to survey " +
new_letter + " next.",
status=Logger.LogStatus.COMMUNICATION)
worker.ticker_survey_list = Scraper.get_ticker_survey_list(
starting_letter=new_letter,
last_ticker_surveyed=Scraper.
FileHandler.get_last_ticker_surveyed(
ticker_starting_letter=new_letter,
relative_path_correction=self.
scraper_server.
relative_path_correction))
self.scraper_server.executive_lock.release(
)
except (KeyboardInterrupt, SystemExit):
Logger.console_log(message="Executive" + str(self.thread_id) +
"shutting down server",
status=Logger.LogStatus.FAIL)
self.scraper_server.mode_lock.acquire()
self.scraper_server.mode = Scraper.ServerModes.SHUT_DOWN
self.scraper_server.mode_lock.release()
def end_connection(self) -> None:
Logger.console_log(message="Ending the TCP Connection.",
status=Logger.LogStatus.EMPHASIS)
self.client.close()
def prompt_for_message(self) -> None:
Logger.console_log(message="", status=Logger.LogStatus.COMMUNICATION)
message = input('What would you like to send to the server?')
self.send_message(message=message)
