def updateStockData_US_Weekly(root_path,
from_date,
till_date,
storeType,
force_check=False):
symbols = getStocksList_US(root_path).index
pbar = tqdm(total=len(symbols))
if storeType == 2: # or storeType == 1:
# count = 10
for stock in symbols:
startTime, message = updateSingleStockData(root_path, stock,
from_date, till_date,
force_check)
outMessage = '%-*s fetched in: %.4s seconds' % (6, stock,
(time.time() -
startTime))
pbar.set_description(outMessage)
pbar.update(1)
# count = count - 1
# if count == 0: break
if storeType == 1:
log_errors = []
log_update = []
# Parallel mode is not suitable in CSV storage mode, since no lock is added to limit csv file IO.
with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor:
# Start the load operations and mark each future with its URL
future_to_stock = {
executor.submit(updateSingleStockData, root_path, symbol,
from_date, till_date, force_check): symbol
for symbol in symbols
}
for future in concurrent.futures.as_completed(future_to_stock):
stock = future_to_stock[future]
try:
startTime, message = future.result()
except Exception as exc:
startTime = time.time()
log_errors.append('%r generated an exception: %s' %
(stock, exc))
len_errors = len(log_errors)
if len_errors % 5 == 0:
print(log_errors[(len_errors - 5):])
else:
if len(message) > 0: log_update.append(message)
outMessage = '%-*s fetched in: %.4s seconds' % (6, stock,
(time.time() -
startTime))
pbar.set_description(outMessage)
pbar.update(1)
if len(log_errors) > 0: print(log_errors)
# if len(log_update) > 0: print(log_update)
pbar.close()
return symbols
def process_all_stocks_data(root_path, window=5):
symbols = getStocksList_US(root_path).index.values.tolist()
pbar = tqdm(total=len(symbols))
day_selection = []
week_selection = []
month_selection = []
# for index in range(0, window):
# day_window = []
# day_selection.append(day_window)
# week_window = []
# week_selection.append(week_window)
# month_window = []
# month_selection.append(month_window)
startTime_1 = time.time()
for symbol in symbols:
startTime = processing_stock_data(root_path, symbol, window,
day_selection, week_selection,
month_selection)
outMessage = '%-*s processed in: %.4s seconds' % (6, symbol,
(time.time() -
startTime))
pbar.set_description(outMessage)
pbar.update(1)
print('total processing in: %.4s seconds' % ((time.time() - startTime_1)))
# with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor:
# # Start the load operations and mark each future with its URL
# future_to_stock = {executor.submit(processing_stock_data, root_path, symbol, window, day_selection, week_selection, month_selection): symbol for symbol in symbols}
# for future in concurrent.futures.as_completed(future_to_stock):
# stock = future_to_stock[future]
# try:
# startTime = future.result()
# except Exception as exc:
# startTime = time.time()
# print('%r generated an exception: %s' % (stock, exc))
# outMessage = '%-*s processed in: %.4s seconds' % (6, stock, (time.time() - startTime))
# pbar.set_description(outMessage)
# pbar.update(1)
# day_week_selection = []
# week_month_selection = []
# day_month_selection = []
# all_selection = []
# count = []
day_week_selection = list(set(day_selection) & set(week_selection))
week_month_selection = list(set(week_selection) & set(month_selection))
day_month_selection = list(set(day_selection) & set(month_selection))
all_selection = list(set(day_week_selection) & set(week_month_selection))
#day_selection = list(set(day_selection) - set(all_selection))
#week_selection = list(set(week_selection) - set(all_selection))
#month_selection = list(set(month_selection) - set(all_selection))
# sumUp = len(day_week_selection[index]) + len(week_month_selection[index]) + len(day_month_selection[index]) + len(all_selection[index])
# count.insert(0,sumUp)
print("all_selection", len(all_selection), sorted(all_selection))
print("day_week_selection", len(day_week_selection),
sorted(day_week_selection))
print("week_month_selection", len(week_month_selection),
sorted(week_month_selection))
print("day_month_selection", len(day_month_selection),
sorted(day_month_selection))
print("/n ------------------------ /n")
# plt.plot(range(0, len(count)), count)
# plt.title('A simple chirp')
# plt.show()
print("day_selection", len(day_selection), sorted(day_selection))
print("week_selection", len(week_selection), sorted(week_selection))
print("month_selection", len(month_selection), sorted(month_selection))