def app(symbols, start, end, threads, timeframe, folder, header):
if start > end:
return
lock = threading.Lock()
global day_counter
total_days = how_many_days(start, end)
if total_days == 0:
return
last_fetch = deque([], maxlen=5)
update_progress(day_counter, total_days, -1, threads)
def do_work(symbol, day, csv):
global day_counter
star_time = time.time()
Logger.info("Fetching day {0}".format(day))
try:
csv.append(day, decompress(symbol, day, fetch_day(symbol, day)))
except Exception as e:
print("ERROR for {0}, {1} Exception : {2}".format(
day, symbol, str(e)))
elapsed_time = time.time() - star_time
last_fetch.append(elapsed_time)
with lock:
day_counter += 1
Logger.info("Day {0} fetched in {1}s".format(day, elapsed_time))
futures = []
with concurrent.futures.ThreadPoolExecutor(
max_workers=threads) as executor:
files = {
symbol: CSVDumper(symbol, timeframe, start, end, folder, header)
for symbol in symbols
}
for symbol in symbols:
for day in days(start, end):
futures.append(
executor.submit(do_work, symbol, day, files[symbol]))
for future in concurrent.futures.as_completed(futures):
if future.exception() is None:
update_progress(day_counter, total_days, avg(last_fetch),
threads)
else:
Logger.error("An error happen when fetching data : ",
future.exception())
Logger.info("Fetching data terminated")
for file in files.values():
file.dump()
update_progress(day_counter, total_days, avg(last_fetch), threads)
def do_work(symbol, day, csv):
global day_counter
star_time = time.time()
Logger.info("Fetching day {0}".format(day))
try:
csv.append(day, decompress(symbol, day, fetch_day(symbol, day)))
except Exception as e:
print("ERROR for {0}, {1} Exception : {2}".format(
day, symbol, str(e)))
elapsed_time = time.time() - star_time
last_fetch.append(elapsed_time)
with lock:
day_counter += 1
Logger.info("Day {0} fetched in {1}s".format(day, elapsed_time))
