def app(symbols, start, end, threads, timeframe, folder, header): if start > end: return lock = threading.Lock() global day_counter total_days = how_many_days(start, end) if total_days == 0: return last_fetch = deque([], maxlen=5) update_progress(day_counter, total_days, -1, threads) def do_work(symbol, day, csv): global day_counter star_time = time.time() Logger.info("Fetching day {0}".format(day)) try: csv.append(day, decompress(symbol, day, fetch_day(symbol, day))) except Exception as e: print("ERROR for {0}, {1} Exception : {2}".format( day, symbol, str(e))) elapsed_time = time.time() - star_time last_fetch.append(elapsed_time) with lock: day_counter += 1 Logger.info("Day {0} fetched in {1}s".format(day, elapsed_time)) futures = [] with concurrent.futures.ThreadPoolExecutor( max_workers=threads) as executor: files = { symbol: CSVDumper(symbol, timeframe, start, end, folder, header) for symbol in symbols } for symbol in symbols: for day in days(start, end): futures.append( executor.submit(do_work, symbol, day, files[symbol])) for future in concurrent.futures.as_completed(futures): if future.exception() is None: update_progress(day_counter, total_days, avg(last_fetch), threads) else: Logger.error("An error happen when fetching data : ", future.exception()) Logger.info("Fetching data terminated") for file in files.values(): file.dump() update_progress(day_counter, total_days, avg(last_fetch), threads)
def do_work(symbol, day, csv): global day_counter star_time = time.time() Logger.info("Fetching day {0}".format(day)) try: csv.append(day, decompress(symbol, day, fetch_day(symbol, day))) except Exception as e: print("ERROR for {0}, {1} Exception : {2}".format( day, symbol, str(e))) elapsed_time = time.time() - star_time last_fetch.append(elapsed_time) with lock: day_counter += 1 Logger.info("Day {0} fetched in {1}s".format(day, elapsed_time))