def _bulk_download(self, urls):
results = []
for i, batched_urls in enumerate(batch(urls, self.batch_size, self.sleep)):
for r in self.__bulk_download(batched_urls):
results.append((r.url, r.content))
logger.debug("Downloaded %d of %d urls" % ((i * self.batch_size) + self.batch_size, len(urls)))
logger.debug("got %d results" % len(results))
return results
def _bulk_download(self, urls):
results = []
for i, batched_urls in enumerate(batch(urls, self.batch_size, self.sleep)):
for r in self.__bulk_download(batched_urls):
results.append((r.url, r.content))
logger.debug('Downloaded %d of %d urls' % (
(i * self.batch_size) + self.batch_size,
len(urls)
))
logger.debug('got %d results' % len(results))
return results
def initialize_store(self):
symbols = get_symbols_list('WIKI')
info = get_symbols_info(symbols)
# HACK: find the most recent valid trade date by searching through
# all of the last_trade_dates for the most frequent date
dates = defaultdict(lambda: 0)
for d in info:
dates[d['last_trade_date']] += 1
items = dates.items()
items.sort(key=lambda x: x[1])
last_valid_trade_date = items[-1][0]
logger.debug('last_valid_trade_date: %s' % last_valid_trade_date)
# filter out all symbols which are no longer active
symbols = [d['symbol'] for d in info
if d['last_trade_date'] == last_valid_trade_date]
logger.debug('expecting %d symbols' % len(symbols))
# download daily data
self._store.set_dfs(self._provider.download(symbols))
def initialize_store(self):
symbols = get_symbols_list('WIKI')
info = get_symbols_info(symbols)
# HACK: find the most recent valid trade date by searching through
# all of the last_trade_dates for the most frequent date
dates = defaultdict(lambda: 0)
for d in info:
dates[d['last_trade_date']] += 1
items = dates.items()
items.sort(key=lambda x: x[1])
last_valid_trade_date = items[-1][0]
logger.debug('last_valid_trade_date: %s' % last_valid_trade_date)
# filter out all symbols which are no longer active
symbols = [
d['symbol'] for d in info
if d['last_trade_date'] == last_valid_trade_date
]
logger.debug('expecting %d symbols' % len(symbols))
# download daily data
self._store.set_dfs(self._provider.download(symbols))