def download_today(stock_list): ''' Download the latest stock prices and return them as a list Incoming data is csv format without a header. Each line in the input contains the following fields (in this order): stock, date, open, range, close, volume where range is a string of the format: 'open - close' The return value of this function is a list of dicts with the row headers as keys. Types are set to datetime for date, int for volume and float for all prices. The keys of the dicts are: stock, date, open, high, low, close, volume Exceptions to this are: * If a stock was not found on Yahoo, its dict will only have 1 key (stock) * If a stock did not have valid data (yet), its dict will only have 2 keys: (stock, date) ''' if not stock_list: text = 'WARNING: empty stock_list on {}'.format(datetime.date.today()) notify_admin(text) return None data = [] max_num_stocks = 200 # yahoo limitation sub_lists = [stock_list[i:i+max_num_stocks] for i in range(0,len(stock_list),max_num_stocks)] for sub_list in sub_lists: url = _yahoo_today_url(sub_list) print url response = urlopen(url) yahoo_data = reader(response, delimiter=',') header = ('stock', 'date', 'open', 'range', 'close', 'volume') for row, stock in zip(yahoo_data, sub_list): values = {'stock': stock} for key, value in zip(header,row): print key, value if 'N/A' in value: values = {'stock': stock, 'date': values.get('date', 'N/A')} break if key == 'date': (m, d, y) = [int(x) for x in value.split('/')] values['date'] = datetime.date(y, m, d) elif key == 'range': lo, hi = value.split(' - ') values['low'] = float(lo) values['high'] = float(hi) elif key == 'volume': values['volume'] = int(value) elif key == 'stock': if value != stock.name: values = {'stock': stock} break elif key == 'open' or key == 'close': values[key] = float(value) else: raise KeyError data.append(values) return data
def download_history(stock, fromdate, todate): ''' Download historical stock prices and store them in the database. Download prices for <stock> from <fromdate> to <todate> from Yahoo and store them in the database. The prices and volume are adjusted for share splits but not for dividends and other adjustments. Incoming data is csv format with the first row as a header. This function returns a list of dicts with the row headers as keys. Types are set to datetime for date, int for volume and float for all prices. The stock instance is added to each dict. Returns None if no valid response was received from Yahoo. ''' url = _yahoo_history_url(stock.name, fromdate, todate) print url response = urlopen(url) yahoo_data = reader(response, delimiter=',') header = [d.lower().replace(' ', '_') for d in yahoo_data.next()] if '<' in header[0]: return None data = [] for row in yahoo_data: values = {'stock': stock} for key, value in zip(header,row): if key == 'date': (y, m, d) = [int(x) for x in value.split('-')] values['date'] = datetime.date(y, m, d) elif key == 'adj_close': values['adj_close'] = float(value) elif key == 'volume': values['volume'] = int(value) else: values[key] = float(value) data.append(values) if not data: text = 'ERROR: {} not found or invalid date range'.format(stock.name) notify_admin(text) return warning = _unsplit(data) #modifies data! if warning: text = 'WARNING for {}:\n'.format(stock.name, '\n -'.join(warning)) notify_admin(text) return data