def test_sanitize_dates_int(self): start_int = 2018 end_int = 2019 expected_start = pd.to_datetime(dt.datetime(start_int, 1, 1)) expected_end = pd.to_datetime(dt.datetime(end_int, 1, 1)) assert _sanitize_dates(start_int, end_int) == (expected_start, expected_end)
def __init__( self, symbols, start=None, end=None, retry_count=3, pause=0.1, timeout=30, session=None, freq=None, ): self.symbols = symbols start, end = _sanitize_dates(start or self.default_start_date, end) self.start = start self.end = end if not isinstance(retry_count, int) or retry_count < 0: raise ValueError("'retry_count' must be integer larger than 0") self.retry_count = retry_count self.pause = pause self.timeout = timeout self.pause_multiplier = 1 self.session = _init_session(session) self.freq = freq self.headers = None
def _get_data(name, start=dt.datetime(2010, 1, 1), end=dt.datetime.today()): """ Get data for the given name from OECD. Date format is datetime Returns a DataFrame. """ start, end = _sanitize_dates(start, end) if not isinstance(name, compat.string_types): raise ValueError('data name must be string') # API: https://data.oecd.org/api/sdmx-json-documentation/ url = '{0}/{1}/all/all?'.format(_URL, name) def fetch_data(url, name): resp = _urlopen(url) resp = resp.read() resp = resp.decode('utf-8') data = read_jsdmx(resp) try: idx_name = data.index.name # hack for pandas 0.16.2 data.index = pd.to_datetime(data.index) data = data.sort_index() data = data.truncate(start, end) data.index.name = idx_name except ValueError: pass return data df = fetch_data(url, name) return df
def _get_data(name, start=dt.datetime(2010, 1, 1), end=dt.datetime.today()): """ Get data for the given name from OECD. Date format is datetime Returns a DataFrame. """ start, end = _sanitize_dates(start, end) if not isinstance(name, compat.string_types): raise ValueError("data name must be string") # API: https://data.oecd.org/api/sdmx-json-documentation/ url = "{0}/{1}/all/all?".format(_URL, name) def fetch_data(url, name): resp = _urlopen(url) resp = resp.read() resp = resp.decode("utf-8") data = read_jsdmx(resp) try: idx_name = data.index.name # hack for pandas 0.16.2 data.index = pd.to_datetime(data.index) data = data.sort_index() data = data.truncate(start, end) data.index.name = idx_name except ValueError: pass return data df = fetch_data(url, name) return df
def _get_data_one(sym, start, end, interval, retry_count, pause): """ Get historical data for the given name from google. Date format is datetime Returns a DataFrame. """ start, end = _sanitize_dates(start, end) # www.google.com/finance/historical?q=GOOG&startdate=Jun+9%2C+2011&enddate=Jun+8%2C+2013&output=csv params = {"q": sym, "startdate": start.strftime("%b %d, %Y"), "enddate": end.strftime("%b %d, %Y"), "output": "csv"} url = _encode_url(_URL, params) return _retry_read_url(url, retry_count, pause, "Google")
def __init__(self, symbols, start=None, end=None, retry_count=3, pause=0.1, timeout=30, session=None): self.symbols = symbols start, end = _sanitize_dates(start, end) self.start = start self.end = end if not isinstance(retry_count, int) or retry_count < 0: raise ValueError("'retry_count' must be integer larger than 0") self.retry_count = retry_count self.pause = pause self.timeout = timeout self.session = _init_session(session, retry_count)
def test_sanitize_invalid_dates(self): with pytest.raises(ValueError): _sanitize_dates(2019, 2018) with pytest.raises(ValueError): _sanitize_dates("2019-01-01", "2018-01-01") with pytest.raises(ValueError): _sanitize_dates("20199", None)
def _get_data_one(sym, start, end, interval, retry_count, pause): """ Get historical data for the given name from google. Date format is datetime Returns a DataFrame. """ start, end = _sanitize_dates(start, end) # www.google.com/finance/historical?q=GOOG&startdate=Jun+9%2C+2011&enddate=Jun+8%2C+2013&output=csv params = { 'q': sym, 'startdate': start.strftime('%b %d, %Y'), 'enddate': end.strftime('%b %d, %Y'), 'output': "csv" } url = _encode_url(_URL, params) return _retry_read_url(url, retry_count, pause, 'Google')
def _get_data_one(sym, start, end, interval, retry_count, pause): """ Get historical data for the given name from yahoo. Date format is datetime Returns a DataFrame. """ start, end = _sanitize_dates(start, end) params = { 's': sym, 'a': start.month - 1, 'b': start.day, 'c': start.year, 'd': end.month - 1, 'e': end.day, 'f': end.year, 'g': interval, 'ignore': '.csv' } url = _encode_url(_URL, params) return _retry_read_url(url, retry_count, pause, 'Yahoo!')
def _get_data(name, start=dt.datetime(2010, 1, 1), end=dt.datetime.today()): """ Get data for the given name from the St. Louis FED (FRED). Date format is datetime Returns a DataFrame. If multiple names are passed for "series" then the index of the DataFrame is the outer join of the indicies of each series. """ start, end = _sanitize_dates(start, end) if not is_list_like(name): names = [name] else: names = name urls = [_URL + '%s' % n + '/downloaddata/%s' % n + '.csv' for n in names] def fetch_data(url, name): with urlopen(url) as resp: data = read_csv(resp, index_col=0, parse_dates=True, header=None, skiprows=1, names=["DATE", name], na_values='.') try: return data.truncate(start, end) except KeyError: # pragma: no cover if data.ix[3].name[7:12] == 'Error': raise IOError("Failed to get the data. Check that {0!r} is " "a valid FRED series.".format(name)) raise df = concat([fetch_data(url, n) for url, n in zip(urls, names)], axis=1, join='outer') return df
def test_sanitize_dates_defaults(self): default_start = pd.to_datetime(dt.date.today() - dt.timedelta(days=365 * 5)) default_end = pd.to_datetime(dt.date.today()) assert _sanitize_dates(None, None) == (default_start, default_end)
def test_sanitize_dates(self, input_date): expected_start = pd.to_datetime(input_date) expected_end = pd.to_datetime(dt.date.today()) result = _sanitize_dates(input_date, None) assert result == (expected_start, expected_end)
def DataReader(name, data_source=None, start=None, end=None, retry_count=3, pause=0.001): """ Imports data from a number of online sources. Currently supports Yahoo! Finance, Google Finance, St. Louis FED (FRED) and Kenneth French's data library. Parameters ---------- name : str or list of strs the name of the dataset. Some data sources (yahoo, google, fred) will accept a list of names. data_source: {str, None} the data source ("yahoo", "yahoo-actions", "google", "fred", or "ff") start : {datetime, None} left boundary for range (defaults to 1/1/2010) end : {datetime, None} right boundary for range (defaults to today) retry_count : {int, 3} Number of times to retry query request. pause : {numeric, 0.001} Time, in seconds, to pause between consecutive queries of chunks. If single value given for symbol, represents the pause between retries. Examples ---------- # Data from Yahoo! Finance gs = DataReader("GS", "yahoo") # Corporate Actions (Dividend and Split Data) with ex-dates from Yahoo! Finance gs = DataReader("GS", "yahoo-actions") # Data from Google Finance aapl = DataReader("AAPL", "google") # Data from FRED vix = DataReader("VIXCLS", "fred") # Data from Fama/French ff = DataReader("F-F_Research_Data_Factors", "famafrench") ff = DataReader("F-F_Research_Data_Factors_weekly", "famafrench") ff = DataReader("6_Portfolios_2x3", "famafrench") ff = DataReader("F-F_ST_Reversal_Factor", "famafrench") """ start, end = _sanitize_dates(start, end) if data_source == "yahoo": return get_data_yahoo(symbols=name, start=start, end=end, adjust_price=False, chunksize=25, retry_count=retry_count, pause=pause) elif data_source == "yahoo-actions": return get_data_yahoo_actions(symbol=name, start=start, end=end, retry_count=retry_count, pause=pause) elif data_source == "google": return get_data_google(symbols=name, start=start, end=end, chunksize=25, retry_count=retry_count, pause=pause) elif data_source == "fred": return get_data_fred(name, start, end) elif data_source == "famafrench": return get_data_famafrench(name) elif data_source == "oecd": return get_data_oecd(name, start, end) else: raise NotImplementedError( "data_source=%r is not implemented" % data_source)
def _get_data(symbol, start=None, end=None, retry_count=3, pause=0.001): """ Returns DataFrame of historical corporate actions (dividends and stock splits) from symbols, over date range, start to end. All dates in the resulting DataFrame correspond with dividend and stock split ex-dates. Parameters ---------- sym : string with a single Single stock symbol (ticker). start : string, (defaults to '1/1/2010') Starting date, timestamp. Parses many different kind of date representations (e.g., 'JAN-01-2010', '1/1/10', 'Jan, 1, 1980') end : string, (defaults to today) Ending date, timestamp. Same format as starting date. retry_count : int, default 3 Number of times to retry query request. pause : int, default 0 Time, in seconds, of the pause between retries. """ start, end = _sanitize_dates(start, end) params = { 's': symbol, 'a': start.month - 1, 'b': start.day, 'c': start.year, 'd': end.month - 1, 'e': end.day, 'f': end.year, 'g': 'v' } url = _encode_url(_URL, params) for _ in range(retry_count): try: with urlopen(url) as resp: lines = resp.read() except _network_error_classes: pass else: actions_index = [] actions_entries = [] for line in csv.reader(StringIO(bytes_to_str(lines))): # Ignore lines that aren't dividends or splits (Yahoo # add a bunch of irrelevant fields.) if len(line) != 3 or line[0] not in ('DIVIDEND', 'SPLIT'): continue action, date, value = line if action == 'DIVIDEND': actions_index.append(to_datetime(date)) actions_entries.append({ 'action': action, 'value': float(value) }) elif action == 'SPLIT' and ':' in value: # Convert the split ratio to a fraction. For example a # 4:1 split expressed as a fraction is 1/4 = 0.25. denominator, numerator = value.split(':', 1) split_fraction = float(numerator) / float(denominator) actions_index.append(to_datetime(date)) actions_entries.append({ 'action': action, 'value': split_fraction }) return DataFrame(actions_entries, index=actions_index) time.sleep(pause) raise IOError("after %d tries, Yahoo! did not " \ "return a 200 for url %r" % (retry_count, url))
def DataReader(name, data_source=None, start=None, end=None, retry_count=3, pause=0.001): """ Imports data from a number of online sources. Currently supports Yahoo! Finance, Google Finance, St. Louis FED (FRED) and Kenneth French's data library. Parameters ---------- name : str or list of strs the name of the dataset. Some data sources (yahoo, google, fred) will accept a list of names. data_source: {str, None} the data source ("yahoo", "yahoo-actions", "google", "fred", or "ff") start : {datetime, None} left boundary for range (defaults to 1/1/2010) end : {datetime, None} right boundary for range (defaults to today) retry_count : {int, 3} Number of times to retry query request. pause : {numeric, 0.001} Time, in seconds, to pause between consecutive queries of chunks. If single value given for symbol, represents the pause between retries. Examples ---------- # Data from Yahoo! Finance gs = DataReader("GS", "yahoo") # Corporate Actions (Dividend and Split Data) with ex-dates from Yahoo! Finance gs = DataReader("GS", "yahoo-actions") # Data from Google Finance aapl = DataReader("AAPL", "google") # Data from FRED vix = DataReader("VIXCLS", "fred") # Data from Fama/French ff = DataReader("F-F_Research_Data_Factors", "famafrench") ff = DataReader("F-F_Research_Data_Factors_weekly", "famafrench") ff = DataReader("6_Portfolios_2x3", "famafrench") ff = DataReader("F-F_ST_Reversal_Factor", "famafrench") """ start, end = _sanitize_dates(start, end) if data_source == "yahoo": return get_data_yahoo(symbols=name, start=start, end=end, adjust_price=False, chunksize=25, retry_count=retry_count, pause=pause) elif data_source == "yahoo-actions": return get_data_yahoo_actions(symbol=name, start=start, end=end, retry_count=retry_count, pause=pause) elif data_source == "google": return get_data_google(symbols=name, start=start, end=end, chunksize=25, retry_count=retry_count, pause=pause) elif data_source == "fred": return get_data_fred(name, start, end) elif data_source == "famafrench": return get_data_famafrench(name) elif data_source == "oecd": return get_data_oecd(name, start, end) else: raise NotImplementedError("data_source=%r is not implemented" % data_source)