def interpolate(x: pd.Series, dates: Union[List[date], List[time], pd.Series] = None, method: Interpolate = Interpolate.INTERSECT) -> pd.Series: """ Interpolate over specified dates or times Stepwize interpolation of series based on dates in second series: >>> a = generate_series(100) >>> b = generate_series(100) >>> interpolate(a, b, Interpolate.INTERSECT) """ if dates is None: dates = x if isinstance(dates, pd.Series): align_series = dates else: align_series = pd.Series(np.nan, dates) if method == Interpolate.INTERSECT: # Only returns a value for valid dates return x.align(align_series, 'inner')[0] if method == Interpolate.NAN: # Value will be NaN for dates not present in the series return x.align(align_series, 'right')[0] if method == Interpolate.ZERO: # Value will be zero for dates not present in the series align_series = pd.Series(0.0, dates) return x.align(align_series, 'right', fill_value=0)[0] if method == Interpolate.STEP: # Value of the previous valid point return __interpolate_step(x, align_series) else: raise MqValueError('Unknown intersection type: ' + method)
def test_align_multiindex(): # GH 10665 midx = pd.MultiIndex.from_product( [range(2), range(3), range(2)], names=("a", "b", "c")) idx = pd.Index(range(2), name="b") s1 = Series(np.arange(12, dtype="int64"), index=midx) s2 = Series(np.arange(2, dtype="int64"), index=idx) # these must be the same results (but flipped) res1l, res1r = s1.align(s2, join="left") res2l, res2r = s2.align(s1, join="right") expl = s1 tm.assert_series_equal(expl, res1l) tm.assert_series_equal(expl, res2r) expr = Series([0, 0, 1, 1, np.nan, np.nan] * 2, index=midx) tm.assert_series_equal(expr, res1r) tm.assert_series_equal(expr, res2l) res1l, res1r = s1.align(s2, join="right") res2l, res2r = s2.align(s1, join="left") exp_idx = pd.MultiIndex.from_product( [range(2), range(2), range(2)], names=("a", "b", "c")) expl = Series([0, 1, 2, 3, 6, 7, 8, 9], index=exp_idx) tm.assert_series_equal(expl, res1l) tm.assert_series_equal(expl, res2r) expr = Series([0, 0, 1, 1] * 2, index=exp_idx) tm.assert_series_equal(expr, res1r) tm.assert_series_equal(expr, res2l)
class Align(object): def setup(self): size = 5 * 10**5 rng = np.arange(0, 10**13, 10**7) stamps = np.datetime64('now').view('i8') + rng idx1 = np.sort(np.random.choice(stamps, size, replace=False)) idx2 = np.sort(np.random.choice(stamps, size, replace=False)) self.ts1 = Series(np.random.randn(size), idx1) self.ts2 = Series(np.random.randn(size), idx2) def time_series_align_int64_index(self): self.ts1 + self.ts2 def time_series_align_left_monotonic(self): self.ts1.align(self.ts2, join='left')
def test_subtracting_two_series_with_unordered_index_and_all_nan_index( self, data_result, data_expected ): # GH 38439 a_index_result = MultiIndex.from_tuples(data_result[0]) b_index_result = MultiIndex.from_tuples(data_result[1]) a_series_result = Series(data_result[2], index=a_index_result) b_series_result = Series(data_result[3], index=b_index_result) result = a_series_result.align(b_series_result) a_index_expected = MultiIndex.from_tuples(data_expected[0]) b_index_expected = MultiIndex.from_tuples(data_expected[1]) a_series_expected = Series(data_expected[2], index=a_index_expected) b_series_expected = Series(data_expected[3], index=b_index_expected) a_series_expected.index = a_series_expected.index.set_levels( [ a_series_expected.index.levels[0].astype("float"), a_series_expected.index.levels[1].astype("float"), ] ) b_series_expected.index = b_series_expected.index.set_levels( [ b_series_expected.index.levels[0].astype("float"), b_series_expected.index.levels[1].astype("float"), ] ) tm.assert_series_equal(result[0], a_series_expected) tm.assert_series_equal(result[1], b_series_expected)
class Align(object): def setup(self): size = 5 * 10**5 rng = np.arange(0, 10**13, 10**7) stamps = np.datetime64('now').view('i8') + rng idx1 = np.sort(np.random.choice(stamps, size, replace=False)) idx2 = np.sort(np.random.choice(stamps, size, replace=False)) self.ts1 = Series(np.random.randn(size), idx1) self.ts2 = Series(np.random.randn(size), idx2) def time_series_align_int64_index(self): self.ts1 + self.ts2 def time_series_align_left_monotonic(self): self.ts1.align(self.ts2, join='left')
def test_align_with_dataframe_method(method): # GH31788 ser = Series(range(3), index=range(3)) df = pd.DataFrame(0.0, index=range(3), columns=range(3)) result_ser, result_df = ser.align(df, method=method) tm.assert_series_equal(result_ser, ser) tm.assert_frame_equal(result_df, df)
def test_align_series(self): rng = period_range('1/1/2000', '1/1/2010', freq='A') ts = Series(np.random.randn(len(rng)), index=rng) result = ts + ts[::2] expected = ts + ts expected[1::2] = np.nan tm.assert_series_equal(result, expected) result = ts + _permute(ts[::2]) tm.assert_series_equal(result, expected) # it works! for kind in ['inner', 'outer', 'left', 'right']: ts.align(ts[::2], join=kind) msg = "Input has different freq=D from PeriodIndex\\(freq=A-DEC\\)" with tm.assert_raises_regex(period.IncompatibleFrequency, msg): ts + ts.asfreq('D', how="end")
def test_align_series(self): rng = period_range('1/1/2000', '1/1/2010', freq='A') ts = Series(np.random.randn(len(rng)), index=rng) result = ts + ts[::2] expected = ts + ts expected[1::2] = np.nan tm.assert_series_equal(result, expected) result = ts + _permute(ts[::2]) tm.assert_series_equal(result, expected) # it works! for kind in ['inner', 'outer', 'left', 'right']: ts.align(ts[::2], join=kind) msg = "Input has different freq=D from PeriodIndex\\(freq=A-DEC\\)" with tm.assert_raises_regex(period.IncompatibleFrequency, msg): ts + ts.asfreq('D', how="end")
def test_series_align_aware(self): idx1 = date_range('2001', periods=5, freq='H', tz='US/Eastern') ser = Series(np.random.randn(len(idx1)), index=idx1) ser_central = ser.tz_convert('US/Central') # # different timezones convert to UTC new1, new2 = ser.align(ser_central) assert new1.index.tz == pytz.UTC assert new2.index.tz == pytz.UTC
def test_series_align_aware(self): idx1 = date_range("2001", periods=5, freq="H", tz="US/Eastern") ser = Series(np.random.randn(len(idx1)), index=idx1) ser_central = ser.tz_convert("US/Central") # # different timezones convert to UTC new1, new2 = ser.align(ser_central) assert new1.index.tz == pytz.UTC assert new2.index.tz == pytz.UTC
def test_align_left_fewer_levels(): # GH#45224 left = Series([2], index=pd.MultiIndex.from_tuples([(1, 3)], names=["a", "c"])) right = Series([1], index=pd.MultiIndex.from_tuples([(1, 2, 3)], names=["a", "b", "c"])) result_left, result_right = left.align(right) expected_right = Series([1], index=pd.MultiIndex.from_tuples( [(1, 3, 2)], names=["a", "c", "b"])) expected_left = Series([2], index=pd.MultiIndex.from_tuples( [(1, 3, 2)], names=["a", "c", "b"])) tm.assert_series_equal(result_left, expected_left) tm.assert_series_equal(result_right, expected_right)
def test_str_cat_align_indexed(index_or_series, join): # https://github.com/pandas-dev/pandas/issues/18657 box = index_or_series s = Series(["a", "b", "c", "d"], index=["a", "b", "c", "d"]) t = Series(["D", "A", "E", "B"], index=["d", "a", "e", "b"]) sa, ta = s.align(t, join=join) # result after manual alignment of inputs expected = sa.str.cat(ta, na_rep="-") if box == Index: s = Index(s) sa = Index(sa) expected = Index(expected) result = s.str.cat(t, join=join, na_rep="-") assert_series_or_index_equal(result, expected)
def align_method_SERIES(left: Series, right, align_asobject: bool = False): """ align lhs and rhs Series """ # ToDo: Different from align_method_FRAME, list, tuple and ndarray # are not coerced here # because Series has inconsistencies described in #13637 if isinstance(right, ABCSeries): # avoid repeated alignment if not left.index.equals(right.index): if align_asobject: # to keep original value's dtype for bool ops left = left.astype(object) right = right.astype(object) left, right = left.align(right, copy=False) return left, right
def __interpolate_step(x: pd.Series, dates: pd.Series = None) -> pd.Series: if x.empty: raise MqValueError('Cannot perform step interpolation on an empty series') first_date = pd.Timestamp(dates.index[0]) if isinstance(x.index[0], pd.Timestamp) else dates.index[0] # locate previous valid date or take first value from series prev = x.index[0] if first_date < x.index[0] else x.index[x.index.get_loc(first_date, 'pad')] current = x[prev] curve = x.align(dates, 'right', )[0] # only need values from dates for knot in curve.iteritems(): if np.isnan(knot[1]): curve[knot[0]] = current else: current = knot[1] return curve
def test_align_series_combinations(self): df = DataFrame({"a": [1, 3, 5], "b": [1, 3, 5]}, index=list("ACE")) s = Series([1, 2, 4], index=list("ABD"), name="x") # frame + series res1, res2 = df.align(s, axis=0) exp1 = DataFrame( { "a": [1, np.nan, 3, np.nan, 5], "b": [1, np.nan, 3, np.nan, 5] }, index=list("ABCDE"), ) exp2 = Series([1, 2, np.nan, 4, np.nan], index=list("ABCDE"), name="x") tm.assert_frame_equal(res1, exp1) tm.assert_series_equal(res2, exp2) # series + frame res1, res2 = s.align(df) tm.assert_series_equal(res1, exp2) tm.assert_frame_equal(res2, exp1)
def pairs_selection_test(price1: pd.Series, price2: pd.Series, max_lag: int, convenient_periods: int) -> bool: """ Parameters ---------- price1 : pd.Series Price series of 1st security. price2 : pd.Series Price series of 2nd security. max_lag: int convenient_periods: int Specify the trading period. This will be use to filter out pairs whereby its half-life is not coherent with the trading period. Returns ------- bool whether the pairs pass the selection test. """ price1.replace([np.inf, -np.inf], np.nan, inplace=True) price1.dropna(inplace=True) price2.replace([np.inf, -np.inf], np.nan, inplace=True) price2.dropna(inplace=True) price1, price2 = price1.align(price2, join='inner') # Cointegrated pairs; Propose that the Engle-Granger test is run for the # two possible selections of the dependent variable and that the combination # that generated the lowest t-statistic is selected. _, pval1, _ = coint(price1, price2) _, pval2, _ = coint(price2, price1) pval = min(pval1, pval2) if pval >= .01: return False spread = np.log(price1 / price2) # Mean-reverting Hurst exponent; It aims to constrain false positives, # possibly arising as an effect of the multiple comparisons problem. The # condition imposed is that the Hurst exponent associated with the spread # of a given pair is enforced to be smaller than 0.5, assuring the process # leans towards mean-reversion. lags = range(2, max_lag) tau = [ np.sqrt(np.std(np.subtract(spread[lag:], spread[:-lag]))) for lag in lags ] poly = np.polyfit(np.log(lags), np.log(tau), 1) hurst_exponent = poly[0] * 2 if hurst_exponent >= 0.5: return False # Suitable half-life; the value of theta which is obtained by running a # linear regression on the difference between mean of spread and spread, # and the difference between tomorrow's value of spread and today's value model = sm.OLS((np.mean(spread) - spread).iloc[:-1], (spread.shift(-1) - spread).iloc[:-1]) results = model.fit() half_life = -np.log(2) / results.params[0] if half_life > convenient_periods: return False # Monthly Mean Crossing; Enforce that every spread crosses its mean at least # once per month, to provide enough liquidity. spread_mu = spread.resample('MS').transform('mean') delta_sign = (np.sign(spread - spread_mu).diff().dropna() != 0).astype(int) num_of_cross_per_year = delta_sign.resample('Y').sum() num_of_year = len(num_of_cross_per_year) if (num_of_cross_per_year >= 12).sum() / num_of_year < 1: return False return True
def test_align_broadcast_axis(self): # GH 13194 # First four tests for DataFrame.align(Index) # For 'right' join df = DataFrame(np.array([[1., 2.], [3., 4.]]), columns=list('AB')) ts = Series([5., 6., 7.]) result = df.align(ts, join='right', axis=0, broadcast_axis=1) expected1 = DataFrame(np.array([[1., 2.], [3., 4.], [pd.np.nan, pd.np.nan]]), columns=list('AB')) expected2 = DataFrame(np.array([[5., 5.], [6., 6.], [7., 7.]]), columns=list('AB')) assert_frame_equal(result[0], expected1) assert_frame_equal(result[1], expected2) # For 'right' join on different index result = df.align(ts, join='right', axis=1, broadcast_axis=1) expected1 = DataFrame(np.array([[1., 2.], [3., 4.]]), columns=list('AB')) expected2 = DataFrame(np.array([[5., 5.], [6., 6.], [7., 7.]]), columns=list('AB')) assert_frame_equal(result[0], expected1) assert_frame_equal(result[1], expected2) # For 'left' join result = df.align(ts, join='left', axis=0, broadcast_axis=1) expected1 = DataFrame(np.array([[1., 2.], [3., 4.]]), columns=list('AB')) expected2 = DataFrame(np.array([[5., 5.], [6., 6.]]), columns=list('AB')) assert_frame_equal(result[0], expected1) assert_frame_equal(result[1], expected2) # For 'left' join on different axis result = df.align(ts, join='left', axis=1, broadcast_axis=1) expected1 = DataFrame(np.array([[1., 2.], [3., 4.]]), columns=list('AB')) expected2 = DataFrame(np.array([[5., 5.], [6., 6.], [7., 7.]]), columns=list('AB')) assert_frame_equal(result[0], expected1) assert_frame_equal(result[1], expected2) # Series.align(DataFrame) tests, 'outer' join result = ts.align(df, join='outer', axis=0, broadcast_axis=1) expected1 = DataFrame(np.array([[5., 5.], [6., 6.], [7., 7.]]), columns=list('AB')) expected2 = DataFrame(np.array([[1., 2.], [3., 4.], [pd.np.nan, pd.np.nan]]), columns=list('AB')) assert_frame_equal(result[0], expected1) assert_frame_equal(result[1], expected2) # Series.align(DataFrame) tests, 'inner' join result = ts.align(df, join='inner', axis=0, broadcast_axis=1) expected1 = DataFrame(np.array([[5., 5.], [6., 6.]]), columns=list('AB')) expected2 = DataFrame(np.array([[1., 2.], [3., 4.]]), columns=list('AB')) assert_frame_equal(result[0], expected1) assert_frame_equal(result[1], expected2)
def interpolate(x: pd.Series, dates: Union[List[date], List[time], pd.Series] = None, method: Interpolate = Interpolate.INTERSECT) -> pd.Series: """ Interpolate over specified dates or times :param x: timeseries to interpolate :param dates: array of dates/times or another series to interpolate :param method: interpolation method (default: intersect) :return: timeseries with specified dates **Usage** Interpolate the series X over the dates specified by the dates parameter. This can be an array of dates or another series, in which case the index of the series will be used to specify dates Interpolation methods: ========= ======================================================================== Type Behavior ========= ======================================================================== intersect Resultant series only has values on the intersection of dates /times. Will only contain intersection of valid dates / times in the series nan Resultant series only has values on the intersection of dates /times. Value will be NaN for dates not present in the series zero Resultant series has values on all requested dates / times. The series will have a value of zero where the requested date or time was not present in the series step Resultant series has values on all requested dates / times. The series will use the value of the previous valid point if requested date does not exist. Values prior to the first date will be equivalent to the first available value ========= ======================================================================== **Examples** Stepwize interpolation of series based on dates in second series: >>> a = generate_series(100) >>> b = generate_series(100) >>> interpolate(a, b, Interpolate.INTERSECT) **See also** :func:`sub` """ if dates is None: dates = x if isinstance(dates, pd.Series): align_series = dates else: align_series = pd.Series(np.nan, dates) if method == Interpolate.INTERSECT: return x.align(align_series, 'inner')[0] if method == Interpolate.NAN: return x.align(align_series, 'right')[0] if method == Interpolate.ZERO: align_series = pd.Series(0.0, dates) return x.align(align_series, 'right', fill_value=0)[0] if method == Interpolate.STEP: return __interpolate_step(x, align_series) else: raise MqValueError('Unknown intersection type: ' + method)
def test_align_series(self, join_type): rng = period_range('1/1/2000', '1/1/2010', freq='A') ts = Series(np.random.randn(len(rng)), index=rng) ts.align(ts[::2], join=join_type)
def test_align_series(self, join_type): rng = period_range('1/1/2000', '1/1/2010', freq='A') ts = Series(np.random.randn(len(rng)), index=rng) ts.align(ts[::2], join=join_type)
# In[11]: s1 = s[:4] s1 # In[10]: s2 = s[1:] s2 # In[12]: s1.align(s2) # In[13]: df.align(df2, join ='inner') # In[ ]: # In[ ]: #filter and column selection in single statement
def test_align_periodindex(join_type): rng = period_range("1/1/2000", "1/1/2010", freq="A") ts = Series(np.random.randn(len(rng)), index=rng) # TODO: assert something? ts.align(ts[::2], join=join_type)