def add_block(entity_scope: str, entity_code: str, bs: InMemoryBlockStore, sd, ed, ad) -> None: """ Adds a block to a block store :param str entity_scope: The scope of the entity to add :param str entity_code: The code of the entity to add :param InMemoryBlockStore bs: The block store to add the block to :param sd: The effectiveAt start date :param ed: The effectiveAt end date :param ad: The asAt date :return: None """ # Create a block using the provided dates b = PerformanceDataSet(from_date=sd, to_date=ed, asat=ad) global src # For each date and group (DataFrame) for d, g in src.get_perf_data( entity_scope=entity_scope, entity_code=entity_code, from_date=b.from_date, to_date=b.to_date, asat=b.asat ).groupby('date'): # Populate the block with each PerformanceDataPoint in chronological order b.add_values(date=d, data_source=g.apply(lambda r: (r['key'], r['mv'], r['net']), axis=1)) # Add the populated block to the block store bs.add_block(entity_scope=entity_scope, entity_code=entity_code, block=b)
def test_two_periods(fund, expected_cum_ror, expected_correction): bs = InMemoryBlockStore() rs = ReturnSource( dataset='Ret1', portfolio=fund, filename=Path(__file__).parent.parent.joinpath("test-data.xlsx")) Returns(bs).import_data(test_scope, fund, rs, '2020-01-01', '2020-01-11', '2020-01-11').import_data(test_scope, fund, rs, '2020-01-08', '2020-01-15', '2020-01-15') def check_returns(locked): df = pd.DataFrame.from_records( Performance(test_scope, fund, None, bs).report(locked, '2020-01-01', '2020-01-15', '2020-01-15', fields=[DAY, WTD])) assert len(df) == 15 cum_ror = df['inception'].iloc[-1] correction = df['correction'].iloc[-4] assert cum_ror == pytest.approx(expected_cum_ror, abs=0.00005) assert correction == pytest.approx( expected_correction if locked else 0.0, abs=0.00005) # Check the returns in locked mode check_returns(True) # And, using the same block-store, run in 'true' mode check_returns(False)
def test_get_change_parameters_are_as_expected(scenario): # These will test that the logic for unlocked queries is correct global testcases global src scenario = testcases[scenario] prf = Performance(entity_scope="Test", entity_code="GetChangesParams", src=src, block_store=InMemoryBlockStore()) # Create an initial block prf.get_performance(locked=True, start_date='2019-12-31', end_date='2020-01-05', asat='2020-01-06', create=True) with pytest.raises(ChangeCalled) as cc: prf.get_performance(False, *scenario[0]) for actual, expected, field in zip( cc.value.args[2:], scenario[1], ['last_eff_date', 'last_asat_date', 'required_asat_date']): assert actual == expected, field
def test_get_changes_not_called_if_block_exists(): global src # Create an instance of the Performance class for generating reports prf = Performance(entity_scope="test", entity_code="get_changes", src=src, block_store=InMemoryBlockStore()) # Create a block - as an initial block, get_changes will not be called. No exception raised. p = list( prf.get_performance(locked=True, start_date='2019-12-31', end_date='2020-01-05', asat='2020-01-05', create=True)) assert len(p) == 6 # Subsequent calls can reuse the original block - no exception should be thrown # Get performance for the exact same period as the original block p = list( prf.get_performance(locked=True, start_date='2019-12-31', end_date='2020-01-05', asat='2020-01-05')) assert len(p) == 6 # Get performance for a subset of the original block with the start trimmed p = list( prf.get_performance(locked=True, start_date='2020-01-03', end_date='2020-01-05', asat='2020-01-05')) assert len(p) == 3 # Get performance for a subset of the original block with the end trimmed p = list( prf.get_performance(locked=True, start_date='2019-12-31', end_date='2020-01-04', asat='2020-01-05')) assert len(p) == 5 # Get performance for a subset of the original block with both the ends trimmed p = list( prf.get_performance(locked=True, start_date='2020-01-03', end_date='2020-01-04', asat='2020-01-05')) assert len(p) == 2
def test_perf_start_date_is_honoured(): prf = list( Performance(entity_scope="test", entity_code="start_date", src=src1, block_store=InMemoryBlockStore(), perf_start='2019-12-31').report(False, '2020-01-03', '2020-01-05', '2020-01-10', fields=[DAY])) cum_ror = prf[-1]['inception'] assert cum_ror == pytest.approx(0.187343)
def test_5yr_vol(recording): vol_fields = [ DAY, VOL_INC, VOL_1YR, VOL_3YR, VOL_5YR, ANN_VOL_3YR, ANN_VOL_5YR, ANN_VOL_INC ] src = SeededSource() src.add_seeded_perf_data("test", "test_5yr_vol", "2014-12-31", 24106) df = pd.DataFrame.from_records( Performance(entity_scope="test", entity_code="test_5yr_vol", src=src, block_store=InMemoryBlockStore()).report( False, '2013-12-31', '2020-03-05', '2020-03-05', fields=vol_fields))[['date', 'mv'] + vol_fields] # Test the boundaries of each year # This is where the different period volatilities # Will diverge from the inception volatility subset = df[df['date'].isin( set( dates('2014-12-31', '2015-01-01', '2015-12-31', '2016-01-01', '2016-12-31', '2017-01-01', '2017-12-31', '2018-01-01', '2018-12-31', '2019-01-01', '2019-12-31', '2020-01-01', '2020-03-05')))] # To record tests, use pytest --recording=test_5yr_vol filename = Path(__file__).parent.joinpath('expected', '5yr_vol.pk') if 'test_5yr_vol' in recording: # Record the result and save as the expectation subset.to_pickle(filename, protocol=0) # Also save a temporary csv version for manual review nicer(df).to_csv('5yr_vol.csv', index=False) else: # Load expected values and compare with the generated data target = pd.read_pickle(filename) # Compare with the expectation pd.testing.assert_frame_equal(subset, target)
def test_change_parameters_not_called_when_inside_locked_period(): prf = Performance(entity_scope="Test", entity_code="GetChangesParams", src=src, block_store=InMemoryBlockStore()) # Create an initial block prf.get_performance(True, '2019-12-31', '2020-01-05', '2020-01-06', create=True) # Query locked performance within the boundary r = prf.get_performance(True, '2020-01-02', '2020-01-03', '2020-01-10') # No exception thrown means we will get here assert r is not None
def test_sharpe_ratios(recording): fields = [ DAY, AGE_DAYS, ROLL_YEAR, ROLL_3YR, ROLL_5YR, ANN_INC, ANN_1YR, ANN_3YR, ANN_5YR, VOL_INC, VOL_1YR, VOL_3YR, VOL_5YR, ANN_VOL_INC, ANN_VOL_1YR, ANN_VOL_3YR, ANN_VOL_5YR, RISK_FREE_1YR, RISK_FREE_3YR, RISK_FREE_5YR, SHARPE_1YR, SHARPE_3YR, SHARPE_5YR ] src = SeededSource(rfr_func=risk_free_rates) src.add_seeded_perf_data("test", "sharpe_ratio", '2013-12-31', 24106) df = pd.DataFrame.from_records( Performance(entity_scope="test", entity_code="sharpe_ratio", src=src, block_store=InMemoryBlockStore()).report( locked=False, start_date='2013-12-31', end_date='2020-03-05', asat='2020-03-05', fields=fields))[['date', 'mv', 'inception'] + fields] # Test cases subset = df # To record tests, use pytest --recording=test_sharpe filename = Path(__file__).parent.joinpath('expected', 'sharpe_ratios.pk') if 'test_sharpe_ratios' in recording: # Record the result and save as the expectation subset.to_pickle(filename, protocol=0) # Also save a temporary csv version for manual review nicer(df).to_csv('sharpe_ratios.csv', index=False) else: # Load expected values and compare with the generated data target = pd.read_pickle(filename) # Compare with the expectation pd.testing.assert_frame_equal(subset, target)
def test_cumulative_returns(fund, expected): bs = InMemoryBlockStore() portfolio_code = str(uuid.uuid4()) Returns(bs).import_data( test_scope, portfolio_code, ReturnSource( dataset='Ret1', portfolio=fund, filename=Path(__file__).parent.parent.joinpath("test-data.xlsx")), '2020-01-01', '2020-01-11', '2020-01-11') df = pd.DataFrame.from_records( Performance(test_scope, portfolio_code, None, bs).report(False, '2020-01-11', '2020-01-11', '2020-01-11')) assert len(df) == 1 cum_ror = df['inception'].iloc[-1] assert cum_ror == pytest.approx(expected, abs=0.00005)
def test_change_parameters_called_when_outside_locked_period(): global src prf = Performance(entity_scope="Test", entity_code="GetChangesParams", src=src, block_store=InMemoryBlockStore()) # Create an initial block prf.get_performance(True, '2019-12-31', '2020-01-05', '2020-01-06', create=True) # Query locked performance, crossing the boundary with pytest.raises(ChangeCalled) as cc: prf.get_performance(True, '2020-01-02', '2020-01-10', '2020-01-10') assert cc.value.args[2] == as_date('2020-01-05') assert cc.value.args[3] == as_date('2020-01-06') assert cc.value.args[4] == as_date('2020-01-10')
def create_demo_sheet(): fields = [ AGE_DAYS, DAY, WTD, MTD, QTD, YTD, ROLL_WEEK, ROLL_MONTH, ROLL_QTR, ROLL_YEAR, ROLL_3YR, ROLL_5YR, ANN_INC, ANN_1YR, ANN_3YR, ANN_5YR, VOL_INC, VOL_1YR, VOL_3YR, VOL_5YR, ANN_VOL_INC, ANN_VOL_1YR, ANN_VOL_3YR, ANN_VOL_5YR, RISK_FREE_1YR, RISK_FREE_3YR, RISK_FREE_5YR, SHARPE_1YR, SHARPE_3YR, SHARPE_5YR ] df = pd.DataFrame.from_records( Performance( mock_src.SeededSource('2013-12-31', 24106, rfr_func=risk_free_rates, max=0.02, trend=0.0005), InMemoryBlockStore()).report( False, '2013-12-31', '2020-03-05', '2020-03-05', fields=fields))[['date', AGE_DAYS, 'mv', 'inception'] + fields[1:]] nicer(df).to_csv('demo.csv', index=False)
def test_scenario(locked, expected): with api_cacher.CachingApi("scenario1") as api: prf = Performance('JLH', 'FUND1', LusidSource(api, config), InMemoryBlockStore()) # Lock the May period prf.get_performance(True, "2019-05-01", "2019-05-31", "2020-03-25T09:00", create=True) # Lock the June period prf.get_performance(True, "2019-05-31", "2019-06-30", "2020-03-25T10:00", create=True) # Lock the July period prf.get_performance(True, "2019-06-30", "2019-07-31", "2020-03-25T11:00", create=True) fields = [DAY, WTD] df = pd.DataFrame.from_records( prf.report(locked, '2019-05-01', '2019-07-31', '2020-03-25T11:00', fields=fields) )[[ 'date', 'mv', 'inception', 'flows', 'correction', 'flow_correction' ] + fields] assert len(df) == 92 assert df.tail(1).iloc[0]['inception'] == pytest.approx(0.002099104) actual = df['correction'].sum() assert actual == pytest.approx(expected, abs=0.000005)
def create_composite(composite_code, date_range_1, date_range_2, date_range_3, **kwargs): nonlocal performance_api composite.create_composite(composite_scope=test_scope, composite_code=composite_code) composite.add_composite_member(composite_scope=test_scope, composite_code=composite_code, member_scope=test_scope, member_code="P1", from_date=date_range_1[0], to_date=date_range_1[1]) composite.add_composite_member(composite_scope=test_scope, composite_code=composite_code, member_scope=test_scope, member_code="P2", from_date=date_range_2[0], to_date=date_range_2[1]) composite.add_composite_member(composite_scope=test_scope, composite_code=composite_code, member_scope=test_scope, member_code="P3", from_date=date_range_3[0], to_date=date_range_3[1]) cs = CompositeSource(composite=composite, performance_api=performance_api, **kwargs) return Performance(test_scope, composite_code, cs, InMemoryBlockStore(), perf_start='2018-03-18')
def test_volatility_example(): # Test based upon the example here: # http://invest-made-easy.blogspot.com/2013/03/understanding-volatility-and-sharpe.html # The calculated standard deviation of the monthly returns should be 2.25% vol_fields = [DAY, VOL_INC, ANN_VOL_INC] df = pd.DataFrame.from_records( Performance(entity_scope="test", entity_code="test_5yr_vol", src=mock_src.MockSource( 'VolEx1', filename=Path(__file__).parent.parent.joinpath( "test-data.xlsx")), block_store=InMemoryBlockStore()).report( False, '2019-12-31', '2020-01-11', '2020-01-11', fields=vol_fields))[['date', 'mv'] + vol_fields] volatility = df[VOL_INC].values[-1] assert volatility == pytest.approx(0.0225, abs=0.00005)
standard_fields = ['date', 'key', 'mv', 'flows', 'inception', 'correction'] field_set = [ DAY, WTD, MTD, QTD, YTD, ROLL_WEEK, ROLL_MONTH, ROLL_YEAR, ROLL_QTR ] # Source for Set1 - used in env1 and env2 src1 = mock_src.MockSource( 'Set1', filename=Path(__file__).parent.parent.joinpath("test-data.xlsx")) src1.get_changes = lambda v, w, x, y, z: as_date('2020-01-08' ) # only needed once # Set up environment #1 - no performance posted env1 = Performance(entity_scope="test", entity_code="env1", src=src1, block_store=InMemoryBlockStore()) # No blocks # Set up environment #2 - locked period, and back-dated changes env2 = Performance(entity_scope="test", entity_code="env2", src=src1, block_store=InMemoryBlockStore()) # Set up environment #3 - 2 years data, and 1 b.p return/day (easy to verify) env3 = Performance(entity_scope="test", entity_code="env3", src=mock_src.SimpleSource('2018-03-05'), block_store=InMemoryBlockStore()) # Set up environment #4 - 2 years data, and 1 b.p return/day, with a weekly flow env4 = Performance(entity_scope="test", entity_code="env4", src=mock_src.SimpleSource('2018-03-05', recurring_flow=300.0),
def test_combinations(locked): bs = InMemoryBlockStore() entity_code = str(uuid.uuid4()) # block 1 add_block(test_scope, entity_code, bs, '2019-12-31', '2020-01-05', '2020-01-05') # block 2 add_block(test_scope, entity_code, bs, '2020-01-06', '2020-01-10', '2020-01-10') # block 3 add_block(test_scope, entity_code, bs, '2020-01-11', '2020-01-11', '2020-01-11') # block 4 - Includes back-dated activity into block 2 add_block(test_scope, entity_code, bs, '2020-01-08', '2020-01-15', '2020-01-15') # block 5 Includes back-dated activity to first item of block 2 add_block(test_scope, entity_code, bs, '2020-01-06', '2020-01-15', '2020-02-01') # block 6 Includes back-dated activity to last item of block 1 add_block(test_scope, entity_code, bs, '2020-01-05', '2020-01-15', '2020-02-02') # block 7 Includes back-dated activity to exclude block 1 entirely add_block(test_scope, entity_code, bs, '2012-12-31', '2020-01-15', '2020-02-03') def run_scenario(entity_scope, entity_code, to_date, asat_date, expected_open_tmv, expected_locked_tmv): blocks = bs.find_blocks(entity_scope, entity_code, '2020-01-03', to_date, asat_date) df = pd.DataFrame.from_records([ (o.date, o.tmv) for o in combine(blocks, locked, '2020-01-03', to_date, asat_date) ], columns=['date', 'tmv']) total = df['tmv'].sum() expected = expected_locked_tmv if locked else expected_open_tmv if debug: print(nicer(df)) print(f"Expected: {expected}, Actual: {total}") assert (total == pytest.approx(expected, 0.001)) # View on 01/10 for 01/09 run_scenario(test_scope, entity_code, '2020-01-09', '2020-01-10', 6061.34, 6061.34) # View on 01/10 for 01/10 run_scenario(test_scope, entity_code, '2020-01-10', '2020-01-10', 7082.17, 7082.17) # View on 01/11 run_scenario(test_scope, entity_code, '2020-01-11', '2020-01-11', 7963.82, 7963.82) # View on 01/15 for 01/11 run_scenario(test_scope, entity_code, '2020-01-11', '2020-01-15', 8295.58, 7963.82) # View on 01/15 for 01/15 run_scenario(test_scope, entity_code, '2020-01-15', '2020-01-15', 12158.41, 11826.65) # View on 02/01 run_scenario(test_scope, entity_code, '2020-01-15', '2020-02-01', 2425.39, 11826.65) # View on 02/02 run_scenario(test_scope, entity_code, '2020-01-15', '2020-02-02', 1606.25, 11826.65) # View on 02/03 run_scenario(test_scope, entity_code, '2020-01-15', '2020-02-03', 39.0, 11826.65)
entity_code=entity_code, from_date=b.from_date, to_date=b.to_date, asat=b.asat ).groupby('date'): # Populate the block with each PerformanceDataPoint in chronological order b.add_values(date=d, data_source=g.apply(lambda r: (r['key'], r['mv'], r['net']), axis=1)) # Add the populated block to the block store bs.add_block(entity_scope=entity_scope, entity_code=entity_code, block=b) @pytest.mark.parametrize( "test_name, bs", [ ("InMemoryBlockStore", InMemoryBlockStore()), ("StructuredResultsBlockStore", BlockStoreStructuredResults(api_factory=api_factory)) ] ) def test_single_block(test_name: str, bs: IBlockStore) -> None: """ Test that find_blocks works as expected when there is a single block in the store :return: None """ entity_code = str(uuid.uuid4()) # Add a block to the block store add_block(test_scope, entity_code, bs, sd='2020-01-05', ed='2020-01-10', ad='2020-01-10') def pb(b: PerformanceDataSet) -> None:
def test_cumulative_ror(): entity_scope="test" entity_code="cumulative_ror" prf = Performance(entity_scope=entity_scope, entity_code=entity_code, src=src, block_store=InMemoryBlockStore()) def check_performance(expected_len, expected_ror, *args, **kwargs): perf_data = list(prf.get_performance(*args, **kwargs)) df = pd.DataFrame.from_records( [(o.date,o.tmv,o.ror,o.cum_fctr) for o in perf_data], columns=['date','mv','ror','fctr']) #print(nicer(df)) cum_ror = perf_data[-1].cum_fctr assert len(perf_data) == expected_len assert cum_ror == pytest.approx(expected_ror,0.00001) return df # Step 1 - Post performance up to 5th Jan - asat 5th Jan check_performance(6,1.187343,True,'2019-12-31','2020-01-05','2020-01-05',create=True) # Step 2 - Post performance up to 10th Jan - asat 10th Jan check_performance(11,1.458329,True,'2019-12-31','2020-01-10','2020-01-10',create=True) # Step 3 - Post performance up to 12th Jan - asat 15th Jan check_performance(13,1.293112,True,'2019-12-31','2020-01-12','2020-01-15',create=True) # Step 4 - Post performance up to 15th Jan - asat 16th Jan check_performance(16,1.646827,True,'2019-12-31','2020-01-15','2020-01-16',create=True) # Test 5 - Query actual performance up to 15th Jan - asat 16th Jan check_performance(16,1.646827,False,'2019-12-31','2020-01-15','2020-01-16') # Test 6 - Query actual performance from 3rd up to 9th Jan - asat 10th Jan check_performance(7,1.378071,False,'2020-01-03','2020-01-09','2020-01-10') # Test 7 - Query actual performance from 3rd up to 9th Jan - asat 15th Jan check_performance(7,1.370092,False,'2020-01-03','2020-01-09','2020-01-15') # Test 8 - Query locked performance from 3rd up to 9th Jan - asat 10th Jan check_performance(7,1.378071,True,'2020-01-03','2020-01-09','2020-01-10') # Test 9 - Query locked performance from 3rd up to 9th Jan - asat 15th Jan check_performance(7,1.378071,True,'2020-01-03','2020-01-09','2020-01-15') # Test 10 - Query locked performance up to 13th Jan - asat 3rd Feb check_performance(14,1.39204,True,'2019-12-15','2020-01-13','2020-02-03') # Test 11 - Query actual performance up to 13th Jan - asat 3rd Feb check_performance(14,1.0,False,'2019-12-15','2020-01-13','2020-02-03') # Test 11 - Query actual performance up to 13th Jan - asat 2nd Feb check_performance(14,0.00285714,False,'2019-12-15','2020-01-13','2020-02-02') # Test 11 - Query locked performance up to 13th Jan - asat 2nd Feb check_performance(14,1.39204,True,'2019-12-15','2020-01-13','2020-02-02') # The block store should only contain the 4 locked blocks assert len(prf.block_store.blocks[f"{entity_scope}_{entity_code}"]) == 4
def test_composite_methods(test_name, composite): in_memory_block_store = InMemoryBlockStore() seeded_source = SeededSource() seeded_source.add_seeded_perf_data(entity_scope=test_scope, entity_code="P1", start_date='2018-03-05', seed=24106) seeded_source.add_seeded_perf_data(entity_scope=test_scope, entity_code="P2", start_date='2018-03-05', seed=12345) seeded_source.add_seeded_perf_data(entity_scope=test_scope, entity_code="P3", start_date='2018-10-16', seed=33333) performance_api = PortfolioPerformanceApi( block_store=InMemoryBlockStore(), portfolio_performance_source=seeded_source) # utility to create composite from the three portfolios def create_composite(composite_code, date='2018-03-19', **kwargs): nonlocal performance_api composite.create_composite(composite_scope=test_scope, composite_code=composite_code) composite.add_composite_member(composite_scope=test_scope, composite_code=composite_code, member_scope=test_scope, member_code="P1", from_date='2018-03-05', to_date=None) composite.add_composite_member(composite_scope=test_scope, composite_code=composite_code, member_scope=test_scope, member_code="P2", from_date='2018-03-05', to_date=None) composite.add_composite_member(composite_scope=test_scope, composite_code=composite_code, member_scope=test_scope, member_code="P3", from_date='2018-03-05', to_date=None) cs = CompositeSource(composite=composite, performance_api=performance_api, **kwargs) return Performance(test_scope, composite_code, cs, InMemoryBlockStore(), perf_start='2018-03-18') # AND Composites, created using the different methods c1 = create_composite(str(uuid.uuid4())) c2 = create_composite(str(uuid.uuid4()), composite_mode="equal") c3 = create_composite(str(uuid.uuid4()), composite_mode="agg") p1 = Performance(test_scope, "P1", src=seeded_source, block_store=in_memory_block_store, perf_start='2018-03-05') p2 = Performance(test_scope, "P2", src=seeded_source, block_store=in_memory_block_store, perf_start='2018-03-05') p3 = Performance(test_scope, "P3", src=seeded_source, block_store=in_memory_block_store, perf_start='2018-10-16') def run_performance(p): return pd.DataFrame.from_records( p.report(locked=False, start_date='2018-03-05', end_date='2018-12-31', asat='2019-01-05'))[['date', 'inception', 'mv', 'wt']] # WHEN we calculate the performance for all the portfolios and the # composites # THEN the cumulative return on the final date should be as expected for df, exp_cum in zip( map(run_performance, [p1, p2, p3, c1, c2, c3]), [4.272095, 5.552946, 0.779682, 4.415115, 4.639544, 4.415115]): last = df.tail(1).iloc[0] assert last['inception'] == pytest.approx(exp_cum)