def add_block(entity_scope: str, entity_code: str, bs: InMemoryBlockStore, sd, ed, ad) -> None: """ Adds a block to a block store :param str entity_scope: The scope of the entity to add :param str entity_code: The code of the entity to add :param InMemoryBlockStore bs: The block store to add the block to :param sd: The effectiveAt start date :param ed: The effectiveAt end date :param ad: The asAt date :return: None """ # Create a block using the provided dates b = PerformanceDataSet(from_date=sd, to_date=ed, asat=ad) global src # For each date and group (DataFrame) for d, g in src.get_perf_data( entity_scope=entity_scope, entity_code=entity_code, from_date=b.from_date, to_date=b.to_date, asat=b.asat ).groupby('date'): # Populate the block with each PerformanceDataPoint in chronological order b.add_values(date=d, data_source=g.apply(lambda r: (r['key'], r['mv'], r['net']), axis=1)) # Add the populated block to the block store bs.add_block(entity_scope=entity_scope, entity_code=entity_code, block=b)
def test_lazy_loading(): # Keep track of how many times the loader is called. counter = 0 def dummy_loader(): nonlocal counter counter += 1 return [] pds = PerformanceDataSet('2018-03-05', '2018-03-19', '2018-03-19', loader=dummy_loader) # Before we get the data points, counter should be 0 assert counter == 0 r = pds.get_data_points() # After get the data points, counter should be 1 assert counter == 1 r2 = pds.get_data_points() # If we get the points again, the counter is still 1 assert counter == 1 # and it is the same object we got earlier assert r is r2
def __init__(self, scope, portfolio): super().__init__() self.scope = scope self.portfolio = portfolio self.path = os.path.join( PerformanceConfiguration.item('LocalStorePath', 'blocks'), scope, portfolio) # Load existing blocks (if any) # Create a loader function for a block def wrap(idx): def loader(): return pd.read_pickle(f'{self.path}.block-{idx+1}') return loader try: df = pd.read_csv(f'{self.path}.idx', parse_dates=['from_date', 'to_date', 'asat']) except: return # File doesn't exist. Not a problem at this stage for i, r in df.iterrows(): block = PerformanceDataSet(r['from_date'], r['to_date'], r['asat'], loader=wrap(i)) super().add_block(self.scope, self.portfolio, block)
def add_block(self, entity_scope: str, entity_code: str, block: PerformanceDataSet, performance_scope: str = None) -> PerformanceDataSet: """ This adds a block to the BlockStore for the specified entity. :param str entity_scope: The scope of the entity to add the block for. :param str entity_code: The code of the entity to get blocks for. Together with the entity_scope this uniquely identifies the entity. :param PerformanceDataSet block: The block to add to the BlockStore :param str performance_scope: The scope of the BlockStore to use, this is the scope in LUSID to use when adding the block to the Structured Result Store :return: PerformanceDataSet block: The block that was added to the BlockStore along with the asAt time of the operation """ if performance_scope is None: performance_scope = "PerformanceBlockStore" serialised_block = serialise(block) code = self._create_result_id(entity_scope, entity_code, block.from_date, block.to_date) effective_at = block.to_date structured_results_api = StructuredResultDataApi( self.api_factory.build(StructuredResultDataApi)) response = structured_results_api.upsert_structured_result_data( scope=performance_scope, request_body={ code: UpsertStructuredResultDataRequest( id=StructuredResultDataId(source=self.source, code=code, effective_at=effective_at, result_type=self.result_type), data=StructuredResultData(document_format="Json", version=block.version, name="PerformanceDataSet", document=serialised_block)) }) as_at_time = list(response.values.values())[0] entity_id = self._create_id_from_scope_code(entity_scope, entity_code) self.blocks[entity_id].append((code, as_at_time, performance_scope)) if block.asat is None: block.asat = as_at_time return block
def test_local_block(fs): # NOTE : Using the fake file-system # Set global config file paths block_path = os.path.join('folder', 'sub-folder') PerformanceConfiguration.set_global_config(LocalStorePath=block_path) # Create a block store bs = LocalBlockStore('SCOPE', 'NAME') bs.add_block('SCOPE', 'NAME', PerformanceDataSet('2018-03-05', '2018-03-19', '2020-03-19')) bs.add_block('SCOPE', 'NAME', PerformanceDataSet('2018-03-20', '2018-05-31', '2020-03-19')) # Make sure folder have been created assert os.listdir(block_path) == ['SCOPE'] # Make sure files are created contents = os.listdir(os.path.join(block_path, 'SCOPE')) assert 'NAME.idx' in contents assert 'NAME.block-1' in contents assert 'NAME.block-2' in contents
def create_test_case(performance_scope: str, entity_scope: str, entity_code: str, start_date, end_date, upserted_pdps: List[Dict], persisted_pdps: List[Dict]): """ Creates a test case for testing the upsert of returns :param str performance_scope: The scope of the block store to use to store performance returns :param str entity_scope: The scope of the entity to store returns against :param str entity_code: The code of the entity to store returns against :param start_date: The start date of the returns :param end_date: The end date of the returns :param List[Dict] upserted_pdps: The keyword arguments to create the PerformanceDataPoints to upsert :param List[Dict] persisted_pdps: The keyword arguments for the PerformanceDataPoints expected to be Persisted in the BlockStore :return: List: The created test case to use """ return [ performance_scope, entity_scope, entity_code, { "set1": PerformanceDataSetRequest(data_points=[ PerformanceDataPointRequest(**pdp) for pdp in upserted_pdps ]) }, UpsertReturnsResponse(successes={ "set1": PerformanceDataSetResponse( from_date=start_date, to_date=end_date, previous=PerformanceDataPointResponse(**upserted_pdps[-1]), data_points=[ PerformanceDataPointResponse(**pdp) for pdp in upserted_pdps ]) }, failures={}), [ PerformanceDataSet( from_date=start_date, to_date=end_date, data_points=[ PerformanceDataPoint(**pdp) for pdp in persisted_pdps ], previous=PerformanceDataPoint(**persisted_pdps[-1])) ] ]
def add_block(self, entity_scope: str, entity_code: str, block: PerformanceDataSet, performance_scope: str = None) -> PerformanceDataSet: """ This adds a block to the BlockStore for the specified entity. :param str entity_scope: The scope of the entity to get blocks for. :param str entity_code: The code of the entity to get blocks for. Together with the entity_scope this uniquely identifies the entity. :param PerformanceDataSet block: The block to add to the BlockStore :param str performance_scope: The scope to use in the BlockStore. This has no meaning and is not implemented in the InMemory implementation. """ entity_id = self._create_id_from_scope_code(entity_scope, entity_code) self.blocks[entity_id].append(block) if block.asat is None: # If the block has no asAt time, add one block.asat = datetime.now(pytz.UTC) return block
def upsert_portfolio_returns(performance_scope: str, portfolio_scope: str, portfolio_code: str, request_body: Dict[str, PerformanceDataSetRequest], block_store: IBlockStore): """ Upsert returns into a block store for a given portfolio. :param str performance_scope: The scope of the BlockStore to use to store the performance returns :param str portfolio_scope: The scope of the Portfolio the returns are associated with :param str portfolio_code: The code of the Portfolio the returns are associated with, together with the scope this uniquely identifies the Portfolio :param Dict[str, PerformanceDataSetRequest] request_body: The body of the request containing the PerformanceDataSets to persist as blocks :param IBlockStore block_store: The block store to use to persist the blocks :return: UpsertReturnsResponse: The response to the Upsert request """ # 2) Do Validation # Validate no duplication on dates of PerformanceDataPoint inside PerformanceDataSet # Validate no duplicates of PerformanceDataSets (based on unique code using from_date, to_date, entity_scope, entity_code) # 3) Cast to PerformanceDataSets pds = {} # For each PerformanceDataSet for correlation_id, performance_data_set_request in request_body.items(): start_date = performance_data_set_request.start_date end_date = performance_data_set_request.end_date # Sort all the PerformanceDataPoint into chronological order performance_data_point_chronological = sorted( performance_data_set_request.data_points, key=lambda x: x.date) # Ensure that the start_date and end_date are available if start_date is None: start_date = performance_data_point_chronological[0].date if end_date is None: end_date = performance_data_point_chronological[-1].date # Create a new PerformanceDataSet pd = PerformanceDataSet(from_date=start_date, to_date=end_date) # Add the returns to the PerformanceDataSet in the form of a number of PerformanceDataPoint [ pd.add_returns(date=pdp.date, weight=getattr(pdp, "weight", 0), ror=pdp.ror) for pdp in performance_data_point_chronological ] pds[correlation_id] = pd # 4) Persist each PerformanceDataSet in the BlockStore pds = { correlation_id: block_store.add_block(entity_scope=portfolio_scope, entity_code=portfolio_code, block=pd, performance_scope=performance_scope) for correlation_id, pd in pds.items() } # 5) Cast to Response objects pd_responses = {} # For each PerformanceDataSet for correlation_id, performance_data_set in pds.items(): # Create the response object pd = PerformanceDataSetResponse( from_date=performance_data_set.from_date, to_date=performance_data_set.to_date, asat=performance_data_set.asat, data_points=[ PerformanceDataPointResponse(**performance_data_point.__dict__) for performance_data_point in performance_data_set.data_points ], previous=PerformanceDataPointResponse( **performance_data_set.latest_data_point.__dict__)) pd_responses[correlation_id] = pd return UpsertReturnsResponse(pd_responses, {})