def load_transaction_batch( api_factory: lusid.utilities.ApiClientFactory, transaction_batch: list, **kwargs ) -> lusid.models.UpsertPortfolioTransactionsResponse: """ Upserts a batch of transactions into LUSID :param lusid.utilities.ApiClientFactory api_factory: The api factory to use :param str scope: The scope of the Transaction Portfolio to upsert the transactions into :param str code: The code of the Transaction Portfolio, together with the scope this uniquely identifies the portfolio :param list[lusid.models.TransactionRequest] transaction_batch: The batch of transactions to upsert :return: lusid.models.UpsertPortfolioTransactionsResponse: The response from LUSID """ if "scope" not in list(kwargs.keys()): raise KeyError( "You are trying to load transactions without a scope, please ensure that a scope is provided." ) if "code" not in list(kwargs.keys()): raise KeyError( "You are trying to load transactions without a portfolio code, please ensure that a code is provided." ) return api_factory.build( lusid.api.TransactionPortfoliosApi ).upsert_transactions( scope=kwargs["scope"], code=kwargs["code"], transactions=transaction_batch )
def check_property_definitions_exist_in_scope_single( api_factory: lusid.utilities.ApiClientFactory, property_key: str) -> (bool, str): """ This function takes a list of property keys and looks to see which property definitions already exist inside LUSID :param lusid.utilities.ApiClientFactory api_factory: The ApiFactory to use :param str property_key: The property key to get from LUSID :return: bool, str exists, data_type: Whether or not the property definition exists and if it does its data type """ data_type = None try: response = api_factory.build( lusid.PropertyDefinitionsApi).get_property_definition( domain=property_key.split("/")[0], scope=property_key.split("/")[1], code=property_key.split("/")[2], ) exists = True data_type = response.data_type_id.code except lusid.exceptions.ApiException: exists = False return exists, data_type
def load_quote_batch( api_factory: lusid.utilities.ApiClientFactory, quote_batch: list, **kwargs ) -> lusid.models.UpsertQuotesResponse: """ Upserts a batch of quotes into LUSID :param lusid.utilities.ApiClientFactory api_factory: The api factory to use :param list[lusid.models.UpsertQuoteRequest] quote_batch: The batch of quotes to upsert :param str scope: The scope to upsert the quotes into :return: lusid.models.UpsertQuotesResponse: The response from LUSID """ if "scope" not in list(kwargs.keys()): raise KeyError( "You are trying to load quotes without a scope, please ensure that a scope is provided." ) return api_factory.build(lusid.api.QuotesApi).upsert_quotes( scope=kwargs["scope"], quotes={ "_".join( [ quote.quote_id.quote_series_id.instrument_id, quote.quote_id.quote_series_id.instrument_id_type, str(quote.quote_id.effective_at), ] ): quote for quote in quote_batch }, )
def _get_portfolio_holdings( api_factory: lusid.utilities.ApiClientFactory, scope: str, code: str, **kwargs) -> Dict[str, List[lusid.models.PortfolioHolding]]: """ This function gets the holdings of a Portfolio from LUSID. Parameters ---------- api_factory : lusid.utilities.ApiClientFactory The api factory to use scope : str The scope of the Portfolio code : str The code of the Portfolio, with the scope this uniquely identifiers the Portfolio effective_at : datetime The effective datetime at which to get the Portfolio Group as_at : datetime The as at datetime at which to get the Portfolio Group filter : str by_taxlots : bool property_keys : list[str] Returns ------- response : Dict[str, List[lusid.models.PortfolioHolding]] The list of PortfolioHolding keyed by the unique combination of the Portfolio's scope and code Other Parameters ------ effective_at : datetime The effective datetime at which to get the Portfolio Group as_at : datetime The as at datetime at which to get the Portfolio Group filter : str The filter to use to filter the holdings by_taxlots : bool Whether or not to break the holdings down into individual tax lots property_keys : list[str] The list of property keys to decorate onto the holdings, must be from the Instrument domain thread_pool The thread pool to run this function in """ # Filter out the relevant keyword arguments as the LUSID API will raise an exception if given extras lusid_keyword_arguments = { key: value for key, value in kwargs.items() if key in ["effective_at", "as_at", "filter", "by_taxlots", "property_keys"] } # Call LUSID to get the holdings for the Portfolio response = lusid.api.TransactionPortfoliosApi( api_factory.build(lusid.api.TransactionPortfoliosApi)).get_holdings( scope=scope, code=code, **lusid_keyword_arguments) # Key the response with the unique scope/code combination return {f"{scope} : {code}": response.values}
def load_instrument_batch( api_factory: lusid.utilities.ApiClientFactory, instrument_batch: list, **kwargs ) -> lusid.models.UpsertInstrumentsResponse: """ Upserts a batch of instruments to LUSID :param lusid.utilities.ApiClientFactory api_factory: The api factory to use :param list[lusid.models.InstrumentDefinition] instrument_batch: The batch of instruments to upsert :return: UpsertInstrumentsResponse: The response from LUSID """ # Ensure that the list of allowed unique identifiers exists if "unique_identifiers" not in list(kwargs.keys()): unique_identifiers = cocoon.instruments.get_unique_identifiers( api_factory=api_factory ) else: unique_identifiers = kwargs["unique_identifiers"] @checkargs def get_alphabetically_first_identifier_key( instrument: lusid.models.InstrumentDefinition, unique_identifiers: list ): """ Gets the alphabetically first occurring unique identifier on an instrument and use it as the correlation id on the request :param lusid.models.InstrumentDefinition instrument: The instrument to create a correlation id for :param list[str] unique_identifiers: The list of allowed unique identifiers :return: str: The correlation id to use on the request """ unique_identifiers_populated = list( set(unique_identifiers).intersection( set(list(instrument.identifiers.keys())) ) ) unique_identifiers_populated.sort() first_unique_identifier_alphabetically = unique_identifiers_populated[0] return f"{first_unique_identifier_alphabetically}: {instrument.identifiers[first_unique_identifier_alphabetically].value}" return api_factory.build(lusid.api.InstrumentsApi).upsert_instruments( instruments={ get_alphabetically_first_identifier_key( instrument, unique_identifiers ): instrument for instrument in instrument_batch } )
def load_portfolio_batch( api_factory: lusid.utilities.ApiClientFactory, portfolio_batch: list, **kwargs ) -> lusid.models.Portfolio: """ Upserts a batch of portfolios to LUSID :param lusid.utilities.ApiClientFactory api_factory: The api factory to use :param list[lusid.models.CreateTransactionPortfolioRequest] portfolio_batch: The batch of portfolios to create :param kwargs: 'scope', 'code' arguments required for the API call :return: lusid.models.Portfolio: The response from LUSID """ if "scope" not in list(kwargs.keys()): raise KeyError( "You are trying to load transactions without a scope, please ensure that a scope is provided." ) if "code" not in list(kwargs.keys()): raise KeyError( "You are trying to load transactions without a portfolio code, please ensure that a code is provided." ) try: return api_factory.build(lusid.api.PortfoliosApi).get_portfolio( scope=kwargs["scope"], code=kwargs["code"] ) # Add in here upsert portfolio properties if it does exist except lusid.exceptions.ApiException as e: if e.status == 404: return api_factory.build( lusid.api.TransactionPortfoliosApi ).create_portfolio( scope=kwargs["scope"], transaction_portfolio=portfolio_batch[0] ) else: return e
def instrument_search_single( api_factory: lusid.utilities.ApiClientFactory, search_request: lusid.models.InstrumentSearchProperty, ) -> list: """ Conducts an instrument search with a single search request :param lusid.utilities.ApiClientFactory api_factory: The api factory to use :param lusid.models.InstrumentSearchProperty search_request: The search request :return: list[lusid.models.InstrumentMatch]: The results of the search """ return lusid.api.SearchApi(api_factory.build( lusid.api.SearchApi)).instruments_search(symbols=[search_request])
def get_unique_identifiers(api_factory: lusid.utilities.ApiClientFactory): """ Tests getting the unique instrument identifiers :param lusid.utilities.ApiClientFactory api_factory: The LUSID api factory to use :return: list[str]: The property keys of the available identifiers """ # Get the allowed instrument identifiers from LUSID identifiers = api_factory.build( lusid.api.InstrumentsApi).get_instrument_identifier_types() # Return the identifiers that are configured to be unique return [ identifier.identifier_type for identifier in identifiers.values if identifier.is_unique_identifier_type ]
def _get_portfolio_group(api_factory: lusid.utilities.ApiClientFactory, scope: str, code: str, **kwargs) -> lusid.models.PortfolioGroup: """ This function gets a Portfolio Group from LUSID. Parameters ---------- api_factory : lusid.utilities.ApiClientFactory The api factory to use scope : str The scope of the Portfolio Group code : str The code of the Portfolio Group, with the scope this uniquely identifiers the Portfolio Group Returns ------- response : lusid.models.PortfolioGroup The Portfolio Group Other Parameters ------ effective_at : datetime The effective datetime at which to get the Portfolio Group as_at : datetime The as at datetime at which to get the Portfolio Group thread_pool The thread pool to run this function in """ # Filter out the relevant keyword arguments as the LUSID API will raise an exception if given extras lusid_keyword_arguments = { key: value for key, value in kwargs.items() if key in ["effective_at", "as_at"] } # Call LUSID to get the portfolio group response = lusid.api.PortfolioGroupsApi( api_factory.build(lusid.api.PortfolioGroupsApi)).get_portfolio_group( scope=scope, code=code, **lusid_keyword_arguments) return response
def load_transaction_batch( api_factory: lusid.utilities.ApiClientFactory, transaction_batch: list, **kwargs ) -> lusid.models.UpsertPortfolioTransactionsResponse: """ Upserts a batch of transactions into LUSID Parameters ---------- api_factory : lusid.utilities.ApiClientFactory The api factory to use code : str The code of the TransactionPortfolio to upsert the transactions into transaction_batch : list[lusid.models.TransactionRequest] The batch of transactions to upsert kwargs code -The code of the TransactionPortfolio to upsert the transactions into Returns ------- lusid.models.UpsertPortfolioTransactionsResponse The response from LUSID """ if "scope" not in list(kwargs.keys()): raise KeyError( "You are trying to load transactions without a scope, please ensure that a scope is provided." ) if "code" not in list(kwargs.keys()): raise KeyError( "You are trying to load transactions without a portfolio code, please ensure that a code is provided." ) return api_factory.build( lusid.api.TransactionPortfoliosApi ).upsert_transactions( scope=kwargs["scope"], code=kwargs["code"], transaction_request=transaction_batch, )
def instrument_search_single( api_factory: lusid.utilities.ApiClientFactory, search_request: lusid.models.InstrumentSearchProperty, **kwargs, ) -> list: """ Conducts an instrument search with a single search request Parameters ---------- api_factory : lusid.utilities.ApiClientFactory The Api factory to use search_request : lusid.models.InstrumentSearchProperty The search request kwargs Returns ------- list[lusid.models.InstrumentMatch] The results of the search """ return lusid.api.SearchApi(api_factory.build( lusid.api.SearchApi)).instruments_search(symbols=[search_request])
def resolve_instruments( api_factory: lusid.utilities.ApiClientFactory, data_frame: pd.DataFrame, identifier_mapping: dict, ): """ This function attempts to resolve each row of the file to an instrument in LUSID Parameters ---------- api_factory : lusid.utilities.ApiClientFactory An instance of the Lusid Api Factory data_frame : pd.DataFrame The DataFrame containing the transactions or holdings to resolve to unique instruments identifier_mapping : dict The column mapping between allowable identifiers in LUSID and identifier columns in the dataframe Returns ------- _data_frame : pd.DataFrame The input DataFrame with resolution columns added """ if "Currency" not in list(identifier_mapping.keys() ) and "Instrument/default/Currency" not in list( identifier_mapping.keys()): raise KeyError( """There is no column specified in the identifier_mapping to identify whether or not an instrument is cash. Please specify add the key "Currency" or "Instrument/default/Currency" to your mapping. If no instruments are cash you can set the value to be None""") if "Currency" in list(identifier_mapping.keys()): identifier_mapping["Instrument/default/Currency"] = identifier_mapping[ "Currency"] del identifier_mapping["Currency"] # Check that the values of the mapping exist in the DataFrame if not (set(identifier_mapping.values()) <= set(data_frame.columns)): raise KeyError( "there are values in identifier_mapping that are not columns in the dataframe" ) # Get the allowable instrument identifiers from LUSID response = api_factory.build( InstrumentsApi).get_instrument_identifier_types() """ # Collect the names and property keys for the identifiers and concatenate them allowable_identifier_names = [identifier.identifier_type for identifier in response.values] allowable_identifier_keys = [identifier.property_key for identifier in response.values] allowable_identifiers = allowable_identifier_names + allowable_identifier_keys # Check that the identifiers in the mapping are all allowed to be used in LUSID if not (set(identifier_mapping['identifier_mapping'].keys()) <= set(allowable_identifiers)): raise Exception( 'there are LUSID identifiers in the identifier_mapping which are not configured in LUSID') """ # Copy the data_frame to ensure the original isn't modified _data_frame = data_frame.copy(deep=True) # Set up the result Pandas Series to track resolution found_with = pd.Series(index=_data_frame.index, dtype=np.dtype(object)) resolvable = pd.Series(index=_data_frame.index, dtype=np.dtype(bool)) luid = pd.Series(index=_data_frame.index, dtype=np.dtype(object)) comment = pd.Series(index=_data_frame.index, dtype=np.dtype(object)) logging.info("Beginning instrument resolution process") # Iterate over each row in the DataFrame for index, row in _data_frame.iterrows(): if index % 10 == 0: logging.info(f"Up to row {index}") # Initialise list to hold the identifiers used to resolve found_with_current = [] # Initialise a value of False for the row's resolvability to an instrument in LUSID resolvable_current = False # Initilise the LUID value luid_current = None # Initialise the comment value comment_current = "No instruments found for the given identifiers" # Takes the currency resolution function and applies it currency = row[identifier_mapping["Instrument/default/Currency"]] if not pd.isna(currency): resolvable_current = True found_with_current.append(currency) luid_current = currency comment_current = "Resolved as cash with a currency" search_requests = [ models.InstrumentSearchProperty( key=f"Instrument/default/{identifier_lusid}" if "Instrument/" not in identifier_lusid else identifier_lusid, value=row[identifier_dataframe], ) for identifier_lusid, identifier_dataframe in identifier_mapping.items() if not pd.isnull(row[identifier_dataframe]) ] # Call LUSID to search for instruments attempts = 0 if len(search_requests) > 0: while attempts < 3: try: response = api_factory.build(SearchApi).instruments_search( instrument_search_property=search_requests, mastered_only=True) break except lusid.exceptions.ApiException as error_message: attempts += 1 comment_current = f"Failed to find instrument due to LUSID error during search due to status {error_message.status} with reason {error_message.reason}" time.sleep(5) if attempts == 3: # Update the luid series luid.iloc[index] = luid_current # Update the found with series found_with.iloc[index] = found_with_current # Update the resolvable series resolvable.iloc[index] = resolvable_current # Update the comment series comment.iloc[index] = comment_current continue search_request_number = -1 for result in response: search_request_number += 1 # If there are matches if len(result.mastered_instruments) == 1: # Add the identifier responsible for the successful search request to the list found_with_current.append( search_requests[search_request_number].key.split( "/")[2]) comment_current = ( "Uniquely resolved to an instrument in the securities master" ) resolvable_current = True luid_current = (result.mastered_instruments[0]. identifiers["LusidInstrumentId"].value) break elif len(result.mastered_instruments) > 1: comment_current = f'Multiple instruments found for the instrument using identifier {search_requests[search_request_number].key.split("/")[2]}' resolvable_current = False luid_current = np.NaN # Update the luid series luid.iloc[index] = luid_current # Update the found with series found_with.iloc[index] = found_with_current # Update the resolvable series resolvable.iloc[index] = resolvable_current # Update the comment series comment.iloc[index] = comment_current # Add the series to the dataframe _data_frame["resolvable"] = resolvable _data_frame["foundWith"] = found_with _data_frame["LusidInstrumentId"] = luid _data_frame["comment"] = comment return _data_frame
def create_property_definitions_from_file( api_factory: lusid.utilities.ApiClientFactory, scope: str, domain: str, data_frame: pd.DataFrame, missing_property_columns: list, ): """ Creates the property definitions for all the columns in a file :param lusid.utilities.ApiClientFactory api_factory: The ApiFactory to use :param str scope: The scope to create the property definitions in :param str domain: The domain to create the property definitions in :param Pandas Series data_frame: The dataframe dtypes to add definitions for :param list[str] missing_property_columns: The columns that property defintions are missing for :return: dict property_key_mapping: A mapping of data_frame columns to property keys """ missing_property_data_frame = data_frame.loc[:, missing_property_columns] # Ensure that all data types in the file have been mapped if not (set([ str(data_type) for data_type in missing_property_data_frame.dtypes.unique() ]) <= set(global_constants["data_type_mapping"])): raise TypeError( """There are data types in the data_frame which have not been mapped to LUSID data types, please ensure that all data types have been mapped before retrying""" ) # Initialise a dictionary to hold the keys property_key_mapping = {} # Iterate over the each column and its data type for column_name, data_type in missing_property_data_frame.dtypes.iteritems( ): # Make the column name LUSID friendly lusid_friendly_code = cocoon.utilities.make_code_lusid_friendly( column_name) # If there is no data Pandas infers a type of float, would prefer to infer object if missing_property_data_frame[column_name].isnull().all(): logging.warning( f"{column_name} is null, no type can be inferred it will be treated as a string" ) data_type = "object" data_frame[column_name] = data_frame[column_name].astype( "object", copy=False) # Create a request to define the property, assumes value_required is false for all property_request = lusid.models.CreatePropertyDefinitionRequest( domain=domain, scope=scope, code=lusid_friendly_code, value_required=False, display_name=column_name, data_type_id=lusid.models.ResourceId( scope="system", code=global_constants["data_type_mapping"][str(data_type)], ), ) # Call LUSID to create the new property property_response = api_factory.build( lusid.PropertyDefinitionsApi).create_property_definition( definition=property_request) logging.info( f"Created - {property_response.key} - with datatype {property_response.data_type_id.code}" ) # Grab the key off the response to use when referencing this property in other LUSID calls property_key_mapping[column_name] = property_response.key return property_key_mapping, data_frame
def load_portfolio_group_batch( api_factory: lusid.utilities.ApiClientFactory, portfolio_group_batch: list, **kwargs, ) -> lusid.models.PortfolioGroup: """ Upserts a batch of portfolios to LUSID Parameters ---------- api_factory : lusid.utilities.ApiClientFactory the api factory to use portfolio_group_batch : list[lusid.models.CreateTransactionPortfolioRequest] The batch of portfilios to create scope : str The scope to create the portfolio group in code : str The code of the portfolio group to create kwargs Returns ------- lusid.models.PortfolioGroup The response from LUSID """ updated_request = group_request_into_one( portfolio_group_batch[0].__class__.__name__, portfolio_group_batch, ["values"], ) if "scope" not in list(kwargs.keys()): raise KeyError( "You are trying to load a portfolio group without a scope, please ensure that a scope is provided." ) if "code" not in list(kwargs.keys()): raise KeyError( "You are trying to load a portfolio group without a portfolio code, please ensure that a code is provided." ) try: current_portfolio_group = api_factory.build( lusid.api.PortfolioGroupsApi ).get_portfolio_group(scope=kwargs["scope"], code=kwargs["code"]) # Capture all portfolios - the ones currently in group + the new ones to be added all_portfolios_to_add = ( updated_request.values + current_portfolio_group.portfolios ) current_portfolios_in_group = [ code for code in updated_request.values if code in current_portfolio_group.portfolios ] if len(current_portfolios_in_group) > 0: for code in current_portfolios_in_group: logging.info( f"The portfolio {code.code} with scope {code.scope} is already in group {current_portfolio_group.id.code}" ) # Parse out new portfolios only new_portfolios = [ code for code in all_portfolios_to_add if code not in current_portfolio_group.portfolios ] for code, scope in set( [(resource.code, resource.scope) for resource in new_portfolios] ): try: current_portfolio_group = api_factory.build( lusid.api.PortfolioGroupsApi ).add_portfolio_to_group( scope=kwargs["scope"], code=kwargs["code"], effective_at=datetime.now(tz=pytz.UTC), resource_id=lusid.models.ResourceId(scope=scope, code=code), ) except lusid.exceptions.ApiException as e: logging.error(json.loads(e.body)["title"]) return current_portfolio_group # Add in here upsert portfolio properties if it does exist except lusid.exceptions.ApiException as e: if e.status == 404: return api_factory.build( lusid.api.PortfolioGroupsApi ).create_portfolio_group( scope=kwargs["scope"], create_portfolio_group_request=updated_request, ) else: return e
def load_instrument_property_batch( api_factory: lusid.utilities.ApiClientFactory, property_batch: list, **kwargs ) -> [lusid.models.UpsertInstrumentPropertiesResponse]: """ Add properties to the set instruments Parameters ---------- api_factory : lusid.utilities.ApiClientFactory The api factory to use property_batch : list[lusid.models.UpsertInstrumentPropertyRequest] Properties to add, identifiers will be resolved to a LusidInstrumentId, where an identifier resolves to more than one LusidInstrumentId the property will be added to all matching instruments kwargs Returns ------- list[lusid.models.UpsertInstrumentPropertiesResponse] the response from LUSID """ results = [] for request in property_batch: search_request = lusid.models.InstrumentSearchProperty( key=f"instrument/default/{request.identifier_type}", value=request.identifier, ) # find the matching instruments mastered_instruments = api_factory.build( lusid.api.SearchApi ).instruments_search( instrument_search_property=[search_request], mastered_only=True ) # flat map the results to a list of luids luids = [ luid for luids in [ list( map( lambda m: m.identifiers["LusidInstrumentId"].value, mastered.mastered_instruments, ) ) for mastered in [matches for matches in mastered_instruments] ] for luid in luids ] if len(luids) == 0: continue properties_request = [ lusid.models.UpsertInstrumentPropertyRequest( identifier_type="LusidInstrumentId", identifier=luid, properties=request.properties, ) for luid in luids ] results.append( api_factory.build( lusid.api.InstrumentsApi ).upsert_instruments_properties(properties_request) ) return results
def load_holding_batch( api_factory: lusid.utilities.ApiClientFactory, holding_batch: list, **kwargs ) -> lusid.models.HoldingsAdjustment: """ Upserts a batch of holdings into LUSID Parameters ---------- api_factory : lusid.utilities.ApiClientFactory The api factory to use holding_batch : list[lusid.models.AdjustHoldingRequest] The batch of holdings scope : str The scope to upsert holdings into code : str The code of the portfolio to upsert holdings into effective_at : str/Datetime/np.datetime64/np.ndarray/pd.Timestamp The effective date of the holdings batch kwargs Returns ------- lusid.models.HoldingsAdjustment The response from LUSID """ if "scope" not in list(kwargs.keys()): raise KeyError( "You are trying to load transactions without a scope, please ensure that a scope is provided." ) if "code" not in list(kwargs.keys()): raise KeyError( "You are trying to load transactions without a portfolio code, please ensure that a code is provided." ) if "effective_at" not in list(kwargs.keys()): raise KeyError( """There is no mapping for effective_at in the required mapping, please add it""" ) # If only an adjustment has been specified if ( "holdings_adjustment_only" in list(kwargs.keys()) and kwargs["holdings_adjustment_only"] ): return api_factory.build( lusid.api.TransactionPortfoliosApi ).adjust_holdings( scope=kwargs["scope"], code=kwargs["code"], effective_at=str(DateOrCutLabel(kwargs["effective_at"])), adjust_holding_request=holding_batch, ) return api_factory.build(lusid.api.TransactionPortfoliosApi).set_holdings( scope=kwargs["scope"], code=kwargs["code"], effective_at=str(DateOrCutLabel(kwargs["effective_at"])), adjust_holding_request=holding_batch, )