def process_args(api, args): if args.input: df = pd.concat( [ lpt.read_input(input_file, dtype=str).fillna("") for input_file in args.input ], ignore_index=True, sort=False, ) results = (batch_upsert_quotes(api, args.scope, c) for c in lpt.chunk(df, 2000)) # Create dict of LUSID upsert quotes failures failures = { k: v for i in results if i.is_right for k, v in i.right.content.failed.items() } # Update return df with the errors from LUSID updated_failures = update_failures(failures, df) # Check no api exceptions for any of the batches for f in results: if any(f.is_left): return f.left # If there were any LUSID failures, return the df if len(failures) > 0: return Either.Right(updated_failures) return Either.Right("Success")
def transactions(r=None): if args.transactions: df = lpt.read_input(args.transactions) if args.map: mi.map_instruments(api, df, "instrument_uid"), def load_transactions(portfolio, txns): return api.call.upsert_transactions( args.scope, portfolio, transactions=api.from_df(txns, api.models.TransactionRequest), ) if args.portfolio.lower().startswith("col:"): # Multiple portfolios contained in the file. Read the ID from the columns portfolio_column = args.portfolio[4:] def load_groups(iterator): try: portfolio, df = next(iterator) print("Transactions: {}".format(portfolio)) return load_transactions( portfolio, df.drop(portfolio_column, axis=1) ).bind(lambda r: load_groups(iterator)) except StopIteration: return Either.Right(None) return load_groups(iter(df.groupby(portfolio_column))) else: # one-off load. The portfolio id is provided return load_transactions(args.portfolio, df) return Either.Right(None)
def load_groups(iterator): try: portfolio, df = next(iterator) print("Holdings: {}".format(portfolio)) return set_holdings( portfolio, df).bind(lambda r: load_groups(iterator)) except StopIteration: return Either.Right(None)
def load_groups(iterator): try: portfolio, df = next(iterator) print("Transactions: {}".format(portfolio)) return load_transactions( portfolio, df.drop(portfolio_column, axis=1) ).bind(lambda r: load_groups(iterator)) except StopIteration: return Either.Right(None)
def batch_query(instr_type, prefix, outstanding): if len(outstanding) > 0: batch = outstanding[:500] # records to process now remainder = outstanding[500:] # remaining records # called if the get_instruments() succeeds def get_success(result): get_found = result.content.values get_failed = result.content.failed # Update successfully found items update_mappings(get_found, prefix) if len(get_failed) > 0: if instr_type == "ClientInternal": # For un-mapped internal codes, we will try to add (upsert) # called if the upsert_instruments() succeeds def add_success(result): add_worked = result.content.values add_failed = result.content.failed if len(add_failed) > 0: return Either.Left( "Failed to add internal instruments") # Update successfully added items update_mappings(add_worked, prefix) # Kick off the next batch return batch_query(instr_type, prefix, remainder) # Create the upsert request from the failed items request = { k: api.models.InstrumentDefinition( name=v.id, identifiers={"ClientInternal": v.id}) for k, v in get_failed.items() } return api.call.upsert_instruments(request).bind( add_success) else: # Instruments are not mapped. Nothing we cando. return Either.Left( "Failed to locate instruments of type {}".format( instr_type)) else: # No failures, kick off the next batch return batch_query(instr_type, prefix, remainder) return api.call.get_instruments( instr_type, list(batch[WORKING].values)).bind(get_success) else: # No records remaining. Return the now-enriched dataframe return Either.Right(df)
def create(): if args.create: return api.call.create_portfolio( args.scope, transaction_portfolio=api.models.CreateTransactionPortfolioRequest( code=args.portfolio, display_name=args.create[0], base_currency=args.create[1], created=lpt.to_date(args.create[2]), accounting_method=map_method(args.accounting_method), sub_holding_keys=args.shk, ), ) else: return Either.Right(None)
def load_groups(iterator, group_dict): try: index, row = next(iterator) return api.call.create_portfolio_group( scope=args.scope, request=api.models.CreatePortfolioGroupRequest( code=row[GROUP_NAME], values=group_dict[row[GROUP_NAME]][PORTFOLIOS], sub_groups=group_dict[row[GROUP_NAME]][SUB_GROUPS], display_name=row[DISPLAY_NAME], description=row[DESCRIPTION], ), ).bind(lambda r: load_groups(iterator, group_dict)) except StopIteration: return Either.Right(None)
def map_type(key, instr_type): prefix = key + ":" subset = df[df[column].str.startswith(prefix)] # See if there are any entries of this type if len(subset) > 0: width = len(prefix) uniques = subset[[column]].drop_duplicates(column) uniques[WORKING] = uniques[column].str.slice(width) def map_success(v): df.loc[subset.index, column] = subset[column].map(mapping_table) return Either.Right(df) return batch_query(instr_type, prefix, uniques).bind(map_success) else: # Nothing to be done, pass the full result back return Either.Right(df)
def reader(self, name: str, func: Callable, *args, **kwargs): """ The responsibility of this function is to act as a wrapper around an API call and read the result of the call :param str name: The name of the API call e.g. BuildTransactions :param Callable func: The API call :return: """ try: fn, result = self.calls.pop(0) assert fn == name return Either.Right(result) except Exception as e: print(f"ERROR READING FROM CACHE:{self.filename}\n" "DELETE THE CACHE FILE AND RETRY") raise e
def map_success(v): df.loc[subset.index, column] = subset[column].map(mapping_table) return Either.Right(df)
def process_args(api, args): def map_method(method): translations = { "FIFO": "FirstInFirstOut", "LIFO": "LastInFirstOut", "HCF": "HighestCostFirst", "LCF": "LowestCostFirst", } return translations.get(method, method) def create(): if args.create: return api.call.create_portfolio( args.scope, create_transaction_portfolio_request=api.models. CreateTransactionPortfolioRequest( code=args.portfolio, display_name=args.create[0], base_currency=args.create[1], created=lpt.to_date(args.create[2]), accounting_method=map_method(args.accounting_method), sub_holding_keys=args.shk, ), ) else: return Either.Right(None) def transactions(r=None): if args.transactions: df = lpt.read_input(args.transactions) if args.map: mi.map_instruments(api, df, "instrument_uid"), def load_transactions(portfolio, txns): return api.call.upsert_transactions( args.scope, portfolio, transaction_request=api.from_df( txns, api.models.TransactionRequest), ) if args.portfolio.lower().startswith("col:"): # Multiple portfolios contained in the file. Read the ID from the columns portfolio_column = args.portfolio[4:] def load_groups(iterator): try: portfolio, df = next(iterator) print("Transactions: {}".format(portfolio)) return load_transactions( portfolio, df.drop( portfolio_column, axis=1)).bind(lambda r: load_groups(iterator)) except StopIteration: return Either.Right(None) return load_groups(iter(df.groupby(portfolio_column))) else: # one-off load. The portfolio id is provided return load_transactions(args.portfolio, df) return Either.Right(None) def positions(r=None): if args.positions: taxlot_fields = [ "units", "cost.amount", "cost.currency", "portfolio_cost", "price", "purchase_date", "settlement_date", ] df = lpt.read_input(args.positions[0]) # Get unique key fields to group by keys = ["instrument_uid" ] + [c for c in df.columns.values if c.startswith("SHK:")] # Fill down any blanks df[keys] = df[keys].fillna(method="ffill") def set_holdings(portfolio, holdings): # Group by the keys and build request for each group return api.call.set_holdings( args.scope, portfolio, lpt.to_date(args.positions[1]), adjust_holding_request=[ api.models.AdjustHoldingRequest( instrument_identifiers=lpt. to_instrument_identifiers(i if len(keys) == 1 else i[0]), sub_holding_keys=lpt.perpetual_upsert( api.models, hld_df, "SHK:"), properties=lpt.perpetual_upsert( api.models, hld_df), tax_lots=api.from_df( hld_df[taxlot_fields], api.models.TargetTaxLotRequest), ) for i, hld_df in holdings.groupby(keys) ], ) if args.portfolio.lower().startswith("col:"): # Multiple portfolios contained in the file. Read the ID from the columns def load_groups(iterator): try: portfolio, df = next(iterator) print("Holdings: {}".format(portfolio)) return set_holdings( portfolio, df).bind(lambda r: load_groups(iterator)) except StopIteration: return Either.Right(None) return load_groups(iter(df.groupby(args.portfolio[4:]))) else: # one-off load. The portfolio id is provided return set_holdings(args.portfolio, df) return Either.Right(None) return (create().bind(positions).bind(transactions).bind( lambda r: Either.Right("Done!")))
def positions(r=None): if args.positions: taxlot_fields = [ "units", "cost.amount", "cost.currency", "portfolio_cost", "price", "purchase_date", "settlement_date", ] df = lpt.read_input(args.positions[0]) # Get unique key fields to group by keys = ["instrument_uid" ] + [c for c in df.columns.values if c.startswith("SHK:")] # Fill down any blanks df[keys] = df[keys].fillna(method="ffill") def set_holdings(portfolio, holdings): # Group by the keys and build request for each group return api.call.set_holdings( args.scope, portfolio, lpt.to_date(args.positions[1]), adjust_holding_request=[ api.models.AdjustHoldingRequest( instrument_identifiers=lpt. to_instrument_identifiers(i if len(keys) == 1 else i[0]), sub_holding_keys=lpt.perpetual_upsert( api.models, hld_df, "SHK:"), properties=lpt.perpetual_upsert( api.models, hld_df), tax_lots=api.from_df( hld_df[taxlot_fields], api.models.TargetTaxLotRequest), ) for i, hld_df in holdings.groupby(keys) ], ) if args.portfolio.lower().startswith("col:"): # Multiple portfolios contained in the file. Read the ID from the columns def load_groups(iterator): try: portfolio, df = next(iterator) print("Holdings: {}".format(portfolio)) return set_holdings( portfolio, df).bind(lambda r: load_groups(iterator)) except StopIteration: return Either.Right(None) return load_groups(iter(df.groupby(args.portfolio[4:]))) else: # one-off load. The portfolio id is provided return set_holdings(args.portfolio, df) return Either.Right(None)
def process_args(api, args): taxlot_fields = [ "units", "cost.amount", "cost.currency", "portfolio_cost", "price", "purchase_date", "settlement_date", ] # Check if is json or csv if "csv" in args.input: df = lpt.read_input(args.input) elif "json" in args.input: # Convert from json to dataframe with open(args.input, "r") as myfile: data = json.load(myfile) df = json_normalize(data, max_level=1) df.columns = df.columns.str.replace("subHoldingKeys.", SHK) df.columns = df.columns.str.replace("properties.", P) columns_to_modify = [ c for c in df.columns.values if c.startswith(SHK) or c.startswith(P) ] for col in columns_to_modify: df[col] = df[col].apply(lambda x: x.get("value", {}).get("labelValue", "")) else: raise Exception( "The file provided: {} is not .json or .csv format.".format(args.input) ) # Check the schema for column in taxlot_fields: if column not in df.columns: df[column] = None keys = ["instrument_uid"] + [c for c in df.columns.values if c.startswith(SHK)] # Fill down any blanks df[keys] = df[keys].fillna(method="ffill") # Group by the keys and build request for each group return api.call.set_holdings( args.scope, args.portfolio, lpt.to_date(args.date), holding_adjustments=[ api.models.AdjustHoldingRequest( instrument_identifiers=lpt.to_instrument_identifiers( i if len(keys) == 1 else i[0] ), sub_holding_keys=lpt.perpetual_upsert(api.models, hld_df, SHK), properties=lpt.perpetual_upsert(api.models, hld_df), tax_lots=api.from_df( hld_df[taxlot_fields], api.models.TargetTaxLotRequest ), ) for i, hld_df in df.groupby(keys) ], ) return Either.Right(None)
def process_args(api, args): # Build the ResourceID variables def resource_id_vars(names): return [api.models.ResourceId(args.scope, code) for code in names] # Sort the dataframe in order for the sub-groups to be created before the groups (takes care of the dependency sub-group group) def sorted_group_df(df): df_sorted = df[df[SUB_GROUPS].isnull()] df_not_sorted = df[df[SUB_GROUPS].notnull()] while not df_not_sorted.empty: for i, row in df_not_sorted.iterrows(): if row[SUB_GROUPS] in df_sorted[GROUP_NAME].to_list(): df_sorted = df_sorted.append(row) df_not_sorted = df_not_sorted.drop(i) if (not any(x in df_sorted[GROUP_NAME].to_list() for x in df_not_sorted[SUB_GROUPS].to_list()) and not df_not_sorted.empty): raise ValueError( "Is a circular dependency between the groups and subgroups, or one of the subgroups does not exist {}" .format(df_not_sorted)) return df_sorted def load_groups(iterator, group_dict): try: index, row = next(iterator) return api.call.create_portfolio_group( scope=args.scope, create_portfolio_group_request=api.models. CreatePortfolioGroupRequest( code=row[GROUP_NAME], created=row.get(CREATED_DATE, None), values=group_dict[row[GROUP_NAME]][PORTFOLIOS], sub_groups=group_dict[row[GROUP_NAME]][SUB_GROUPS], display_name=row[DISPLAY_NAME], description=row[DESCRIPTION], ), ).bind(lambda r: load_groups(iterator, group_dict)) except StopIteration: return Either.Right(None) def create_groups(): df = lpt.read_input(args.input[0]) # Build the group dictionaries {'group_name': {'Portfolios: ['portfolio1]', 'portfolio2'], 'Sub-groups': ['sub-group1', 'sub-group2']} group_dict = {} group_dict.update({ group_name: { SUB_GROUPS: resource_id_vars(group_df[SUB_GROUPS].dropna().unique()), PORTFOLIOS: resource_id_vars(group_df[PORTFOLIOS].dropna().unique()), } for group_name, group_df in df.groupby(GROUP_NAME) }) df = sorted_group_df(df) df.drop_duplicates(subset=[GROUP_NAME, DISPLAY_NAME], keep="last", inplace=True) return load_groups(df.iterrows(), group_dict) return create_groups().bind(lambda r: Either.Right("Done!"))