def fix_duplicate_key_value(records):
"""
Detect whether there are duplicate keyvalues for different records,
if there are, modify the keyvalues to make all keys unique.
"""
keys = []
for record in records:
i = 1
temp_key = record['KeyValue']
while temp_key in keys:
temp_key = record['KeyValue'] + '_' + str(i)
i = i + 1
record['KeyValue'] = temp_key
record['UserTranId1'] = temp_key
keys.append(record['KeyValue'])
# check again
keys = []
for record in records:
if record['KeyValue'] in keys:
logger.error(
'fix_duplicate_key_value(): duplicate keys still exists, key={0}, investment={1}'
.format(record['KeyValue'], record['Investment']))
raise DuplicateKeys()
keys.append(record['KeyValue'])
def create_fx_pairs(transaction_list):
"""
Since FXPurch/FXSale always occur in pairs, read the transaction_list,
create a list of such fx buy/sell pairs.
"""
buy_list, sell_list = filter_fx_trade_list(transaction_list)
if len(buy_list) != len(sell_list):
logger.error(
'create_fx_pairs(): {0} fx buy, {1} fx sell, not equal'.format(
len(buy_list), len(sell_list)))
raise InconsistentFXTrades()
return match(buy_list, sell_list, is_fx_trade_pair)
def get_LocationAccount(portfolio_id):
boc_portfolios = ['12229', '12366', '12528', '12630', '12732', '12733']
jpm_portfolios = ['12548']
if portfolio_id in boc_portfolios:
return 'BOCHK'
elif portfolio_id in jpm_portfolios:
return 'JPM'
else:
logger.error(
'get_LocationAccount(): no LocationAccount found for portfolio id {0}'
.format(portfolio_id))
raise LocationAccountNotFound()
def get_holding(portfolio_id, isin, holding_date):
"""
Get the quantity of holding of a position which needs paydown in a portfolio,
as of certain date.
"""
if portfolio_id == '12229' and holding_date > datetime(2015,10,8) \
and holding_date < datetime(2017,1,16) and isin == 'USG8116KAB82':
# this bond was bought on 2015-10-9, no other buy/sell or transfers
# occurred after that, before 2017-1-16
return 5320000
# information not found
logger.error('get_holding(): could not find holding info for portfolio {0}, isin {1}, on {2}'.
format(portfolio_id, isin, holding_date))
raise HoldingInfoNotFound()
def get_geneva_investment_id(trade_info):
"""
Get the Geneva investment ID for a security in FT file.
If a security has ISIN code, use its ISIN code to load its Geneva id,
otherwise use FT's unique security id to load the Geneva id.
"""
if trade_info['SCTYID_ISIN'] != '':
return get_investment_Ids(trade_info['ACCT_ACNO'], 'ISIN',
trade_info['SCTYID_ISIN'])[0]
elif trade_info['SCTYID_SMSEQ'] != '':
return get_investment_Ids(trade_info['ACCT_ACNO'], 'FT',
trade_info['SCTYID_SMSEQ'])[0]
else:
logger.error(
'get_geneva_investment_id(): no security identifier found.')
raise InvestmentIdNotFound()
def find_paydown_factor(paydown):
"""
Find factor for a paydown transaction.
"""
if abs(paydown['QTY']/paydown['GROSSLCL'] - 1) > 0.000001:
# pay amount != quantity, then the bond is not repaying at par,
# so pay down loss type cannot be 'no loss'.
logger.error('find_paydown_factors(): not repaying at par, isin={0}, on {1}'.
format(paydown['SCTYID_ISIN'], paydown['TRDDATE']))
raise InvalidPaydownType()
holding = get_holding(paydown['ACCT_ACNO'], paydown['SCTYID_ISIN'], paydown['TRDDATE'])
factor = (holding - paydown['QTY'])/holding
if abs(factor - get_factor_Bloomberg(paydown['SCTYID_ISIN'], paydown['TRDDATE'])) > 0.0001:
logger.error('find_paydown_factors(): paydown factor {0} differs from Bloomberg {1}'.
format(factor, get_factor_Bloomberg(paydown['SCTYID_ISIN'], paydown['TRDDATE'])))
raise InconsistentPaydownFactor()
return factor
def validate_cash(line_info):
"""
Validate cash related transactions, namely:
1. FXPurch, FXSale: buy one currency and sell another at the same time.
They always occur in pairs.
2. CashAdd: Cash deposit.
3. CashWth: Cash withdrawal.
4. IATCA, IATCW: cash transfers among accounts under FT control.
These two transactions always occur in pairs.
"""
if line_info['GROSSBAS'] != 0 and line_info['PRINB'] != 0 and line_info['GROSSLCL'] != 0 \
and line_info['FXRATE'] > 0:
pass
else:
logger.error('validate_cash(): GROSSBAS={0}, PRINB={1}, GROSSLCL={2}, fx={3} is not valid'.
format(line_info['GROSSBAS'], line_info['PRINB'],
line_info['GROSSLCL'], line_info['FXRATE']))
raise InvalidCashTransaction()
def validate_trade(line_info):
"""
Validate buy/sell transactions.
"""
if line_info['QTY'] > 0 and line_info['TRADEPRC'] > 0 and line_info['PRINB'] > 0 \
and line_info['FXRATE'] > 0:
pass
else:
logger.error('validate_trade(): quantity={0}, price={1}, prinb={2}, fx={3} is not valid'.
format(line_info['QTY'], line_info['TRADEPRC'],
line_info['PRINB'], line_info['FXRATE']))
raise InvalidTradeInfo()
diff = abs(line_info['GROSSBAS'] * line_info['FXRATE'] - line_info['GROSSLCL'])
if diff > 0.01:
logger.error('validate_line_info(): FX validation failed, diff={0}'.format(diff))
raise InvalidTradeInfo()
# for equity trade
diff2 = abs(line_info['PRINB']*line_info['FXRATE']) - line_info['QTY']*line_info['TRADEPRC']
# for bond trade
diff3 = abs(line_info['PRINB']*line_info['FXRATE']) - line_info['QTY']/100*line_info['TRADEPRC']
# print('diff2={0}, diff3={1}'.format(diff2, diff3))
if (abs(diff2) > 0.01 and abs(diff3) > 0.01):
logger.error('validate_trade(): price validation failed')
raise InvalidTradeInfo()
def match(list_a, list_b, matching_function):
"""
Match items in list_a to items in list_b. If all items in list_a find a
matched item in list_b, then a list of tuples are returned, as:
(item_a1, item_b1), (item_a2, item_b2), ... (item_aN, item_bN)
where item_a1 and item_b1 are matched items, etc. Two items in list_a
cannot match to the same item in list_b.
matching_function is a user supplied function to determine whether
an item from list_a is matched to another item in list_b.
If any item in list_a doesn't find a matched item in list_b, then
an exception is thrown.
"""
matched_position = []
for i in range(len(list_a)):
if i < len(matched_position):
continue
matched = False
for j in range(len(list_b)):
if not j in matched_position and matching_function(
list_a[i], list_b[j]):
matched_position.append(j)
matched = True
break
if not matched:
logger.error(
'item {0} in list_a: {1} does not find a matched item in list_b'
.format(i, list_a[i]))
raise MatchedItemNotFound()
# print(matched_position)
return create_matched_list(list_a, list_b, matched_position)
def match_repeat(list_a, list_b, matching_function):
"""
Match items in list_a to items in list_b. If all items in list_a find a
matched item in list_b, then a list of tuples are returned, as:
(item_a1, item_b1), (item_a2, item_b2), ... (item_aN, item_bN)
The difference between this function and the match() function is that
an item in list_b can be matched repeatedly, i.e., two items in list_a
can match to the same item in list_b.
For those items in list_a that don't find a matched item in list_b, they
are returned as a separate list.
"""
matched_position = []
unmatched_position = []
for i in range(len(list_a)):
if i < len(matched_position):
continue
matched = False
for j in range(len(list_b)):
if matching_function(list_a[i], list_b[j]):
matched_position.append(j)
matched = True
break
if not matched:
logger.error(
'item {0} in list_a: {1} does not find a matched item in list_b'
.format(i, list_a[i]))
unmatched_position.append(i)
# print(matched_position)
return create_matched_list(list_a, list_b, matched_position, unmatched_position), \
[list_a[i] for i in unmatched_position]
def validate_line_info(line_info):
for fld in line_info:
if fld in ['ACCT_ACNO', 'SCTYID_SMSEQ', 'SCTYID_SEDOL', 'SCTYID_CUSIP',
'TRANTYP', 'TRANCOD', 'LCLCCY', 'SCTYID_ISIN'] \
and not isinstance(line_info[fld], str):
logger.error('validate_line_info(): field {0} should be string, value={1}'.
format(fld, line_info[fld]))
raise InvalidLineInfo()
if fld in ['QTY', 'GROSSBAS', 'PRINB', 'RGLBVBAS', 'RGLCCYCLS',
'ACCRBAS', 'TRNBVBAS', 'GROSSLCL', 'FXRATE', 'TRADEPRC'] \
and not isinstance(line_info[fld], float):
logger.error('validate_line_info(): field {0} should be float, value={1}'.
format(fld, line_info[fld]))
raise InvalidLineInfo()
if fld in [ 'TRDDATE', 'STLDATE', 'ENTRDATE'] \
and not isinstance(line_info[fld], datetime):
logger.error('validate_line_info(): field {0} should be of type datetime, value={1}'.
format(fld, line_info[fld]))
raise InvalidLineInfo()
if line_info['STLDATE'] < line_info['TRDDATE'] or line_info['ENTRDATE'] < line_info['TRDDATE']:
logger.error('validate_line_info(): invalid dates, trade date={0}, settle day={1}, enterday={2}'.
format(line_info['TRDDATE'], line_info['STLDATE'], line_info['ENTRDATE']))
raise InvalidLineInfo()
# now validate further based on transaction type
if line_info['TRANTYP'] in ['Purch', 'Sale']:
validate_trade(line_info)
elif line_info['TRANTYP'] in ['IATCW', 'IATCA', 'CashAdd', 'CashWth',
'FXSale', 'FXPurch']:
validate_cash(line_info)