def test_init(self):
# Bare init
instance = GroupedList()
self.assertEqual(len(instance), 0)
self.assertIs(instance.grouped, False)
self.assertIs(instance.key, None)
# init list
instance = GroupedList([])
self.assertEqual(len(instance), 0)
self.assertIs(instance.grouped, False)
self.assertIs(instance.key, None)
def doTrades(
self,
transactions: Iterable[Trade],
session: sqlalchemy.orm.session.Session,
securities: SecuritiesMap,
account: models.FiAccount,
default_currency: str,
) -> List[models.Transaction]:
"""Preprocess trade transactions and send to merge_trade().
The logic here eliminates unwanted trades (e.g. FX) and groups trades
to net out canceled trades.
"""
apply_cancels = make_canceller(
filterfunc=self.is_trade_cancel,
matchfunc=self.are_trade_cancel_pair,
sortfunc=self.sort_trades_to_cancel,
)
_merge_trade = functools.partial(
merge_trade,
session=session,
securities=securities,
account=account,
default_currency=default_currency,
get_trade_sort_algo=self.get_trade_sort_algo,
)
transactions = (
GroupedList(transactions).filter(self.is_security_trade).groupby(
self.fingerprint_trade).bind(apply_cancels).filter(
operator.attrgetter(
"units")) # Removes net 0 unit transactions
.map(_merge_trade).flatten())[:]
return transactions
def exercise(
transactions: List[ofxtools.models.TRANSFER],
session: sqlalchemy.orm.session.Session,
securities: ofx.reader.SecuritiesMap,
account: models.FiAccount,
default_currency: str,
memo: str,
) -> List[models.Transaction]:
"""
HACK based on assumption that all transactions in list represent a
single transaction
"""
# Not ready for prime time
return []
def group_key(tx):
return ((tx.uniqueidtype, tx.uniqueid), tx.tferaction, tx.postype)
def net_exercises(tx0, tx1):
units0 = tx0.units
if tx0.postype == "SHORT":
units0 *= -1
tx0.postype = "LONG"
units1 = tx1.units
if tx1.postype == "SHORT":
units1 *= -1
tx0.units += tx1.units
return tx0
def sort_key(tx):
return tx.tferaction
txs = (GroupedList(transactions).groupby(group_key).reduce(
net_exercises).flatten().sorted(sort_key))
txs = txs.pop(None)
assert len(txs) == 2
dest, src = txs
assert dest.tferaction == "IN"
assert src.tferaction == "OUT"
security = securities[(dest.uniqueidtype, dest.uniqueid)]
fromsecurity = securities[(src.uniqueidtype, src.uniqueid)]
# FIXME - exercise cash is sent as INVBANKTRAN; can't get it from
# just the TRANSFERS which are dispatched to here.
tx = ofx.reader.merge_transaction(
session,
type="exercise",
fiaccount=account,
uniqueid=src.fitid,
datetime=src.dttrade,
memo=memo,
security=security,
units=dest.units,
cash=0,
fromfiaccount=None,
fromsecurity=fromsecurity,
fromunits=src.units,
)
return [tx]
def doCashTransactions(
self,
transactions: List[CashTransaction],
session: sqlalchemy.orm.session.Session,
securities: SecuritiesMap,
account: models.FiAccount,
default_currency: str,
) -> List[models.Transaction]:
"""
Preprocess cash transactions and send to merge_retofcap().
The logic here filters only for return of capital transactions;
groups them to apply reversals; nets cash transactions remaining in
each group; and persists to the database after applying any final
preprocessing applied by cash_premerge_hook().
It's important to net cash transactions remaining in a group that
aren't cancelled, since the cash totals of reversing transactions
often don't match the totals of the transactions being reversed
(e.g. partial reversals to recharacterize to/from payment in lieu).
"""
apply_cancels = make_canceller(
filterfunc=self.is_cash_cancel,
matchfunc=lambda x, y: x.total == -1 * y.total,
sortfunc=self.sort_cash_for_cancel,
)
_merge_retofcap = functools.partial(
merge_retofcap,
session=session,
securities=securities,
account=account,
default_currency=default_currency,
)
transactions_ = (
GroupedList(transactions).filter(self.is_retofcap).groupby(
self.fingerprint_cash).bind(apply_cancels).reduce(net_cash).
filter(operator.attrgetter("total")) # Removes net $0 transactions
.map(self.cash_premerge_hook).map(_merge_retofcap).flatten())[:]
return transactions_
def setUp(self):
self.items = GroupedList(range(10))
self.grouped_items = self.items.groupby(lambda x: x % 3)
class GroupedListTestCase(unittest.TestCase):
def setUp(self):
self.items = GroupedList(range(10))
self.grouped_items = self.items.groupby(lambda x: x % 3)
def test_init(self):
# Bare init
instance = GroupedList()
self.assertEqual(len(instance), 0)
self.assertIs(instance.grouped, False)
self.assertIs(instance.key, None)
# init list
instance = GroupedList([])
self.assertEqual(len(instance), 0)
self.assertIs(instance.grouped, False)
self.assertIs(instance.key, None)
def testGroupby(self):
# Flat
items = self.items.groupby(lambda x: x % 2)
self.assertIsInstance(items, GroupedList)
self.assertEqual(items.grouped, True)
self.assertEqual(items.key, None)
self.assertEqual(len(items), 2)
item0, item1 = items
self.assertIsInstance(item0, GroupedList)
self.assertFalse(item0.grouped)
self.assertFalse(item0.key)
self.assertEqual(list(item0), [0, 2, 4, 6, 8])
self.assertIsInstance(item1, GroupedList)
self.assertFalse(item1.grouped)
self.assertTrue(item1.key)
self.assertEqual(list(item1), [1, 3, 5, 7, 9])
# Nested
container = self.grouped_items.groupby(lambda x: x % 2)
self.assertIsInstance(container, GroupedList)
self.assertEqual(container.grouped, True)
self.assertEqual(container.key, None)
self.assertEqual(len(container), 3)
subcontainer0, subcontainer1, subcontainer2 = container
self.assertIsInstance(subcontainer0, GroupedList)
self.assertEqual(subcontainer0.grouped, True)
self.assertEqual(subcontainer0.key, 0)
self.assertEqual(len(subcontainer0), 2)
items0, items1 = subcontainer0
self.assertIsInstance(items0, GroupedList)
self.assertEqual(items0.grouped, False)
self.assertEqual(items0.key, 0)
self.assertEqual(list(items0), [0, 6])
self.assertIsInstance(items1, GroupedList)
self.assertEqual(items1.grouped, False)
self.assertEqual(items1.key, 1)
self.assertEqual(list(items1), [3, 9])
self.assertIsInstance(subcontainer1, GroupedList)
self.assertEqual(subcontainer1.grouped, True)
self.assertEqual(subcontainer1.key, 1)
self.assertEqual(len(subcontainer1), 2)
items0, items1 = subcontainer1
self.assertIsInstance(items0, GroupedList)
self.assertEqual(items0.grouped, False)
self.assertEqual(items0.key, 0)
self.assertEqual(list(items0), [4])
self.assertIsInstance(items1, GroupedList)
self.assertEqual(items1.grouped, False)
self.assertEqual(items1.key, 1)
self.assertEqual(list(items1), [1, 7])
self.assertIsInstance(subcontainer2, GroupedList)
self.assertEqual(subcontainer2.grouped, True)
self.assertEqual(subcontainer2.key, 2)
self.assertEqual(len(subcontainer2), 2)
items0, items1 = subcontainer2
self.assertIsInstance(items0, GroupedList)
self.assertEqual(items0.grouped, False)
self.assertEqual(items0.key, 0)
self.assertEqual(list(items0), [2, 8])
self.assertIsInstance(items1, GroupedList)
self.assertEqual(items1.grouped, False)
self.assertEqual(items1.key, 1)
self.assertEqual(list(items1), [5])
def testFlatten(self):
# Flat
items = self.items.flatten()
self.assertEqual(items, self.items)
# Nested
items = self.grouped_items.flatten()
self.assertIsInstance(items, GroupedList)
self.assertEqual(items.grouped, False)
self.assertEqual(items.key, None)
self.assertEqual(list(items), [0, 3, 6, 9, 1, 4, 7, 2, 5, 8])
def testBind(self):
# Besides other GroupedList methods, the main external direct use of bind()
# is for functions to cancel trades/divs, so we'll use such to test bind().
# Flat
trades = [
Trade(
fitid="3723709320",
dttrade=datetime.datetime(2011, 5, 9, 0, 0),
memo="CONVERA CORPORATION - SPINOFF",
uniqueidtype="CONID",
uniqueid="132118505",
units=Decimal("-0.276942"),
currency="USD",
total=Decimal("0.000002769"),
reportdate=datetime.datetime(2013, 8, 1, 0, 0),
notes=[""],
orig_tradeid=None,
),
Trade(
fitid="3831648707",
dttrade=datetime.datetime(2011, 5, 9, 0, 0),
memo="CONVERA CORPORATION - SPINOFF",
uniqueidtype="CONID",
uniqueid="132118505",
units=Decimal("0.276942"),
currency="USD",
total=Decimal("-0.000002769"),
reportdate=datetime.datetime(2013, 9, 20, 0, 0),
notes=["Ca"],
orig_tradeid=None,
),
Trade(
fitid="3831652905",
dttrade=datetime.datetime(2011, 5, 9, 0, 0),
memo="CONVERA CORPORATION - SPINOFF",
uniqueidtype="CONID",
uniqueid="132118505",
units=Decimal("-0.276942"),
currency="USD",
total=Decimal("56.412710421"),
reportdate=datetime.datetime(2013, 9, 20, 0, 0),
notes=[""],
orig_tradeid=None,
),
Trade(
fitid="3964505548",
dttrade=datetime.datetime(2011, 5, 9, 0, 0),
memo="CONVERA CORPORATION - SPINOFF",
uniqueidtype="CONID",
uniqueid="132118505",
units=Decimal("0.276942"),
currency="USD",
total=Decimal("-56.412710421"),
reportdate=datetime.datetime(2013, 11, 18, 0, 0),
notes=["Ca"],
orig_tradeid=None,
),
Trade(
fitid="3964508206",
dttrade=datetime.datetime(2011, 5, 9, 0, 0),
memo="CONVERA CORPORATION - SPINOFF",
uniqueidtype="CONID",
uniqueid="132118505",
units=Decimal("-0.276942"),
currency="USD",
total=Decimal("1477.3194048"),
reportdate=datetime.datetime(2013, 11, 18, 0, 0),
notes=[""],
orig_tradeid=None,
),
]
net = GroupedList(trades).bind(
make_canceller(
filterfunc=lambda tx: "Ca" in tx.notes,
matchfunc=lambda tx0, tx1: tx0.units == -tx1.units,
sortfunc=None,
)
)
self.assertIsInstance(net, GroupedList)
self.assertFalse(net.grouped)
self.assertIsNone(net.key)
self.assertEqual(len(net), 1)
tx = net[0]
self.assertEqual(tx.fitid, "3964508206")
self.assertEqual(tx.dttrade, datetime.datetime(2011, 5, 9, 0, 0))
self.assertEqual(tx.memo, "CONVERA CORPORATION - SPINOFF")
self.assertEqual(tx.uniqueidtype, "CONID")
self.assertEqual(tx.uniqueid, "132118505")
self.assertEqual(tx.units, Decimal("-0.276942"))
self.assertEqual(tx.currency, "USD")
self.assertEqual(tx.total, Decimal("1477.3194048"))
self.assertEqual(tx.reportdate, datetime.datetime(2013, 11, 18, 0, 0))
self.assertEqual(tx.notes, [""])
# Nested
container = GroupedList(
[GroupedList(key=0), GroupedList(trades, key=1)], grouped=True, key=None
).bind(
make_canceller(
filterfunc=lambda tx: "Ca" in tx.notes,
matchfunc=lambda tx0, tx1: tx0.units == -tx1.units,
sortfunc=None,
)
)
self.assertIsInstance(container, GroupedList)
self.assertTrue(container.grouped)
self.assertIsNone(container.key)
self.assertEqual(len(container), 2)
items0, items1 = container
self.assertIsInstance(items0, GroupedList)
self.assertFalse(items0.grouped)
self.assertEqual(items0.key, 0)
self.assertEqual(len(items0), 0)
self.assertIsInstance(items1, GroupedList)
self.assertFalse(items1.grouped)
self.assertEqual(items1.key, 1)
self.assertEqual(len(items1), 1)
tx = items1[0]
self.assertEqual(tx.fitid, "3964508206")
self.assertEqual(tx.dttrade, datetime.datetime(2011, 5, 9, 0, 0))
self.assertEqual(tx.memo, "CONVERA CORPORATION - SPINOFF")
self.assertEqual(tx.uniqueidtype, "CONID")
self.assertEqual(tx.uniqueid, "132118505")
self.assertEqual(tx.units, Decimal("-0.276942"))
self.assertEqual(tx.currency, "USD")
self.assertEqual(tx.total, Decimal("1477.3194048"))
self.assertEqual(tx.reportdate, datetime.datetime(2013, 11, 18, 0, 0))
self.assertEqual(tx.notes, [""])
def testSort(self):
# Flat
items = self.items.sort(lambda x: -x)
self.assertEqual(list(items), list(range(9, -1, -1)))
# Nested
container = self.grouped_items.sort(lambda x: -x)
self.assertIsInstance(container, GroupedList)
self.assertEqual(container.grouped, True)
self.assertEqual(container.key, None)
self.assertEqual(len(container), 3)
items0, items1, items2 = container
self.assertIsInstance(items0, GroupedList)
self.assertEqual(items0.grouped, False)
self.assertEqual(items0.key, 0)
self.assertEqual(list(items0), [9, 6, 3, 0])
self.assertIsInstance(items1, GroupedList)
self.assertEqual(items1.grouped, False)
self.assertEqual(items1.key, 1)
self.assertEqual(list(items1), [7, 4, 1])
self.assertIsInstance(items2, GroupedList)
self.assertEqual(items2.grouped, False)
self.assertEqual(items2.key, 2)
self.assertEqual(list(items2), [8, 5, 2])
def testFilter(self):
# Flat
items = self.items.filter(lambda x: x % 2)
self.assertIsInstance(items, GroupedList)
self.assertFalse(items.grouped)
self.assertIsNone(items.key)
self.assertEqual(list(items), [1, 3, 5, 7, 9])
# Nested
container = self.grouped_items.filter(lambda x: x % 2)
self.assertIsInstance(container, GroupedList)
self.assertTrue(container.grouped)
self.assertIsNone(container.key)
self.assertEqual(len(container), 3)
subcont0, subcont1, subcont2 = container[:]
self.assertIsInstance(subcont0, GroupedList)
self.assertFalse(subcont0.grouped)
self.assertEqual(subcont0.key, 0)
self.assertEqual(list(subcont0), [3, 9])
self.assertIsInstance(subcont1, GroupedList)
self.assertFalse(subcont1.grouped)
self.assertEqual(subcont1.key, 1)
self.assertEqual(list(subcont1), [1, 7])
self.assertIsInstance(subcont2, GroupedList)
self.assertFalse(subcont1.grouped)
self.assertEqual(subcont2.key, 2)
self.assertEqual(list(subcont2), [5])
def testMap(self):
# Flat
items = self.items.map(lambda x: 2 * x)
self.assertIsInstance(items, GroupedList)
self.assertFalse(items.grouped)
self.assertIsNone(items.key)
self.assertEqual(list(items), [0, 2, 4, 6, 8, 10, 12, 14, 16, 18])
# Nested
container = self.grouped_items.map(lambda x: 2 * x)
self.assertIsInstance(container, GroupedList)
self.assertTrue(container.grouped)
self.assertIsNone(container.key)
self.assertEqual(len(container), 3)
subcont0, subcont1, subcont2 = container[:]
self.assertIsInstance(subcont0, GroupedList)
self.assertFalse(subcont0.grouped)
self.assertEqual(subcont0.key, 0)
self.assertEqual(list(subcont0), [0, 6, 12, 18])
self.assertIsInstance(subcont1, GroupedList)
self.assertFalse(subcont1.grouped)
self.assertEqual(subcont1.key, 1)
self.assertEqual(list(subcont1), [2, 8, 14])
self.assertIsInstance(subcont2, GroupedList)
self.assertFalse(subcont1.grouped)
self.assertEqual(subcont2.key, 2)
self.assertEqual(list(subcont2), [4, 10, 16])
def testReduce(self):
# Flat
items = self.items.reduce(operator.add)
self.assertIsInstance(items, GroupedList)
self.assertFalse(items.grouped)
self.assertIsNone(items.key)
self.assertEqual(len(items), 1)
item = items[0]
self.assertEqual(item, 45)
# Nested
container = self.grouped_items.reduce(operator.add)
self.assertIsInstance(container, GroupedList)
self.assertTrue(container.grouped)
self.assertIsNone(container.key)
self.assertEqual(len(container), 3)
subcont0, subcont1, subcont2 = container[:]
self.assertIsInstance(subcont0, GroupedList)
self.assertFalse(subcont0.grouped)
self.assertEqual(subcont0.key, 0)
self.assertEqual(list(subcont0), [18])
self.assertIsInstance(subcont1, GroupedList)
self.assertFalse(subcont1.grouped)
self.assertEqual(subcont1.key, 1)
self.assertEqual(list(subcont1), [12])
self.assertIsInstance(subcont2, GroupedList)
self.assertFalse(subcont1.grouped)
self.assertEqual(subcont2.key, 2)
self.assertEqual(list(subcont2), [15])
def testBind(self):
# Besides other GroupedList methods, the main external direct use of bind()
# is for functions to cancel trades/divs, so we'll use such to test bind().
# Flat
trades = [
Trade(
fitid="3723709320",
dttrade=datetime.datetime(2011, 5, 9, 0, 0),
memo="CONVERA CORPORATION - SPINOFF",
uniqueidtype="CONID",
uniqueid="132118505",
units=Decimal("-0.276942"),
currency="USD",
total=Decimal("0.000002769"),
reportdate=datetime.datetime(2013, 8, 1, 0, 0),
notes=[""],
orig_tradeid=None,
),
Trade(
fitid="3831648707",
dttrade=datetime.datetime(2011, 5, 9, 0, 0),
memo="CONVERA CORPORATION - SPINOFF",
uniqueidtype="CONID",
uniqueid="132118505",
units=Decimal("0.276942"),
currency="USD",
total=Decimal("-0.000002769"),
reportdate=datetime.datetime(2013, 9, 20, 0, 0),
notes=["Ca"],
orig_tradeid=None,
),
Trade(
fitid="3831652905",
dttrade=datetime.datetime(2011, 5, 9, 0, 0),
memo="CONVERA CORPORATION - SPINOFF",
uniqueidtype="CONID",
uniqueid="132118505",
units=Decimal("-0.276942"),
currency="USD",
total=Decimal("56.412710421"),
reportdate=datetime.datetime(2013, 9, 20, 0, 0),
notes=[""],
orig_tradeid=None,
),
Trade(
fitid="3964505548",
dttrade=datetime.datetime(2011, 5, 9, 0, 0),
memo="CONVERA CORPORATION - SPINOFF",
uniqueidtype="CONID",
uniqueid="132118505",
units=Decimal("0.276942"),
currency="USD",
total=Decimal("-56.412710421"),
reportdate=datetime.datetime(2013, 11, 18, 0, 0),
notes=["Ca"],
orig_tradeid=None,
),
Trade(
fitid="3964508206",
dttrade=datetime.datetime(2011, 5, 9, 0, 0),
memo="CONVERA CORPORATION - SPINOFF",
uniqueidtype="CONID",
uniqueid="132118505",
units=Decimal("-0.276942"),
currency="USD",
total=Decimal("1477.3194048"),
reportdate=datetime.datetime(2013, 11, 18, 0, 0),
notes=[""],
orig_tradeid=None,
),
]
net = GroupedList(trades).bind(
make_canceller(
filterfunc=lambda tx: "Ca" in tx.notes,
matchfunc=lambda tx0, tx1: tx0.units == -tx1.units,
sortfunc=None,
)
)
self.assertIsInstance(net, GroupedList)
self.assertFalse(net.grouped)
self.assertIsNone(net.key)
self.assertEqual(len(net), 1)
tx = net[0]
self.assertEqual(tx.fitid, "3964508206")
self.assertEqual(tx.dttrade, datetime.datetime(2011, 5, 9, 0, 0))
self.assertEqual(tx.memo, "CONVERA CORPORATION - SPINOFF")
self.assertEqual(tx.uniqueidtype, "CONID")
self.assertEqual(tx.uniqueid, "132118505")
self.assertEqual(tx.units, Decimal("-0.276942"))
self.assertEqual(tx.currency, "USD")
self.assertEqual(tx.total, Decimal("1477.3194048"))
self.assertEqual(tx.reportdate, datetime.datetime(2013, 11, 18, 0, 0))
self.assertEqual(tx.notes, [""])
# Nested
container = GroupedList(
[GroupedList(key=0), GroupedList(trades, key=1)], grouped=True, key=None
).bind(
make_canceller(
filterfunc=lambda tx: "Ca" in tx.notes,
matchfunc=lambda tx0, tx1: tx0.units == -tx1.units,
sortfunc=None,
)
)
self.assertIsInstance(container, GroupedList)
self.assertTrue(container.grouped)
self.assertIsNone(container.key)
self.assertEqual(len(container), 2)
items0, items1 = container
self.assertIsInstance(items0, GroupedList)
self.assertFalse(items0.grouped)
self.assertEqual(items0.key, 0)
self.assertEqual(len(items0), 0)
self.assertIsInstance(items1, GroupedList)
self.assertFalse(items1.grouped)
self.assertEqual(items1.key, 1)
self.assertEqual(len(items1), 1)
tx = items1[0]
self.assertEqual(tx.fitid, "3964508206")
self.assertEqual(tx.dttrade, datetime.datetime(2011, 5, 9, 0, 0))
self.assertEqual(tx.memo, "CONVERA CORPORATION - SPINOFF")
self.assertEqual(tx.uniqueidtype, "CONID")
self.assertEqual(tx.uniqueid, "132118505")
self.assertEqual(tx.units, Decimal("-0.276942"))
self.assertEqual(tx.currency, "USD")
self.assertEqual(tx.total, Decimal("1477.3194048"))
self.assertEqual(tx.reportdate, datetime.datetime(2013, 11, 18, 0, 0))
self.assertEqual(tx.notes, [""])