def test_ledger_actions_airdrop():
start_ts = datetime.fromisoformat('2015-01-01').timestamp()
end_ts = datetime.fromisoformat('2019-01-01').timestamp()
buchfink_db = BuchfinkDB(
os.path.join(os.path.dirname(__file__), 'scenarios', 'ledger_actions'))
accountant = buchfink_db.get_accountant()
trades = buchfink_db.get_local_trades_for_account('acc_airdrop')
assert len(trades) == 1
result = accountant.process_history(start_ts, end_ts, trades, [], [], [],
[], [])
assert result['overview']['general_trade_profit_loss'] == '3000'
assert result['overview']['taxable_trade_profit_loss'] == '3000'
assert result['overview']['total_taxable_profit_loss'] == '3000'
ledger_actions = buchfink_db.get_local_ledger_actions_for_account(
'acc_airdrop')
assert len(ledger_actions) == 1
result = accountant.process_history(start_ts, end_ts, trades, [], [], [],
[], ledger_actions)
assert result['overview']['general_trade_profit_loss'] == '2092.35'
assert result['overview']['taxable_trade_profit_loss'] == '2092.35'
assert result['overview']['total_taxable_profit_loss'] == '2092.35'
def fetch(keyword):
"Fetch trades for configured accounts"
buchfink_db = BuchfinkDB()
for account in buchfink_db.get_all_accounts():
if keyword is not None and keyword not in account['name']:
continue
if 'exchange' not in account:
continue
click.echo('Fetching trades for ' + account['name'])
exchange = buchfink_db.get_exchange(account['name'])
api_key_is_valid, error = exchange.validate_api_key()
if not api_key_is_valid:
logger.critical(
'Skipping exchange %s because API key is not valid (%s)',
account['name'], error)
continue
name = account.get('name')
trades = exchange.query_online_trade_history(start_ts=epoch_start_ts,
end_ts=epoch_end_ts)
logger.info('Fetched %d trades from %s', len(trades), name)
with open("trades/" + name + ".yaml", "w") as f:
yaml.dump({"trades": serialize_trades(trades)}, stream=f)
def allowances():
"Show the amount of each asset that you could sell tax-free"
buchfink_db = BuchfinkDB()
num_matched_accounts = 0
all_trades = []
for account in buchfink_db.get_all_accounts():
num_matched_accounts += 1
all_trades.extend(
buchfink_db.get_local_trades_for_account(account['name']))
logger.info('Collected %d trades from %d exchange account(s)',
len(all_trades), num_matched_accounts)
accountant = buchfink_db.get_accountant()
result = accountant.process_history(epoch_start_ts, epoch_end_ts,
all_trades, [], [], [], [])
table = []
for (symbol, (_allowance, buy_price)) in accountant.events.details.items():
table.append([symbol, _allowance, 'TBD', buy_price])
print(
tabulate(table,
headers=[
'Asset', 'Tax-free allowance', 'Tax-free amount',
'Average buy price'
]))
def run_report(buchfink_db: BuchfinkDB, accounts: List[Account],
report_config: ReportConfig):
name = report_config.name
start_ts = report_config.from_dt.timestamp()
end_ts = report_config.to_dt.timestamp()
num_matched_accounts = 0
all_trades = []
all_actions = []
logger.info('Generating report "%s"...', name)
for account in accounts:
num_matched_accounts += 1
all_trades.extend(buchfink_db.get_local_trades_for_account(account))
all_actions.extend(
buchfink_db.get_local_ledger_actions_for_account(account))
logger.info('Collected %d trades / %d actions from %d exchange account(s)',
len(all_trades), len(all_actions), num_matched_accounts)
accountant = buchfink_db.get_accountant()
result = accountant.process_history(start_ts, end_ts, all_trades, [], [],
[], [], all_actions)
accountant.csvexporter.create_files(buchfink_db.reports_directory /
Path(name))
with (buchfink_db.reports_directory / Path(name) /
'report.yaml').open('w') as report_file:
yaml.dump({'overview': result['overview']}, stream=report_file)
logger.info('Report information has been written to: %s',
buchfink_db.reports_directory / Path(name))
if report_config.template:
# Look for templates relative to the data_directory, that is the directory where
# the buchfink.yaml is residing.
env = Environment(loader=FileSystemLoader(buchfink_db.data_directory))
env.globals['datetime'] = datetime
env.globals['float'] = float
env.globals['str'] = str
template = env.get_template(report_config.template)
rendered_report = template.render({
"name": report_config.name,
"title": report_config.title,
"overview": result['overview'],
"events": result['all_events']
})
# we should get ext from template path. could also be json, csv, ...
ext = '.html'
# to save the results
with open(
buchfink_db.reports_directory / Path(name) / ('report' + ext),
"w") as reportf:
reportf.write(rendered_report)
logger.info("Rendered temmplate to 'report%s'.", ext)
return result
def test_ethereum_balances():
start_ts = datetime.fromisoformat('2015-01-01').timestamp()
end_ts = datetime.fromisoformat('2019-01-01').timestamp()
buchfink_db = BuchfinkDB(
os.path.join(os.path.dirname(__file__), 'scenarios', 'ethereum'))
whale = buchfink_db.get_all_accounts()[0]
sheet = buchfink_db.query_balances(whale)
assert sheet.assets['ETH'].amount == FVal('147699.424503407102942053')
def quote(asset: Tuple[str], amount: float, base_asset_: Optional[str], timestamp: Optional[str]):
buchfink_db = BuchfinkDB()
buchfink_db.perform_assets_updates()
base_asset = buchfink_db.get_asset_by_symbol(base_asset_) \
if base_asset_ \
else buchfink_db.get_main_currency()
base_in_usd = FVal(buchfink_db.inquirer.find_usd_price(base_asset))
a_usd = buchfink_db.get_asset_by_symbol('USD')
ds_timestamp = deserialize_timestamp(timestamp) if timestamp else None
historian = PriceHistorian()
for symbol in asset:
asset_ = buchfink_db.get_asset_by_symbol(symbol)
if ds_timestamp:
asset_usd = historian.query_historical_price(
from_asset=asset_,
to_asset=a_usd,
timestamp=ds_timestamp
)
else:
asset_usd = FVal(buchfink_db.inquirer.find_usd_price(asset_))
click.echo('{} {} = {} {}'.format(
click.style(f'{amount}', fg='white'),
click.style(asset_.symbol, fg='green'),
click.style(f'{FVal(amount) * asset_usd / base_in_usd}', fg='white'),
click.style(base_asset.symbol, fg='green')
))
def test_bullrun_config():
buchfink_db = BuchfinkDB(os.path.join(os.path.dirname(__file__), 'scenarios', 'bullrun'))
reports = list(buchfink_db.get_all_reports())
assert len(reports) == 1
report = reports[0]
assert report.name == 'all'
assert report.from_dt.year == 2015
assert report.to_dt.year == 2020
result = run_report(buchfink_db, buchfink_db.get_all_accounts(), report)
assert result['overview']['total_taxable_profit_loss'] == '15000'
def init(directory):
"Initialize new Buchfink directory"
bf_dir = Path(directory).absolute()
logger.debug('Initializing in %s', str(bf_dir))
target_config = bf_dir / 'buchfink.yaml'
init_data = Path(__file__).parent / 'data' / 'init'
if target_config.exists():
click.echo(click.style(
f'Already initialized (buchfink.yaml exists in {bf_dir}), aborting.',
fg='red'
))
sys.exit(1)
for init_file in init_data.iterdir():
logger.debug('Copying %s', init_file.name)
shutil.copyfile(init_file, bf_dir / init_file.name)
buchfink_db = BuchfinkDB(bf_dir)
click.echo(
click.style('Successfully initialized in {0}.'.format(
buchfink_db.data_directory.absolute()
), fg='green')
)
def run(name, from_date, to_date, external):
"Run a full fetch + report cycle"
buchfink_db = BuchfinkDB()
if external:
accounts = [account_from_string(ext, buchfink_db) for ext in external]
else:
accounts = buchfink_db.get_all_accounts()
result = run_report(buchfink_db, accounts, ReportConfig(
name=name,
from_dt=datetime.fromisoformat(from_date),
to_dt=datetime.fromisoformat(to_date)
))
logger.info("Overview: %s", result['overview'])
def test_bullrun_full_taxes():
start_ts = datetime.fromisoformat('2015-01-01').timestamp()
end_ts = datetime.fromisoformat('2019-01-01').timestamp()
buchfink_db = BuchfinkDB(
os.path.join(os.path.dirname(__file__), 'scenarios', 'bullrun'))
trades = buchfink_db.get_local_trades_for_account('exchange1')
assert len(trades) == 2
accountant = buchfink_db.get_accountant()
result = accountant.process_history(start_ts, end_ts, trades, [], [], [],
[], [])
assert result['overview']['general_trade_profit_loss'] == '15000'
assert result['overview']['taxable_trade_profit_loss'] == '15000'
assert result['overview']['total_taxable_profit_loss'] == '15000'
def list_(keyword, account_type, output):
"List accounts"
buchfink_db = BuchfinkDB()
for account in buchfink_db.get_all_accounts():
if keyword is not None and keyword not in account.name:
continue
if account_type is not None and account_type not in account.account_type:
continue
if output is None:
type_and_name = '{0}: {1}'.format(
account.account_type,
click.style(account.name, fg='green')
)
address = ' ({0})'.format(account.address) if account.address is not None else ''
click.echo(type_and_name + address)
else:
click.echo('{0}'.format(getattr(account, output)))
def allowances():
# pylint: disable = W
"Show the amount of each asset that you could sell tax-free"
buchfink_db = BuchfinkDB()
num_matched_accounts = 0
all_trades = []
for account in buchfink_db.get_all_accounts():
num_matched_accounts += 1
all_trades.extend(buchfink_db.get_local_trades_for_account(account.name))
logger.info('Collected %d trades from %d exchange account(s)',
len(all_trades), num_matched_accounts)
accountant = buchfink_db.get_accountant()
currency = buchfink_db.get_main_currency()
currency_in_usd = FVal(buchfink_db.inquirer.find_usd_price(currency))
accountant.process_history(epoch_start_ts, epoch_end_ts, all_trades, [], [], [], [], [])
total_usd = FVal(0)
table = []
raise NotImplementedError()
"""
def trades_(keyword, asset, fetch): # pylint: disable=unused-argument
"Show trades"
buchfink_db = BuchfinkDB()
trades: List[Trade] = []
for account in buchfink_db.get_all_accounts():
if keyword is not None and keyword not in account.name:
continue
trades.extend(buchfink_db.get_local_trades_for_account(account.name))
if asset is not None:
the_asset = buchfink_db.get_asset_by_symbol(asset)
trades = [trade for trade in trades if the_asset in (trade.base_asset, trade.quote_asset)]
trades = sorted(trades, key=attrgetter('timestamp'))
if trades:
table = []
for trade in trades:
table.append([
serialize_timestamp(trade.timestamp),
str(trade.trade_type),
str(trade.amount),
str(trade.base_asset.symbol),
str(trade.amount * trade.rate),
str(trade.quote_asset.symbol),
str(trade.rate),
])
print(tabulate(table, headers=[
'Time',
'Type',
'Amount',
'Quote Asset',
'Amount',
'Base Asset',
'Rate'
]))
def report_(keyword, external, year):
"Generate reports for all report definition and output overview table"
buchfink_db = BuchfinkDB()
results = {}
if external:
accounts = [account_from_string(ext, buchfink_db) for ext in external]
else:
accounts = buchfink_db.get_all_accounts()
if year:
reports = [ReportConfig(
f'adhoc-{_year}',
str(year),
None,
datetime(_year, 1, 1),
datetime(_year + 1, 1, 1)
) for _year in year]
else:
reports = [
report_ for report_ in buchfink_db.get_all_reports()
if keyword is None or keyword in report_.name
]
for report in reports:
name = str(report.name)
results[name] = run_report(buchfink_db, accounts, report)
table = []
for report_name, result in results.items():
table.append([
report_name,
result['overview']['total_profit_loss'],
result['overview']['total_taxable_profit_loss']
])
print(tabulate(table, headers=['Report', 'Profit/Loss', 'Taxable P/L']))
def run(keyword):
"Generate reports for all report definition and output overview table"
buchfink_db = BuchfinkDB()
num_matched_reports = 0
results = {}
for report in buchfink_db.get_all_reports():
name = str(report.name)
if keyword is not None and keyword not in name:
continue
num_matched_reports += 1
results[name] = run_report(buchfink_db, report)
table = []
for report_name, result in results.items():
table.append([
report_name, result['overview']['total_profit_loss'],
result['overview']['total_taxable_profit_loss']
])
print(tabulate(table, headers=['Report', 'Profit/Loss', 'Taxable P/L']))
def report(name, from_, to):
"Run an ad-hoc report on your data"
start_ts = datetime.fromisoformat(from_).timestamp()
end_ts = datetime.fromisoformat(to).timestamp()
buchfink_db = BuchfinkDB()
result = run_report(
buchfink_db,
ReportConfig(name=name,
from_dt=datetime.fromisoformat(from_),
to_dt=datetime.fromisoformat(to)))
logger.info("Overview: %s", result['overview'])
def init(directory):
"Initialize new Buchfink directory"
target_config = os.path.join(directory, 'buchfink.yaml')
if os.path.exists(target_config):
click.echo(
click.style(
'Already initialized (buchfink.yaml exists), aborting.',
fg='red'))
return
initial_config = os.path.join(os.path.dirname(__file__), 'data',
'buchfink.initial.yaml')
shutil.copyfile(initial_config, target_config)
buchfink_db = BuchfinkDB(directory)
click.echo(
click.style('Successfully initialized in {0}.'.format(
buchfink_db.data_directory.absolute()),
fg='green'))
def balances(keyword):
"Show balances across all accounts"
buchfink_db = BuchfinkDB()
balances_sum = {}
usd_value_sum = {}
for account in buchfink_db.get_all_accounts():
if keyword is not None and keyword not in account['name']:
continue
if 'exchange' in account:
exchange = buchfink_db.get_exchange(account['name'])
api_key_is_valid, error = exchange.validate_api_key()
if not api_key_is_valid:
logger.critical(
'Skipping exchange %s because API key is not valid (%s)',
account['name'], error)
continue
balances, error = exchange.query_balances()
if not error:
logger.info('Fetched balances for %d assets from %s',
len(balances.keys()), account['name'])
for asset, balance in balances.items():
amount = balance['amount']
balances_sum[asset] = balances_sum.get(asset,
FVal(0)) + amount
if 'usd_value' in balance:
usd_value_sum[asset] = usd_value_sum.get(
asset, FVal(0)) + balance['usd_value']
elif 'ethereum' in account:
manager = buchfink_db.get_chain_manager(account)
manager.query_balances()
for eth_balance in manager.balances.eth.values():
for asset, balance in eth_balance.asset_balances.items():
amount = balance.amount
balances_sum[asset] = balances_sum.get(asset,
FVal(0)) + amount
usd_value_sum[asset] = usd_value_sum.get(
asset, FVal(0)) + balance.usd_value
elif 'bitcoin' in account:
manager = buchfink_db.get_chain_manager(account)
manager.query_balances()
asset = Asset('BTC')
for balance in manager.balances.btc.values():
amount = balance.amount
balances_sum[asset] = balances_sum.get(asset, FVal(0)) + amount
usd_value_sum[asset] = usd_value_sum.get(
asset, FVal(0)) + balance.usd_value
elif 'file' in account:
account = yaml.load(open(account['file'], 'r'),
Loader=yaml.SafeLoader)
if 'balances' in account:
for balance in account['balances']:
amount = FVal(balance['amount'])
asset = Asset(balance['asset'])
usd_value = amount * buchfink_db.inquirer.find_usd_price(
asset)
balances_sum[asset] = balances_sum.get(asset,
FVal(0)) + amount
usd_value_sum[asset] = usd_value_sum.get(
asset, FVal(0)) + usd_value
currency = buchfink_db.get_main_currency()
currency_in_usd = buchfink_db.inquirer.find_usd_price(currency)
table = []
assets = [
obj[0] for obj in sorted(
usd_value_sum.items(), key=itemgetter(1), reverse=True)
]
balance_in_currency_sum = 0
for asset in assets:
balance = balances_sum[asset]
balance_in_currency = usd_value_sum.get(asset,
FVal(0)) / currency_in_usd
balance_in_currency_sum += balance_in_currency
table.append([
asset, balance, asset.symbol,
round(float(balance_in_currency), 2)
])
table.append(
['Total', None, None,
round(float(balance_in_currency_sum), 2)])
print(
tabulate(table,
headers=[
'Asset', 'Amount', 'Symbol',
'Fiat Value (%s)' % currency.symbol
]))
def test_custom_ethereum_token():
buchfink_db = BuchfinkDB(
os.path.join(os.path.dirname(__file__), 'scenarios', 'custom_token'))
buchfink_db.perform_assets_updates()
assert buchfink_db.get_asset_by_symbol('IMX') is not None
def balances(keyword, minimum_balance, fetch, total, external):
"Show balances across all accounts"
buchfink_db = BuchfinkDB()
assets_sum = {}
assets_usd_sum = {}
liabilities_sum = {}
liabilities_usd_sum = {}
if external:
accounts = [account_from_string(ext, buchfink_db) for ext in external]
else:
accounts = buchfink_db.get_all_accounts()
for account in accounts:
if keyword is not None and keyword not in account.name:
continue
if fetch:
buchfink_db.perform_assets_updates()
buchfink_db.fetch_balances(account)
sheet = buchfink_db.get_balances(account)
for asset, balance in sheet.assets.items():
amount = balance.amount
assets_sum[asset] = assets_sum.get(asset, FVal(0)) + amount
assets_usd_sum[asset] = assets_usd_sum.get(asset, FVal(0)) + balance.usd_value
for liability, balance in sheet.liabilities.items():
amount = balance.amount
liabilities_sum[liability] = liabilities_sum.get(liability, FVal(0)) + amount
liabilities_usd_sum[liability] = liabilities_usd_sum.get(liability, FVal(0)) \
+ balance.usd_value
currency = buchfink_db.get_main_currency()
currency_in_usd = FVal(buchfink_db.inquirer.find_usd_price(currency))
table = []
assets = [obj[0] for obj in sorted(assets_usd_sum.items(), key=itemgetter(1), reverse=True)]
balance_in_currency_sum = 0
small_balances_sum = 0
for asset in assets:
balance = assets_sum[asset]
balance_in_currency = FVal(assets_usd_sum.get(asset, 0)) / currency_in_usd
if balance > ZERO:
if balance_in_currency > FVal(minimum_balance):
table.append([
asset.name,
balance,
asset.symbol,
round(float(balance_in_currency), 2)
])
else:
small_balances_sum += balance_in_currency
balance_in_currency_sum += balance_in_currency
if total:
print(f'Total assets: {round(float(balance_in_currency_sum), 2)} {currency.symbol}')
else:
if small_balances_sum > 0:
table.append(['Others', None, None, round(float(small_balances_sum), 2)])
table.append(['Total', None, None, round(float(balance_in_currency_sum), 2)])
print(tabulate(table, headers=[
'Asset',
'Amount',
'Symbol',
'Fiat Value (%s)' % currency.symbol
]))
if liabilities_sum:
table = []
balance_in_currency_sum = 0
assets = [
obj[0]
for obj
in sorted(liabilities_usd_sum.items(), key=itemgetter(1), reverse=True)
]
for asset in assets:
balance = liabilities_sum[asset]
balance_in_currency = liabilities_usd_sum.get(asset, FVal(0)) / currency_in_usd
if balance > ZERO and balance_in_currency >= FVal(minimum_balance):
balance_in_currency_sum += balance_in_currency
table.append([
asset.name,
balance,
asset.symbol,
round(float(balance_in_currency), 2)
])
table.append(['Total', None, None, round(float(balance_in_currency_sum), 2)])
if total:
print(
f'Total liabilities: '
f'{round(float(balance_in_currency_sum), 2)} {currency.symbol}'
)
else:
print()
print(tabulate(table, headers=[
'Liability',
'Amount',
'Symbol',
'Fiat Value (%s)' % currency.symbol
]))
def fetch_(keyword, account_type, fetch_actions, fetch_balances, fetch_trades, external):
"Fetch trades for configured accounts"
buchfink_db = BuchfinkDB()
buchfink_db.perform_assets_updates()
now = ts_now()
fetch_limited = fetch_actions or fetch_balances or fetch_trades
if external:
accounts = [account_from_string(ext, buchfink_db) for ext in external]
else:
accounts = buchfink_db.get_all_accounts()
# TODO: This should move to BuchfinkDB.get_accounts()
if keyword is not None:
if keyword.startswith('/') and keyword.endswith('/'):
keyword_re = re.compile(keyword[1:-1])
accounts = [acc for acc in accounts if keyword_re.search(acc.name)]
else:
accounts = [acc for acc in accounts if keyword in acc.name]
logger.info(
'Collected %d account(s): %s',
len(accounts),
', '.join([acc.name for acc in accounts])
)
for account in accounts:
if account_type is not None and account_type not in account.account_type:
continue
name = account.name
trades = []
actions = []
if account.account_type == "ethereum":
if not fetch_limited or fetch_actions:
logger.info('Analyzing ethereum transactions for %s', name)
manager = buchfink_db.get_chain_manager(account)
txs = manager.ethereum.transactions.single_address_query_transactions(
account.address,
start_ts=0,
end_ts=now,
with_limit=False,
only_cache=False
)
for txn in txs:
tx_hash = '0x' + txn.tx_hash.hex()
receipt = buchfink_db.get_ethereum_transaction_receipt(tx_hash, manager)
acc_actions = classify_tx(account, tx_hash, txn, receipt)
if actions:
for act in actions:
logger.debug('Found action: %s', act)
actions.extend(acc_actions)
if not fetch_limited or fetch_trades:
logger.info('Fetching uniswap trades for %s', name)
manager = buchfink_db.get_chain_manager(account)
trades = manager.eth_modules['uniswap'].get_trades(
addresses=manager.accounts.eth,
from_timestamp=int(epoch_start_ts),
to_timestamp=int(epoch_end_ts),
only_cache=False
)
elif account.account_type == "exchange":
if not fetch_limited or fetch_trades:
logger.info('Fetching exhange trades for %s', name)
exchange = buchfink_db.get_exchange(name)
api_key_is_valid, error = exchange.validate_api_key()
if not api_key_is_valid:
logger.critical(
'Skipping exchange %s because API key is not valid (%s)',
account.name,
error
)
else:
trades = exchange.query_online_trade_history(
start_ts=epoch_start_ts,
end_ts=epoch_end_ts
)
else:
logger.debug('No way to retrieve trades for %s, yet', name)
annotations_path = "annotations/" + name + ".yaml"
if not fetch_limited or fetch_actions:
if os.path.exists(annotations_path):
annotated = buchfink_db.get_actions_from_file(annotations_path)
else:
annotated = []
logger.info('Fetched %d action(s) (%d annotated) from %s',
len(actions) + len(annotated), len(annotated), name)
actions.extend(annotated)
if actions:
with open("actions/" + name + ".yaml", "w") as yaml_file:
yaml.dump({
"actions": serialize_ledger_actions(actions)
}, stream=yaml_file, sort_keys=True)
if not fetch_limited or fetch_trades:
if os.path.exists(annotations_path):
annotated = buchfink_db.get_trades_from_file(annotations_path)
else:
annotated = []
logger.info('Fetched %d trades(s) (%d annotated) from %s',
len(trades) + len(annotated), len(annotated), name)
trades.extend(annotated)
with open("trades/" + name + ".yaml", "w") as yaml_file:
yaml.dump({
"trades": serialize_trades(trades)
}, stream=yaml_file, sort_keys=True)
if not fetch_limited or fetch_balances:
buchfink_db.fetch_balances(account)
logger.info('Fetched balances from %s', name)