def donate():
counter_file = os.path.join(home_path(), 'warden/counter.json')
templateData = {
"title": "Support this Software",
"current_app": current_app
}
return (render_template('warden/warden_thanks.html', **templateData))
def pickle_it(action='load', filename=None, data=None):
logging.info(f"Pickle {action} file: {filename}")
from warden_modules import home_path
filename = 'warden/' + filename
filename = os.path.join(home_path(), filename)
if action == 'delete':
try:
os.remove(filename)
return ('deleted')
except Exception:
return ('failed')
if action == 'load':
try:
with open(filename, 'rb') as handle:
ld = pickle.load(handle)
logging.info(f"Loaded: Pickle {action} file: {filename}")
return (ld)
except Exception as e:
logging.warning(
f"Error: Pickle {action} file: {filename} error:{e}")
return ("file not found")
else:
with open(filename, 'wb') as handle:
pickle.dump(data, handle, protocol=pickle.HIGHEST_PROTOCOL)
logging.info(f"Saved: Pickle {action} file: {filename}")
return ("saved")
def donated():
counter_file = os.path.join(home_path(),
'warden/counter.json')
templateData = {"title": "Thank You!", "donated": donate_check(), "current_app": current_app}
with open(counter_file, 'w') as fp:
json.dump("donated", fp)
return (render_template('warden/warden_thanks.html', **templateData))
def donate_check():
counter_file = os.path.join(home_path(),
'warden/counter.json')
donated = False
try:
with open(counter_file) as data_file:
json_all = json.loads(data_file.read())
if json_all == "donated":
donated = True
except Exception:
donated = False
return (donated)
def start_hidden_service(app):
if app.controller is None:
return
app.controller.reconnect()
key_path = os.path.abspath(os.path.join(home_path(), ".tor_service_key"))
app.tor_service_id = None
if not os.path.exists(key_path):
service = app.controller.create_ephemeral_hidden_service(
{app.tor_port: app.port}, await_publication=True)
app.tor_service_id = service.service_id
print("* Started a new hidden service with the address of %s.onion" %
app.tor_service_id)
with open(key_path, "w") as key_file:
key_file.write("%s:%s" %
(service.private_key_type, service.private_key))
else:
with open(key_path) as key_file:
key_type, key_content = key_file.read().split(":", 1)
try:
service = app.controller.create_ephemeral_hidden_service(
{app.tor_port: app.port},
key_type=key_type,
key_content=key_content,
await_publication=True,
)
app.tor_service_id = service.service_id
print("")
print(success("✅ Resumed %s.onion" % app.tor_service_id))
from utils import pickle_it
pickle_it('save', 'onion_address.pkl',
app.tor_service_id + '.onion')
print("")
except Exception:
pass
try:
# save address to file
if app.save_tor_address_to is not None:
with open(app.save_tor_address_to, "w") as f:
f.write("%s.onion" % app.tor_service_id)
app.tor_service_id = app.tor_service_id
from utils import pickle_it
pickle_it('save', 'onion_address.pkl', app.tor_service_id + '.onion')
app.tor_enabled = True
except Exception:
pass
def tor_services():
action = request.args.get("action")
if action == 'start':
current_app.settings['SERVER']['onion_server'] = 'True'
update_config()
from stem.control import Controller
from urllib.parse import urlparse
current_app.tor_port = current_app.settings['SERVER'].getint('onion_port')
current_app.port = current_app.settings['SERVER'].getint('port')
from warden_modules import home_path
toraddr_file = os.path.join(home_path(), "onion.txt")
current_app.save_tor_address_to = toraddr_file
proxy_url = "socks5h://localhost:9050"
tor_control_port = ""
try:
tor_control_address = urlparse(proxy_url).netloc.split(":")[0]
if tor_control_address == "localhost":
tor_control_address = "127.0.0.1"
current_app.controller = Controller.from_port(
address=tor_control_address,
port=int(tor_control_port)
if tor_control_port
else "default",
)
except Exception:
current_app.controller = None
from tor import start_hidden_service
start_hidden_service(current_app)
flash(f"Started Tor Hidden Services at {current_app.tor_service_id}.onion", "success")
if action == 'stop':
current_app.settings['SERVER']['onion_server'] = 'False'
update_config()
from tor import stop_hidden_services
stop_hidden_services(current_app)
flash("Stopped Tor Hidden Services", "warning")
return render_template("warden/tor.html",
title="Tor Hidden Services",
current_app=current_app)
def historical_prices(ticker, fx='USD', source=None):
'''
RETURNS a DF with
columns={
'close': 'close',
'open': 'open',
'high': 'high',
'low': 'low',
'volume': 'volume'
}
'''
ticker = ticker.replace(' ', '')
if source and type(source) != list:
raise TypeError("source has to be a list of strings - can be one string inside a list")
try:
source_list = mapping[ticker]
except KeyError:
source_list = [
'cryptocompare',
'twelvedata',
'alphavantage_currency',
'alphavantage_global',
'fmp'
]
from pricing_engine.alphavantage import historical as aa_historical
from pricing_engine.cryptocompare import historical as cc_historical
from pricing_engine.fmp import historical as fmp_historical
from pricing_engine.twelvedata import historical as td_historical
results = pd.DataFrame()
# Gets from each source
for src in source_list:
# Try to load file if exists
filename = (ticker + "_" + fx + ".price")
# Check if file was updated today
from warden_modules import home_path
file_check = os.path.join(home_path(), 'warden/' + filename)
# Try to read from file and check how recent it is
try:
today = datetime.now().date()
filetime = datetime.fromtimestamp(os.path.getctime(file_check))
if filetime.date() == today:
df = pd.read_pickle(file_check)
return (df)
except Exception:
pass
if src == 'alphavantage_currency':
results = aa_historical(ticker, function='DIGITAL_CURRENCY_DAILY')
if src == 'alphavantage_global':
results = aa_historical(ticker, function='TIME_SERIES_DAILY_ADJUSTED')
if src == 'alphavantage_fx':
results = aa_historical(ticker, function='FX_DAILY')
if src == 'cryptocompare':
results = cc_historical(ticker)
if src == 'fmp':
results = fmp_historical(ticker)
if src == 'twelvedata':
results = td_historical(ticker)
# Check if data is valid
if not results.empty:
# Include fx column and convert to currency if needed
if fx != 'USD':
# Get a currency df
df_fx = aa_historical(fx, function='FX_DAILY')
df_fx.index = pd.to_datetime(df_fx.index)
df_fx = df_fx.loc[~df_fx.index.duplicated(keep='first')]
df_fx = df_fx.rename(columns={'close': 'fx_close'})
df_fx = df_fx[['fx_close']]
df_fx['fx_close'] = pd.to_numeric(df_fx.fx_close,
errors='coerce')
df_fx['fx_close'] = 1 / df_fx['fx_close']
# Merge the two dfs:
results.index = pd.to_datetime(results.index)
results = results.loc[~results.index.duplicated(keep='first')]
merge_df = pd.merge(results, df_fx, on='date', how='inner')
merge_df['close'] = merge_df['close'].astype(float)
merge_df['close_converted'] = merge_df['close'] * merge_df[
'fx_close']
results = merge_df
else:
results['fx_close'] = 1
results['close_converted'] = pd.to_numeric(results.close,
errors='coerce')
results.index = results.index.astype('datetime64[ns]')
# Save this file to be used during the same day instead of calling API
pickle_it(action='save', filename=filename, data=results)
# save metadata as well
metadata = {
'source': src,
'last_update': datetime.utcnow()
}
filemeta = (ticker + "_" + fx + ".meta")
pickle_it(action='save', filename=filemeta, data=metadata)
return (results)
else:
logging.info(f"Source {src} does not return any data for {ticker}. Trying other sources.")
if results.empty:
logging.warning(f"Could not retrieve a df for {ticker} from any source")
return (results)
def init_app(app):
from ansi_management import (warning, success, error)
from utils import (create_config, runningInDocker)
from config import Config
from connections import tor_request
warnings.filterwarnings('ignore')
# Load config.ini into app
# --------------------------------------------
# Read Global Variables from warden.config(s)
# Can be accessed like a dictionary like:
# app.settings['PORTFOLIO']['RENEW_NAV']
# --------------------------------------------
config_file = Config.config_file
app.warden_status = {}
# Check for internet connection
internet_ok = internet_connected()
if internet_ok is True:
print(success("✅ Internet Connection"))
else:
print(
error(
"[!] WARden needs internet connection. Check your connection.")
)
print(warning("[!] Exiting"))
exit()
# Config
config_settings = configparser.ConfigParser()
if os.path.isfile(config_file):
config_settings.read(config_file)
app.warden_status['initial_setup'] = False
print(
success(
"✅ Config Loaded from config.ini - edit it for customization"
))
else:
print(
error(
" Config File could not be loaded, created a new one with default values..."
))
create_config(config_file)
config_settings.read(config_file)
app.warden_status['initial_setup'] = True
table_error = False
try:
# create empty instance of LoginManager
app.login_manager = LoginManager()
except sqlite3.OperationalError:
table_error = True
# Create empty instance of SQLAlchemy
app.db = SQLAlchemy()
app.db.init_app(app)
# Import models so tables are created
from models import Trades, User, AccountInfo, TickerInfo, SpecterInfo
app.db.create_all()
# There was an initial error on getting users
# probably because tables were not created yet.
# The above create_all should have solved it so try again.
if table_error:
# create empty instance of LoginManager
app.login_manager = LoginManager()
# If login required - go to login:
app.login_manager.login_view = "warden.login"
# To display messages - info class (Bootstrap)
app.login_manager.login_message_category = "secondary"
app.login_manager.init_app(app)
# Create empty instance of messagehandler
from message_handler import MessageHandler
app.message_handler = MessageHandler()
app.message_handler.clean_all()
# Get Version
print("")
try:
version_file = Config.version_file
with open(version_file, 'r') as file:
current_version = file.read().replace('\n', '')
except Exception:
current_version = 'unknown'
with app.app_context():
app.version = current_version
# Check if there are any users on database, if not, needs initial setup
users = User.query.all()
if users == []:
app.warden_status['initial_setup'] = True
# Check for Cryptocompare API Keys
print("")
check_cryptocompare()
print("")
print(f"[i] Running WARden version: {current_version}")
app.warden_status['running_version'] = current_version
# CHECK FOR UPGRADE
repo_url = 'https://api.github.com/repos/pxsocs/warden/releases'
try:
github_version = tor_request(repo_url).json()[0]['tag_name']
except Exception:
github_version = None
app.warden_status['github_version'] = github_version
if github_version:
print(f"[i] Newest WARden version available: {github_version}")
parsed_github = version.parse(github_version)
parsed_version = version.parse(current_version)
app.warden_status['needs_upgrade'] = False
if parsed_github > parsed_version:
print(warning(" [i] Upgrade Available"))
app.warden_status['needs_upgrade'] = True
if parsed_github == parsed_version:
print(success("✅ You are running the latest version"))
else:
print(warning("[!] Could not check GitHub for updates"))
print("")
# Check if config.ini exists
with app.app_context():
app.settings = config_settings
with app.app_context():
try:
from utils import fxsymbol
app.fx = fxsymbol(config_settings['PORTFOLIO']['base_fx'], 'all')
except KeyError: # Problem with this config, reset
print(error(" [!] Config File needs to be rebuilt"))
print("")
create_config(config_file)
# TOR Server through Onion Address --
# USE WITH CAUTION - ONION ADDRESSES CAN BE EXPOSED!
# WARden needs to implement authentication (coming soon)
if app.settings['SERVER'].getboolean('onion_server'):
from stem.control import Controller
from urllib.parse import urlparse
app.tor_port = app.settings['SERVER'].getint('onion_port')
app.port = app.settings['SERVER'].getint('port')
from warden_modules import home_path
toraddr_file = os.path.join(home_path(), "onion.txt")
app.save_tor_address_to = toraddr_file
proxy_url = "socks5h://localhost:9050"
tor_control_port = ""
try:
tor_control_address = urlparse(proxy_url).netloc.split(":")[0]
if tor_control_address == "localhost":
tor_control_address = "127.0.0.1"
app.controller = Controller.from_port(
address=tor_control_address,
port=int(tor_control_port) if tor_control_port else "default",
)
except Exception:
app.controller = None
from tor import start_hidden_service
start_hidden_service(app)
from routes import warden
from errors.handlers import errors
from api.routes import api
from csv_routes.routes import csv_routes
from user_routes.routes import user_routes
from simulator.routes import simulator
app.register_blueprint(warden)
app.register_blueprint(errors)
app.register_blueprint(api)
app.register_blueprint(csv_routes)
app.register_blueprint(user_routes)
app.register_blueprint(simulator)
# Prepare app to receive Specter Server info
# For the first load, just get a saved file if available
# The background jobs will update later
with app.app_context():
from specter_importer import Specter
app.specter = Specter()
app.specter.refresh_txs(load=True)
app.downloading = False
with app.app_context():
app.runningInDocker = runningInDocker()
with app.app_context():
app.tor = create_tor()
# Check if home folder exists, if not create
home = str(Path.home())
home_path = os.path.join(home, 'warden/')
try:
os.makedirs(os.path.dirname(home_path))
except Exception:
pass
# Start Schedulers
from backgroundjobs import (background_settings_update,
background_specter_update,
background_scan_network,
background_specter_health,
background_mempool_seeker)
def bk_su():
with app.app_context():
background_specter_update()
def bk_stu():
with app.app_context():
background_settings_update()
def bk_scan():
with app.app_context():
background_scan_network()
def bk_specter_health():
with app.app_context():
background_specter_health()
def bk_mempool_health():
with app.app_context():
background_mempool_seeker()
app.scheduler = BackgroundScheduler()
app.scheduler.add_job(bk_su, 'interval', seconds=1)
app.scheduler.add_job(bk_stu, 'interval', seconds=1)
app.scheduler.add_job(bk_scan, 'interval', seconds=1)
app.scheduler.add_job(bk_specter_health, 'interval', seconds=1)
app.scheduler.add_job(bk_mempool_health, 'interval', seconds=1)
app.scheduler.start()
print(success("✅ Background jobs running"))
print("")
app.app_context().push()
print(success("✅ Application startup is complete"))
return app
def warden_page():
# For now pass only static positions, will update prices and other
# data through javascript after loaded. This improves load time
# and refresh speed.
# Get positions and prepare df for delivery
try:
df = positions()
except Exception:
flash("Seems like transactions may be downloading... Check Specter Transactions for status.", "warning")
return redirect(url_for("warden.newtrade"))
if df.empty:
flash("No Transactions Found. You can start by including a transaction below. You may also Connect to Specter or import a CSV file.", "info")
return redirect(url_for("warden.newtrade"))
if df.index.name != 'trade_asset_ticker':
df.set_index('trade_asset_ticker', inplace=True)
df = df[df['is_currency'] == 0].sort_index(ascending=True)
df = df.to_dict(orient='index')
# Create a custody DF
transactions = transactions_fx()
custody_df = transactions.groupby(["trade_account", "trade_asset_ticker"
])[["trade_quantity"]].sum()
# Open Counter, increment, send data
counter_file = os.path.join(home_path(),
'warden/counter.json')
donated = False
try:
with open(counter_file) as data_file:
json_all = json.loads(data_file.read())
if json_all == "donated":
donated = True
else:
counter = int(json_all)
counter += 1
if counter == 25:
flash(
"Looks like you've been using the app frequently. " +
"Awesome! Consider donating.", "info")
if counter == 50:
flash(
"Open Source software is transparent and free. " +
"Support it. Make a donation.", "info")
if counter == 200:
flash(
"Looks like you are a frequent user of the WARden. " +
"Have you donated?", "info")
if counter >= 1000:
flash(
"You've opened this page 1,000 times or more. " +
"Really! Time to make a donation?", "danger")
with open(counter_file, 'w') as fp:
json.dump(counter, fp)
except Exception:
# File wasn't found. Create start at zero
if not donated:
flash(
"Welcome. Consider making a donation " +
"to support this software.", "info")
counter = 0
with open(counter_file, 'w') as fp:
json.dump(counter, fp)
meta = warden_metadata()
# Sort the wallets by balance
sorted_wallet_list = []
try:
for wallet in current_app.specter.wallet_alias_list():
wallet_df = meta['full_df'].loc[meta['full_df']['wallet_alias'] == wallet]
if wallet_df.empty:
balance = 0
else:
balance = wallet_df['amount'].sum()
sorted_wallet_list.append((wallet, balance))
sorted_wallet_list = sorted(sorted_wallet_list, reverse=True, key=itemgetter(1))
sorted_wallet_list = [i[0] for i in sorted_wallet_list]
wallets_exist = True
except Exception:
wallets_exist = False
from api.routes import alert_activity
if not current_app.downloading:
activity = alert_activity()
else:
activity = False
templateData = {
"title": "Portfolio Dashboard",
"warden_metadata": meta,
"portfolio_data": df,
"FX": current_app.settings['PORTFOLIO']['base_fx'],
"donated": donated,
"alerts": activity,
"current_app": current_app,
"sorted_wallet_list": sorted_wallet_list,
"wallets_exist": wallets_exist,
"custody_df": custody_df
}
return (render_template('warden/warden.html', **templateData))
def test_paths(self):
print("Checking that current and home paths can be returned...")
print(f"Current Path: {current_path()}")
print(f"Home Path: {home_path()}")
self.assertIsNotNone(current_path())
self.assertIsNotNone(home_path())