def api_status():
# Doc: https://bitbucket.org/mineboxgmbh/minebox-client-tools/src/master/doc/mb-ui-gateway-api.md#markdown-header-get-status
username = check_login()
outdata = {}
if username:
outdata["logged_in"] = True
outdata["user"] = username
else:
outdata["logged_in"] = False
outdata["user"] = None
mbdata, mb_status_code = get_from_minebd('status')
if mb_status_code == 200:
outdata["minebd_running"] = True
outdata["minebd_encrypted"] = mbdata["hasEncryptionKey"]
outdata["minebd_storage_mounted"] = ismount(MINEBD_STORAGE_PATH)
outdata["restore_running"] = mbdata["restoreRunning"]
outdata["restore_progress"] = mbdata["completedRestorePercent"]
else:
outdata["minebd_running"] = False
outdata["minebd_encrypted"] = None
outdata["minebd_storage_mounted"] = False
outdata["restore_running"] = False
outdata["restore_progress"] = None
hasusers = False
for user in pwd.getpwall():
if user.pw_uid >= 1000 and user.pw_uid < 65500 and user.pw_name != "sia":
hasusers = True
outdata["users_created"] = hasusers
if 'DEMO' in environ:
outdata["backup_type"] = "sia_demo"
else:
outdata["backup_type"] = "sia"
consdata, cons_status_code = get_from_sia('consensus')
if cons_status_code == 200:
outdata["sia_daemon_running"] = True
outdata["consensus_height"] = consdata["height"]
outdata["consensus_synced"] = consdata["synced"]
else:
outdata["sia_daemon_running"] = False
outdata["consensus_height"] = None
outdata["consensus_synced"] = None
walletdata, wallet_status_code = get_from_sia('wallet')
if username and wallet_status_code == 200:
outdata["wallet_unlocked"] = walletdata["unlocked"]
outdata["wallet_encrypted"] = walletdata["encrypted"]
outdata["wallet_confirmed_balance_sc"] = int(walletdata["confirmedsiacoinbalance"]) / H_PER_SC
outdata["wallet_unconfirmed_delta_sc"] = (int(walletdata["unconfirmedincomingsiacoins"]) -
int(walletdata["unconfirmedoutgoingsiacoins"])) / H_PER_SC
else:
outdata["wallet_unlocked"] = None
outdata["wallet_encrypted"] = None
outdata["wallet_confirmed_balance_sc"] = None
outdata["wallet_unconfirmed_delta_sc"] = None
return jsonify(outdata), 200
def update_sia_config():
settings = get_sia_config()
# Renting settings
siadata, sia_status_code = get_from_sia("renter")
if sia_status_code >= 400:
# If we can't get the current settings, no use in comparing to new ones.
return False
renter_params = {}
# If new settings values differ by at least 10% from currently set values,
# only then update Sia with the new settings.
if _absdiff(settings["renter"]["allowance_funds"],
int(siadata["settings"]["allowance"]["funds"])) > 0.1:
renter_params["funds"] = str(settings["renter"]["allowance_funds"])
if _absdiff(settings["renter"]["allowance_period"],
int(siadata["settings"]["allowance"]["period"])) > 0.1:
renter_params["period"] = str(settings["renter"]["allowance_period"])
if renter_params:
current_app.logger.info("Updating Sia renter/allowance settings.")
siadata, sia_status_code = post_to_sia("renter", renter_params)
if sia_status_code >= 400:
current_app.logger.error("Sia error %s: %s" %
(sia_status_code, siadata["message"]))
return False
# Hosting settings
siadata, sia_status_code = get_from_sia('host')
if sia_status_code >= 400:
# If we can't get the current settings, no use in comparing to new ones.
return False
if (not siadata["internalsettings"]["acceptingcontracts"]
and not settings["minebox_sharing"]["enabled"]):
# If hosting is deactivated, pings will call setup_sia_system()
# This will care about settings so we don't do anything here.
return True
host_params = {}
if settings["minebox_sharing"]["enabled"] != siadata["internalsettings"][
"acceptingcontracts"]:
host_params["acceptingcontracts"] = settings["minebox_sharing"][
"enabled"]
for var in [
"mincontractprice", "mindownloadbandwidthprice", "minstorageprice",
"minuploadbandwidthprice", "collateral", "collateralbudget",
"maxcollateral", "maxduration"
]:
if _absdiff(settings["host"][var], int(
siadata["internalsettings"][var])) > 0.1:
host_params[var] = str(settings["host"][var])
if host_params:
current_app.logger.info("Updating Sia host settings.")
siadata, sia_status_code = post_to_sia("host", host_params)
if sia_status_code >= 400:
current_app.logger.error("Sia error %s: %s" %
(sia_status_code, siadata["message"]))
return False
# We're done here :)
return True
def api_contracts():
# Doc: https://bitbucket.org/mineboxgmbh/minebox-client-tools/src/master/doc/mb-ui-gateway-api.md#markdown-header-get-contracts
if not check_login():
return jsonify(
message="Unauthorized access, please log into the main UI."), 401
siadata, sia_status_code = get_from_sia('renter/contracts')
if sia_status_code >= 400:
return jsonify(siadata), sia_status_code
# Create a summary similar to what `siac renter contracts` presents.
# We could expose the full details of a contract in a different route, e.g. /contract/<id>.
contractlist = []
for contract in siadata["contracts"]:
contractlist.append({
"id":
contract["id"],
"host":
contract["netaddress"],
"funds_remaining_sc":
int(contract["renterfunds"]) / H_PER_SC,
"funds_spent_sc":
(int(contract["StorageSpending"]) + int(contract["uploadspending"])
+ int(contract["downloadspending"])) / H_PER_SC,
"fees_spent_sc":
int(contract["fees"]) / H_PER_SC,
"totalcost_sc":
int(contract["totalcost"]) / H_PER_SC,
"data_size":
contract["size"],
"height_end":
contract["endheight"],
"esttime_end":
estimate_timestamp_for_height(contract["endheight"]),
})
return jsonify(contractlist), 200
def api_consensus():
# Doc: https://bitbucket.org/mineboxgmbh/minebox-client-tools/src/master/doc/mb-ui-gateway-api.md#markdown-header-get-consensus
if not check_login():
return jsonify(message="Unauthorized access, please log into the main UI."), 401
siadata, status_code = get_from_sia('consensus')
# For now, just return the info from Sia directly.
return jsonify(siadata), status_code
def api_wallet_status():
# Doc: https://bitbucket.org/mineboxgmbh/minebox-client-tools/src/master/doc/mb-ui-gateway-api.md#markdown-header-get-walletstatus
if not check_login():
return jsonify(
message="Unauthorized access, please log into the main UI."), 401
siadata, sia_status_code = get_from_sia('wallet')
if sia_status_code >= 400:
return jsonify(siadata), sia_status_code
walletdata = {
"encrypted":
siadata["encrypted"],
"unlocked":
siadata["unlocked"],
"confirmedsiacoinbalance":
siadata["confirmedsiacoinbalance"],
"confirmedsiacoinbalance_sc":
int(siadata["confirmedsiacoinbalance"]) / H_PER_SC,
"unconfirmedincomingsiacoins":
siadata["unconfirmedincomingsiacoins"],
"unconfirmedincomingsiacoins_sc":
int(siadata["unconfirmedincomingsiacoins"]) / H_PER_SC,
"unconfirmedoutgoingsiacoins":
siadata["unconfirmedoutgoingsiacoins"],
"unconfirmedoutgoingsiacoins_sc":
int(siadata["unconfirmedoutgoingsiacoins"]) / H_PER_SC,
"siacoinclaimbalance":
siadata["siacoinclaimbalance"],
"siacoinclaimbalance_sc":
int(siadata["siacoinclaimbalance"]) / H_PER_SC,
"siafundbalance":
siadata["siafundbalance"],
"siafundbalance_sc":
int(siadata["siafundbalance"]) / H_PER_SC,
}
return jsonify(walletdata), 200
def api_wallet_address():
# Doc: https://bitbucket.org/mineboxgmbh/minebox-client-tools/src/master/doc/mb-ui-gateway-api.md#markdown-header-get-walletaddress
if not check_login():
return jsonify(message="Unauthorized access, please log into the main UI."), 401
siadata, sia_status_code = get_from_sia('wallet/address')
# Just return the info from Sia directly as it's either an error
# or the address in a field called "address", so pretty straight forward.
return jsonify(siadata), sia_status_code
def api_transactions():
# Doc: https://bitbucket.org/mineboxgmbh/minebox-client-tools/src/master/doc/mb-ui-gateway-api.md#markdown-header-get-transactions
if not check_login():
return jsonify(message="Unauthorized access, please log into the main UI."), 401
consdata, cons_status_code = get_from_sia('consensus')
if cons_status_code == 200:
consensus_height = consdata["height"]
else:
return jsonify(consdata), cons_status_code
blocks_per_day = 24 * 3600 / SEC_PER_BLOCK
offset = int(request.args.get('offsetdays') or 0) * blocks_per_day
endheight = int(consensus_height - offset)
startheight = int(endheight - blocks_per_day)
siadata, sia_status_code = get_from_sia("wallet/transactions?startheight=%s&endheight=%s" % (startheight, endheight))
if sia_status_code >= 400:
return jsonify(siadata), sia_status_code
# For now, just return the info from Sia directly.
return jsonify(siadata), sia_status_code
def check_sia_sync():
# Check if sia is running and in sync before doing other sia actions.
consdata, cons_status_code = get_from_sia('consensus')
if cons_status_code == 200:
if not consdata["synced"]:
return False, "ERROR: sia seems not to be synced. Please try again when the consensus is synced."
else:
return False, "ERROR: sia daemon needs to be running before you can work with it."
return True, ""
def run_sia_setup(startevent, walletdata, hostdata):
# The routes have implicit Flask application context, but the thread needs an explicit one.
# See http://flask.pocoo.org/docs/appcontext/#creating-an-application-context
with app.app_context():
threading.current_thread().name = "sia.setup"
# Tell main thread we are set up.
startevent.set()
# Do the initial setup of the sia system, so uploading and hosting files works.
# 0) Check if sia is running and consensus in sync.
# If wallet is not unlocked:
# 1) Get wallet seed from MineBD.
# 2) If the wallet is not encrypted yet, init the wallet with that seed.
# 3) Unlock the wallet, using the seed as password.
# 4) Fetch our initial allotment of siacoins from Minebox (if applicable).
# 5) Set an allowance for renting, so that we can start uploading backups.
# 6) Set up sia hosting.
success, errmsg = check_sia_sync()
if not success:
app.logger.error(errmsg)
app.logger.info(
"Exiting sia setup because sia is not ready, will try again on next ping."
)
return
if not walletdata["unlocked"]:
seed = get_seed()
if not seed:
app.logger.error(
"Did not get a useful seed, cannot initialize the sia wallet."
)
return
if not walletdata["encrypted"]:
if not init_wallet(seed):
return
if not unlock_wallet(seed):
return
if (walletdata["confirmedsiacoinbalance"] == "0"
and walletdata["unconfirmedoutgoingsiacoins"] == "0"
and walletdata["unconfirmedincomingsiacoins"] == "0"):
# We have an empty wallet, let's try to fetch some siacoins.
fetch_siacoins()
# If we succeeded, we need to wait for the coins to arrive,
# and if we failed, we have no balance and can't set an allowance
# or host files, so in any case, we return here.
return
renterdata, renter_status_code = get_from_sia('renter')
if renter_status_code == 200 and renterdata["settings"]["allowance"][
"funds"] == "0":
# No allowance, let's set one.
if not set_allowance():
return
if not hostdata["internalsettings"]["acceptingcontracts"]:
set_up_hosting()
def fetch_siacoins():
current_app.logger.info("Fetching base allotment of coins from Minebox.")
# Fetch a wallet address to send the siacoins to.
siadata, sia_status_code = get_from_sia("wallet/address")
if sia_status_code >= 400:
current_app.logger.error("Sia error %s: %s" %
(sia_status_code, siadata["message"]))
return False
# Use the address from above to request siacoins from the faucet.
fsdata, fs_status_code = post_to_faucetservice(
"getCoins", {"address": siadata["address"]})
if fs_status_code >= 400:
current_app.logger.error("Faucet error %s: %s" %
(fs_status_code, fsdata["message"]))
return False
return True
def check_backup_prerequisites():
# Check if prerequisites are met to make backups.
success, errmsg = check_sia_sync()
if not success:
return False, errmsg
siadata, sia_status_code = get_from_sia("renter/contracts")
if sia_status_code >= 400:
return False, siadata["message"]
if not siadata["contracts"]:
return False, "No Sia renter contracts, so uploading is not possible."
mbdata, mb_status_code = get_from_minebd('status')
if mb_status_code >= 400:
return False, mbdata["message"]
if mbdata["restoreRunning"] and mbdata["completedRestorePercent"] < 100:
return False, "MineBD is running an incomplete restore, so creating a backup is not possible."
# Potentially check things other than sia.
return True, ""
def api_contractstats():
# Doc: https://bitbucket.org/mineboxgmbh/minebox-client-tools/src/master/doc/mb-ui-gateway-api.md#markdown-header-get-contractstats
if not check_login():
return jsonify(
message="Unauthorized access, please log into the main UI."), 401
siadata, sia_status_code = get_from_sia('renter/contracts')
if sia_status_code >= 400:
return jsonify(siadata), sia_status_code
# List stats about the contracts.
statdata = {
"contract_count": 0,
"data_size": 0,
"totalcost_sc": 0,
"funds_remaining_sc": 0
}
for contract in siadata["contracts"]:
statdata["contract_count"] += 1
statdata["data_size"] += contract["size"]
statdata["totalcost_sc"] += int(contract["totalcost"]) / H_PER_SC
statdata["funds_remaining_sc"] += int(
contract["renterfunds"]) / H_PER_SC
return jsonify(statdata), 200
def api_sia_status():
# Doc: https://bitbucket.org/mineboxgmbh/minebox-client-tools/src/master/doc/mb-ui-gateway-api.md#markdown-header-get-siastatus
if not check_login():
return jsonify(
message="Unauthorized access, please log into the main UI."), 401
bytes_per_tb = 1e12 # not 2 ** 40 as Sia uses SI TB, see https://github.com/NebulousLabs/Sia/blob/v1.2.2/modules/host.go#L14
blocks_per_month = 30 * 24 * 3600 / SEC_PER_BLOCK
sctb_per_hb = H_PER_SC / bytes_per_tb # SC / TB -> hastings / byte
sctbmon_per_hbblk = sctb_per_hb / blocks_per_month # SC / TB / month -> hastings / byte / block
outdata = {}
consdata, cons_status_code = get_from_sia('consensus')
if cons_status_code == 200:
outdata["sia_daemon_running"] = True
outdata["consensus"] = {
"height": consdata["height"],
"synced": consdata["synced"],
}
if consdata["synced"]:
outdata["consensus"]["sync_progress"] = 100
else:
outdata["consensus"]["sync_progress"] = (100 *
consdata["height"] //
estimate_current_height())
else:
outdata["sia_daemon_running"] = False
outdata["consensus"] = {
"height": None,
"synced": None,
"sync_progress": None,
}
verdata, ver_status_code = get_from_sia("daemon/version")
if ver_status_code == 200:
outdata["sia_version"] = verdata["version"]
else:
outdata["sia_version"] = None
walletdata, wallet_status_code = get_from_sia('wallet')
if wallet_status_code == 200:
outdata["wallet"] = {
"unlocked":
walletdata["unlocked"],
"encrypted":
walletdata["encrypted"],
"confirmed_balance_sc":
int(walletdata["confirmedsiacoinbalance"]) / H_PER_SC,
"unconfirmed_delta_sc":
(int(walletdata["unconfirmedincomingsiacoins"]) -
int(walletdata["unconfirmedoutgoingsiacoins"])) / H_PER_SC,
}
else:
outdata["wallet"] = {
"unlocked": None,
"encrypted": None,
"confirmed_balance_sc": None,
"unconfirmed_delta_sc": None,
}
siadata, sia_status_code = get_from_sia("renter/contracts")
if sia_status_code == 200:
outdata["renting"] = {
"contracts":
len(siadata["contracts"]) if siadata["contracts"] else 0
}
else:
outdata["renting"] = {"contracts": None}
siadata, sia_status_code = get_from_sia("renter/files")
if sia_status_code == 200:
if siadata["files"]:
outdata["renting"]["uploaded_files"] = len(siadata["files"])
upsize = 0
for fdata in siadata["files"]:
upsize += fdata["filesize"] * fdata["redundancy"]
outdata["renting"]["uploaded_size"] = upsize
else:
outdata["renting"]["uploaded_files"] = 0
outdata["renting"]["uploaded_size"] = 0
else:
outdata["renting"]["uploaded_files"] = None
outdata["renting"]["uploaded_size"] = None
siadata, sia_status_code = get_from_sia("renter")
if sia_status_code == 200:
outdata["renting"]["allowance_funds_sc"] = int(
siadata["settings"]["allowance"]["funds"]) / H_PER_SC
outdata["renting"]["allowance_months"] = siadata["settings"][
"allowance"]["period"] / blocks_per_month
outdata["renting"]["siacoins_spent"] = (
int(siadata["financialmetrics"]["contractspending"]) +
int(siadata["financialmetrics"]["downloadspending"]) +
int(siadata["financialmetrics"]["storagespending"]) +
int(siadata["financialmetrics"]["uploadspending"])) / H_PER_SC
outdata["renting"]["siacoins_unspent"] = int(
siadata["financialmetrics"]["unspent"]) / H_PER_SC
else:
outdata["renting"]["allowance_funds_sc"] = None
outdata["renting"]["allowance_months"] = None
outdata["renting"]["siacoins_spent"] = None
outdata["renting"]["siacoins_unspent"] = None
siadata, sia_status_code = get_from_sia('host')
if sia_status_code == 200:
outdata["hosting"] = {
"enabled":
siadata["internalsettings"]["acceptingcontracts"],
"maxduration_months":
siadata["internalsettings"]["maxduration"] / blocks_per_month,
"netaddress":
siadata["internalsettings"]["netaddress"],
"collateral_sc":
int(siadata["internalsettings"]["collateral"]) / sctbmon_per_hbblk,
"collateralbudget_sc":
int(siadata["internalsettings"]["collateralbudget"]) / H_PER_SC,
"maxcollateral_sc":
int(siadata["internalsettings"]["maxcollateral"]) / H_PER_SC,
"mincontractprice_sc":
int(siadata["internalsettings"]["mincontractprice"]) / H_PER_SC,
"mindownloadbandwidthprice_sc":
int(siadata["internalsettings"]["mindownloadbandwidthprice"]) /
sctb_per_hb,
"minstorageprice_sc":
int(siadata["internalsettings"]["minstorageprice"]) /
sctbmon_per_hbblk,
"minuploadbandwidthprice_sc":
int(siadata["internalsettings"]["minuploadbandwidthprice"]) /
sctb_per_hb,
"connectabilitystatus":
siadata["connectabilitystatus"],
"workingstatus":
siadata["workingstatus"],
"contracts":
siadata["financialmetrics"]["contractcount"],
"collateral_locked_sc":
int(siadata["financialmetrics"]["lockedstoragecollateral"]) /
H_PER_SC,
"collateral_lost_sc":
int(siadata["financialmetrics"]["loststoragecollateral"]) /
H_PER_SC,
"collateral_risked_sc":
int(siadata["financialmetrics"]["riskedstoragecollateral"]) /
H_PER_SC,
"revenue_sc":
(int(siadata["financialmetrics"]["storagerevenue"]) +
int(siadata["financialmetrics"]["downloadbandwidthrevenue"]) +
int(siadata["financialmetrics"]["uploadbandwidthrevenue"])) /
H_PER_SC,
}
else:
outdata["hosting"] = {
"enabled": None,
}
return jsonify(outdata), 200
def api_status():
# Doc: https://bitbucket.org/mineboxgmbh/minebox-client-tools/src/master/doc/mb-ui-gateway-api.md#markdown-header-get-status
username = check_login()
outdata = {}
outdata["hostname"] = uname()[1]
if username:
outdata["logged_in"] = True
outdata["user"] = username
else:
outdata["logged_in"] = False
outdata["user"] = None
mbdata, mb_status_code = get_from_minebd('status')
if mb_status_code == 200:
outdata["minebd_running"] = True
outdata["minebd_encrypted"] = mbdata["hasEncryptionKey"]
outdata["minebd_storage_mounted"] = ismount(MINEBD_STORAGE_PATH)
ST_RDONLY = 1 # In Python >= 3.2, we could use os.ST_RDONLY directly instead.
outdata["minebd_storage_mount_readonly"] = bool(
os.statvfs(MINEBD_STORAGE_PATH).f_flag & ST_RDONLY)
outdata["restore_running"] = mbdata["restoreRunning"]
outdata["restore_progress"] = mbdata["completedRestorePercent"]
else:
outdata["minebd_running"] = False
outdata["minebd_encrypted"] = None
outdata["minebd_storage_mounted"] = False
outdata["minebd_storage_mount_readonly"] = None
outdata["restore_running"] = False
outdata["restore_progress"] = None
hasusers = False
for user in pwd.getpwall():
if (user.pw_uid >= 1000 and user.pw_uid < 65500
and user.pw_name != "sia" and not user.pw_name.endswith("$")):
hasusers = True
outdata["users_created"] = hasusers
# Only used by minebox-ui which requires minebox-rockstor
if is_rockstor_system():
outdata["user_setup_complete"] = rockstor_user_setup()
if 'DEMO' in environ:
outdata["backup_type"] = "sia_demo"
else:
outdata["backup_type"] = "sia"
consdata, cons_status_code = get_from_sia('consensus')
if cons_status_code == 200:
outdata["sia_daemon_running"] = True
outdata["consensus_height"] = consdata["height"]
outdata["consensus_synced"] = consdata["synced"]
else:
outdata["sia_daemon_running"] = False
outdata["consensus_height"] = None
outdata["consensus_synced"] = None
walletdata, wallet_status_code = get_from_sia('wallet')
if username and wallet_status_code == 200:
outdata["wallet_unlocked"] = walletdata["unlocked"]
outdata["wallet_encrypted"] = walletdata["encrypted"]
outdata["wallet_confirmed_balance_sc"] = int(
walletdata["confirmedsiacoinbalance"]) / H_PER_SC
outdata["wallet_unconfirmed_delta_sc"] = (
int(walletdata["unconfirmedincomingsiacoins"]) -
int(walletdata["unconfirmedoutgoingsiacoins"])) / H_PER_SC
else:
outdata["wallet_unlocked"] = None
outdata["wallet_encrypted"] = None
outdata["wallet_confirmed_balance_sc"] = None
outdata["wallet_unconfirmed_delta_sc"] = None
return jsonify(outdata), 200
def get_upload_status(backupfileinfo, uploadfiles, is_archived=False):
sia_filedata, sia_status_code = get_from_sia("renter/files")
if sia_status_code >= 400:
current_app.logger.error("Error %s getting Sia files: %s",
sia_status_code, sia_filedata["message"])
return False
upstatus = {
"filecount": 0,
"backupsize": 0,
"uploadsize": 0,
"total_uploaded_size": 0,
"uploaded_size": 0,
"fully_available": True,
}
redundancy = []
expiration = []
# create a dict generated from the JSON response.
sia_map = dict((d["siapath"], index)
for (index, d) in enumerate(sia_filedata["files"]))
for finfo in backupfileinfo:
if not is_archived and finfo["siapath"] in sia_map:
upstatus["filecount"] += 1
fdata = sia_filedata["files"][sia_map[finfo["siapath"]]]
upstatus["backupsize"] += fdata["filesize"]
upstatus["total_uploaded_size"] += (fdata["filesize"] *
fdata["uploadprogress"] /
100.0)
if fdata["siapath"] in uploadfiles:
upstatus["uploadsize"] += fdata["filesize"]
upstatus["uploaded_size"] += (fdata["filesize"] *
fdata["uploadprogress"] / 100.0)
redundancy.append(fdata["redundancy"])
expiration.append(fdata["expiration"])
if not fdata["available"]:
upstatus["fully_available"] = False
elif re.match(r".*\.dat$", finfo["siapath"]):
upstatus["filecount"] += 1
upstatus["backupsize"] += finfo["size"]
if finfo["siapath"] in uploadfiles:
upstatus["uploadsize"] += finfo["size"]
if not is_archived:
upstatus["fully_available"] = False
current_app.logger.warn("File %s not found on Sia!",
finfo["siapath"])
else:
current_app.logger.debug(
'File "%s" not on Sia and not matching watched names.',
finfo["siapath"])
# If size is 0, we report 100% progress.
# This is really needed for upload as otherwise a backup with no
# new uploadfiles would never go to 100%.
upstatus["uploadprogress"] = (100.0 * upstatus["uploaded_size"] /
upstatus["uploadsize"]
if upstatus["uploadsize"] else 100)
upstatus["totalprogress"] = (100.0 * upstatus["total_uploaded_size"] /
upstatus["backupsize"]
if upstatus["backupsize"] else 100)
upstatus["min_redundancy"] = min(redundancy) if redundancy else 0
upstatus["earliest_expiration"] = min(expiration) if expiration else 0
return upstatus
def remove_old_backups(status, activebackups):
status["message"] = "Cleaning up old backups"
status["step"] = sys._getframe().f_code.co_name.replace("_", " ")
status["starttime_step"] = time.time()
restartset = get_backups_to_restart()
allbackupnames = get_list()
sia_filedata, sia_status_code = get_from_sia('renter/files')
if sia_status_code == 200:
sia_map = dict((d["siapath"], index)
for (index, d) in enumerate(sia_filedata["files"]))
else:
return False, "ERROR: Sia daemon needs to be running for any uploads."
keepfiles = []
keepset_complete = False
for backupname in allbackupnames:
keep_this_backup = False
if not keepset_complete:
# We don't have all files to keep yet, see if this is our "golden"
# backup, an active or to-restart one - otherwise, schedule removal.
backupfiles, is_finished = get_files(backupname)
if backupname in activebackups or backupname in restartset:
keep_this_backup = True
# If we have an active backup that has metadata uploaded,
# we can consider it "golden".
if (backupname in activebackups
and "metadata_uploaded" in status
and status["metadata_uploaded"]):
current_app.logger.info("Backup %s is complete!" %
backupname)
keepset_complete = True
elif backupfiles and is_finished:
files_missing = False
for bfile in backupfiles:
if (not bfile in sia_map or not sia_filedata["files"][
sia_map[bfile]]["available"]):
files_missing = True
current_app.logger.info("%s is not fully available!" %
bfile)
if not files_missing:
# Yay! A finished backup with all files available!
# Keep this and everything we already collected, but that's it.
current_app.logger.info("Backup %s is fully complete!" %
backupname)
keep_this_backup = True
keepset_complete = True
if keep_this_backup:
current_app.logger.info(
"Keeping %s backup %s" %
("finished" if is_finished else "unfinished", backupname))
# Note that extend does not return anything.
keepfiles.extend(backupfiles)
# Converting to a set eliminates duplicates.
# Convert back to list for type consistency.
keepfiles = list(set(keepfiles))
if not keep_this_backup:
# We already have all files to keep, any older backup can be discarded.
current_app.logger.info("Discard old backup %s" % backupname)
zipname = join(METADATA_BASE, "backup.%s.zip" % backupname)
dirname = join(METADATA_BASE, "backup.%s" % backupname)
if path.isfile(zipname):
os.rename(
zipname,
join(METADATA_BASE, "old.backup.%s.zip" % backupname))
else:
os.rename(dirname,
join(METADATA_BASE, "old.backup.%s" % backupname))
rinfo = _get_repair_paths()
keepsnaps = []
if keepfiles and sia_filedata["files"]:
current_app.logger.info("Removing unneeded files from Sia network")
for siafile in sia_filedata["files"]:
if siafile["siapath"] in keepfiles:
# Get snapname from local paths like
# /mnt/lowerX/data/snapshots/<snapname>/<id>/minebox_v1_<num>.dat
src_snap = rinfo[siafile["siapath"]]["RepairPath"].split(
"/")[5]
# Keep snapshots that are the source of all files to keep, as
# they could still be uploading and therefore need the source.
if not src_snap in keepsnaps:
keepsnaps.append(src_snap)
else:
siadata, sia_status_code = post_to_sia(
"renter/delete/%s" % siafile["siapath"], "")
if sia_status_code != 204:
return False, "ERROR: sia delete error %s: %s" % (
sia_status_code, siadata['message'])
current_app.logger.info(
"The following snapshots need to be kept for potential uploads: %s",
keepsnaps)
current_app.logger.info(
"Removing possibly unneeded old lower-level data snapshots")
for snap in glob(path.join(DATADIR_MASK, "snapshots", "*")):
snapname = path.basename(snap)
zipname = join(METADATA_BASE, "backup.%s.zip" % snapname)
dirname = join(METADATA_BASE, "backup.%s" % snapname)
if (not path.isfile(zipname) and not path.isdir(dirname)
and not snapname in activebackups
and not snapname in keepsnaps):
subprocess.call([BTRFS, 'subvolume', 'delete', snap])
return True, ""
def initiate_uploads(status):
current_app.logger.info('Starting uploads.')
status["message"] = "Starting uploads"
status["step"] = sys._getframe().f_code.co_name.replace("_", " ")
status["starttime_step"] = time.time()
snapname = status["snapname"]
backupname = status["backupname"]
metadir = path.join(METADATA_BASE, backupname)
bfinfo_path = path.join(metadir, INFO_FILENAME)
if path.isfile(bfinfo_path):
remove(bfinfo_path)
sia_filedata, sia_status_code = get_from_sia('renter/files')
if sia_status_code == 200:
siafiles = sia_filedata["files"]
else:
return False, "ERROR: sia daemon needs to be running for any uploads."
# We have a randomly named subdirectory containing the .dat files.
# The subdirectory matches the serial number that MineBD returns.
mbdata, mb_status_code = get_from_minebd('serialnumber')
if mb_status_code == 200:
mbdirname = mbdata["message"]
else:
return False, "ERROR: Could not get serial number from MineBD."
status["backupfileinfo"] = []
status["backupfiles"] = []
status["uploadfiles"] = []
status["backupsize"] = 0
status["uploadsize"] = 0
status["min_redundancy"] = None
status["earliest_expiration"] = None
for filepath in glob(
path.join(DATADIR_MASK, 'snapshots', snapname, mbdirname,
'*.dat')):
fileinfo = stat(filepath)
# Only use files of non-zero size.
if fileinfo.st_size:
filename = path.basename(filepath)
(froot, fext) = path.splitext(filename)
sia_fname = '%s.%s%s' % (froot, int(fileinfo.st_mtime), fext)
if sia_fname in status["backupfiles"]:
# This file is already in the list, and we probably have
# multiple lower disks, so omit this file.
continue
status["backupfiles"].append(sia_fname)
status["backupsize"] += fileinfo.st_size
if (siafiles and any(sf["siapath"] == sia_fname and sf["available"]
and sf["redundancy"] > REDUNDANCY_LIMIT
for sf in siafiles)):
current_app.logger.info(
"%s is part of the set and already uploaded." % sia_fname)
elif (siafiles
and any(sf["siapath"] == sia_fname for sf in siafiles)):
status["uploadsize"] += fileinfo.st_size
status["uploadfiles"].append(sia_fname)
current_app.logger.info(
"%s is part of the set and the upload is already in progress."
% sia_fname)
else:
status["uploadsize"] += fileinfo.st_size
status["uploadfiles"].append(sia_fname)
current_app.logger.info(
"%s has to be uploaded, starting that." % sia_fname)
siadata, sia_status_code = post_to_sia(
'renter/upload/%s' % sia_fname, {'source': filepath})
if sia_status_code != 204:
return False, ("ERROR: sia upload error %s: %s" %
(sia_status_code, siadata["message"]))
status["backupfileinfo"].append({
"siapath": sia_fname,
"size": fileinfo.st_size
})
if not status["backupfiles"]:
return False, "ERROR: The backup set has no files, that's impossible."
with open(bfinfo_path, 'w') as outfile:
json.dump(status["backupfileinfo"], outfile)
return True, ""
def _rebalance_hosting_to_ratio():
settings = get_sia_config()
folderdata = {}
if not settings["minebox_sharing"]["enabled"]:
current_app.logger.warn(
"Sharing not enabled, cannot rebalance hosting space.")
return False
siadata, sia_status_code = get_from_sia("host/storage")
if sia_status_code >= 400:
current_app.logger.error("Sia error %s: %s" %
(sia_status_code, siadata["message"]))
return False
used_space = 0
if siadata["folders"]:
for folder in siadata["folders"]:
folderdata[folder["path"]] = folder
used_space += (folder["capacity"] - folder["capacityremaining"])
success = True
for basepath in glob(HOST_DIR_BASE_MASK):
hostpath = os.path.join(basepath, HOST_DIR_NAME)
if not os.path.isdir(hostpath):
subprocess.call(
['/usr/sbin/btrfs', 'subvolume', 'create', hostpath])
# Make sure the directory is owned by the sia user and group.
uid = pwd.getpwnam("sia").pw_uid
gid = grp.getgrnam("sia").gr_gid
os.chown(hostpath, uid, gid)
hostspace = _get_btrfs_space(hostpath)
if hostspace:
# We need to add already used hosting space to free space in this
# calculation, otherwise hosted files would reduce the hosting
# capacity!
raw_size = ((hostspace["free_est"] + used_space) *
settings["minebox_sharing"]["shared_space_ratio"])
# The actual share size must be a multiple of the granularity.
share_size = int(
(raw_size // SIA_HOST_GRANULARITY) * SIA_HOST_GRANULARITY)
else:
share_size = 0
if hostpath in folderdata:
# Existing folder, (try to) resize it if necessary.
if folderdata[hostpath]["capacity"] != share_size:
current_app.logger.info(
"Resize Sia hosting space at %s to %s MB.", hostpath,
int(share_size // 2**20))
siadata, sia_status_code = post_to_sia(
"host/storage/folders/resize", {
"path": hostpath,
"newsize": share_size
})
if sia_status_code >= 400:
current_app.logger.error(
"Sia error %s: %s" %
(sia_status_code, siadata["message"]))
success = False
else:
# New folder, add it.
current_app.logger.info("Add Sia hosting space at %s with %s MB.",
hostpath, int(share_size // 2**20))
siadata, sia_status_code = post_to_sia("host/storage/folders/add",
{
"path": hostpath,
"size": share_size
})
if sia_status_code >= 400:
current_app.logger.error("Sia error %s: %s" %
(sia_status_code, siadata["message"]))
success = False
return success
def api_wallet_transactions():
# Doc: https://bitbucket.org/mineboxgmbh/minebox-client-tools/src/master/doc/mb-ui-gateway-api.md#markdown-header-get-wallettransactions
if not check_login():
return jsonify(
message="Unauthorized access, please log into the main UI."), 401
# Do something similar to |siac wallet transactions|, see
# https://github.com/NebulousLabs/Sia/blob/master/cmd/siac/walletcmd.go#L443
siadata, sia_status_code = get_from_sia(
"wallet/transactions?startheight=%s&endheight=%s" % (0, 10000000))
if sia_status_code >= 400:
return jsonify(siadata), sia_status_code
showsplits = bool(strtobool(request.args.get("showsplits") or "false"))
onlyconfirmed = bool(
strtobool(request.args.get("onlyconfirmed") or "false"))
tdata = []
alltypes = ["confirmed"]
if not onlyconfirmed:
alltypes.append("unconfirmed")
for ttype in alltypes:
for trans in siadata["{0}transactions".format(ttype)]:
# Note that inputs into a transaction are outgoing currency and outputs
# are incoming, actually.
txn = {
"type": ttype,
"height": trans["confirmationheight"],
"timestamp": trans["confirmationtimestamp"],
"transactionid": trans["transactionid"],
"incoming": {},
"outgoing": {},
"change": 0,
"fundschange": 0,
}
for t_input in trans["inputs"]:
if t_input["walletaddress"]:
# Only process the rest if the address is owned by the wallet.
if t_input["fundtype"] in txn["outgoing"]:
txn["outgoing"][t_input["fundtype"]] += int(
t_input["value"])
else:
txn["outgoing"][t_input["fundtype"]] = int(
t_input["value"])
if t_input["fundtype"].startswith("siafund"):
txn["fundschange"] -= int(t_input["value"])
else:
txn["change"] -= int(t_input["value"])
for t_output in trans["outputs"]:
if t_output["walletaddress"]:
# Only process the rest if the address is owned by the wallet.
if t_output["fundtype"] in txn["incoming"]:
txn["incoming"][t_output["fundtype"]] += int(
t_output["value"])
else:
txn["incoming"][t_output["fundtype"]] = int(
t_output["value"])
if t_input["fundtype"].startswith("siafund"):
txn["fundschange"] += int(t_output["value"])
else:
txn["change"] += int(t_output["value"])
# Convert into data that can be put into JSON properly.
# This also adds _sc values for anything in hastings (not siafunds).
txndata = {
"confirmed": txn["type"] == "confirmed",
"height": txn["height"],
"timestamp": txn["timestamp"],
"transactionid": txn["transactionid"],
"incoming": {},
"outgoing": {},
"incoming_sc": {},
"outgoing_sc": {},
"change": str(txn["change"]),
"change_sc": txn["change"] / H_PER_SC,
"fundschange": str(txn["fundschange"]),
}
for tdirection in ["outgoing", "incoming"]:
for ftype in txn[tdirection]:
txndata[tdirection][ftype] = str(txn[tdirection][ftype])
if not ftype.startswith("siafund"):
txndata["%s_sc" % tdirection][
ftype] = txn[tdirection][ftype] / H_PER_SC
# Only add transaction to display if it either has an actual change or
# we want to show splits.
if txn["change"] or txn["fundschange"] or showsplits:
tdata.append(txndata)
return jsonify(tdata), sia_status_code
def api_ping():
# This can be called to just have the service run something.
# For example, we need to do this early after booting to restart backups
# if needed (via @app.before_first_request).
if not os.path.isfile(MACHINE_AUTH_FILE):
app.logger.info(
"Submit machine authentication to Minebox admin service.")
success, errmsg = submit_machine_auth()
if not success:
app.logger.error(errmsg)
# Look if we need to run some system maintenance tasks.
# Do this here so it runs even if Sia and upper storage are down.
# Note that in the case of updates being available for backup-service,
# this results in a restart and the rest of the ping will not be executed.
success, errmsg = system_maintenance()
if not success:
app.logger.error(errmsg)
# Check for synced sia consensus as a prerequisite to everything else.
success, errmsg = check_sia_sync()
if not success:
# Return early, we need a synced consensus to do anything.
app.logger.debug(errmsg)
app.logger.info(
"Exiting because sia is not ready, let's check again on next ping."
)
return "", 204
if not os.path.ismount(MINEBD_STORAGE_PATH):
current_app.logger.info(
"Upper storage is not mounted (yet), let's check again on next ping."
)
return "", 204
# See if sia is fully set up and do init tasks if needed.
# Setting up hosting is the last step, so if that is not active, we still
# need to do something.
walletdata, wallet_status_code = get_from_sia('wallet')
hostdata, host_status_code = get_from_sia('host')
if wallet_status_code == 200 and host_status_code == 200:
if not hostdata["internalsettings"]["acceptingcontracts"]:
# We need to seed the wallet, set up allowances and hosting, etc.
setup_sia_system(walletdata, hostdata)
elif not walletdata["unlocked"]:
# We should unlock the wallet so new contracts can be made.
unlock_sia_wallet()
# Trigger a backup if the latest is older than 24h.
timenow = int(time.time())
latestbackup = get_latest()
timelatest = int(latestbackup) if latestbackup else 0
if timelatest < timenow - 24 * 3600:
success, errmsg = check_backup_prerequisites()
if success:
bthread = start_backup_thread()
# If no backup is active but the most recent one is not finished,
# perform a restart of backups.
active_backups = get_running_backups()
if not active_backups:
snapname = get_latest()
if snapname:
if not is_finished(snapname):
restart_backups()
else:
# If the upload step is stuck (taking longer than 30 minutes),
# we should restart the sia service.
# See https://github.com/NebulousLabs/Sia/issues/1605
for tname in threadstatus:
if (threadstatus[tname]["snapname"] in active_backups
and threadstatus[tname]["step"] == "initiate uploads"
and threadstatus[tname]["starttime_step"] <
time.time() - 30 * 60):
# This would return True for success but already logs errors.
restart_sia()
# If the list of unfinished backups is significantly larger than active
# backups, we very probably have quite a few backups hanging around
# that we need to cleanup but don't get to routine cleanup (which
# happens only when a backup finishes).
unfinished_backups = get_list()
if len(unfinished_backups) > len(active_backups) + 3:
app.logger.info(
"We have %s unfinished backups but only %s active ones, let's clean up."
% (len(unfinished_backups), len(active_backups)))
start_cleanup_thread()
# Update Sia config if more than 10% off.
update_sia_config()
# See if we need to rebalance the disk space.
rebalance_diskspace()
return "", 204
