def update_sia_config():
settings = get_sia_config()
# Renting settings
siadata, sia_status_code = get_from_sia("renter")
if sia_status_code >= 400:
# If we can't get the current settings, no use in comparing to new ones.
return False
renter_params = {}
# If new settings values differ by at least 10% from currently set values,
# only then update Sia with the new settings.
if _absdiff(settings["renter"]["allowance_funds"],
int(siadata["settings"]["allowance"]["funds"])) > 0.1:
renter_params["funds"] = str(settings["renter"]["allowance_funds"])
if _absdiff(settings["renter"]["allowance_period"],
int(siadata["settings"]["allowance"]["period"])) > 0.1:
renter_params["period"] = str(settings["renter"]["allowance_period"])
if renter_params:
current_app.logger.info("Updating Sia renter/allowance settings.")
siadata, sia_status_code = post_to_sia("renter", renter_params)
if sia_status_code >= 400:
current_app.logger.error("Sia error %s: %s" %
(sia_status_code, siadata["message"]))
return False
# Hosting settings
siadata, sia_status_code = get_from_sia('host')
if sia_status_code >= 400:
# If we can't get the current settings, no use in comparing to new ones.
return False
if (not siadata["internalsettings"]["acceptingcontracts"]
and not settings["minebox_sharing"]["enabled"]):
# If hosting is deactivated, pings will call setup_sia_system()
# This will care about settings so we don't do anything here.
return True
host_params = {}
if settings["minebox_sharing"]["enabled"] != siadata["internalsettings"][
"acceptingcontracts"]:
host_params["acceptingcontracts"] = settings["minebox_sharing"][
"enabled"]
for var in [
"mincontractprice", "mindownloadbandwidthprice", "minstorageprice",
"minuploadbandwidthprice", "collateral", "collateralbudget",
"maxcollateral", "maxduration"
]:
if _absdiff(settings["host"][var], int(
siadata["internalsettings"][var])) > 0.1:
host_params[var] = str(settings["host"][var])
if host_params:
current_app.logger.info("Updating Sia host settings.")
siadata, sia_status_code = post_to_sia("host", host_params)
if sia_status_code >= 400:
current_app.logger.error("Sia error %s: %s" %
(sia_status_code, siadata["message"]))
return False
# We're done here :)
return True
def api_wallet_send():
# Doc: https://bitbucket.org/mineboxgmbh/minebox-client-tools/src/master/doc/mb-ui-gateway-api.md#markdown-header-post-walletsend
if not check_login():
return jsonify(
message="Unauthorized access, please log into the main UI."), 401
# The sia daemon takes the amount in hastings.
# If no amount in hastings is given, we support an amount_sc in siacoins.
if "amount" in request.form:
amount = int(request.form["amount"])
elif "amount_sc" in request.form:
# User decimal.Decimal as float math would not give good enough precision.
decimal.getcontext(
).prec = 36 # 24 decimals for hastings + 12 for SC part
amount = int(decimal.Decimal(request.form["amount_sc"]) * H_PER_SC)
else:
amount = 0
destination = request.form[
"destination"] if "destination" in request.form else ""
siadata, status_code = post_to_sia('wallet/siacoins', {
"amount": str(amount),
"destination": destination
})
if status_code == 200:
# siadata["transactionids"] is a list of IDs of the transactions that
# were created when sending the coins. The last transaction contains
# the output headed to the 'destination'.
return jsonify(transactionids=siadata["transactionids"],
message="%s SC successfully sent to %s." %
(amount / H_PER_SC, destination)), 200
else:
return jsonify(siadata), status_code
def set_up_hosting():
# See https://blog.sia.tech/how-to-run-a-host-on-sia-2159ebc4725 for a
# blog post explaining all steps to do.
settings = get_sia_config()
if not settings["minebox_sharing"]["enabled"]:
return
current_app.logger.info("Setting up sia hosting.")
# Set min*price, collateral, collateralbudget, maxcollateral, maxduration
siadata, sia_status_code = post_to_sia(
"host", {
"mincontractprice":
str(settings["host"]["mincontractprice"]),
"mindownloadbandwidthprice":
str(settings["host"]["mindownloadbandwidthprice"]),
"minstorageprice":
str(settings["host"]["minstorageprice"]),
"minuploadbandwidthprice":
str(settings["host"]["minuploadbandwidthprice"]),
"collateral":
str(settings["host"]["collateral"]),
"collateralbudget":
str(settings["host"]["collateralbudget"]),
"maxcollateral":
str(settings["host"]["maxcollateral"]),
"maxduration":
str(settings["host"]["maxduration"]),
})
if sia_status_code >= 400:
current_app.logger.error("Sia error %s: %s" %
(sia_status_code, siadata["message"]))
return False
# Add path(s) to share for hosting.
if not _rebalance_hosting_to_ratio():
return False
# Announce host
# When we have nice host names, we should announce that because of possibly
# changing IPs down the road.
# announce_params = {"netaddress": "host.minebox.io"}
announce_params = None
siadata, sia_status_code = post_to_sia("host/announce", announce_params)
if sia_status_code >= 400:
current_app.logger.error("Sia error %s: %s" %
(sia_status_code, siadata["message"]))
return False
return True
def unlock_wallet(seed):
current_app.logger.info("Unlocking sia wallet.")
siadata, sia_status_code = post_to_sia('wallet/unlock',
{"encryptionpassword": seed})
if sia_status_code >= 400:
current_app.logger.error("Sia error %s: %s" %
(sia_status_code, siadata["message"]))
return False
# Typically, we get a 204 status code here.
return True
def set_allowance():
current_app.logger.info("Setting an allowance for renting out files.")
settings = get_sia_config()
siadata, sia_status_code = post_to_sia(
'renter', {
"funds": str(settings["renter"]["allowance_funds"]),
"period": str(settings["renter"]["allowance_period"]),
})
if sia_status_code >= 400:
current_app.logger.error("Sia error %s: %s" %
(sia_status_code, siadata["message"]))
return False
# Typically, we get a 204 status code here.
return True
def remove_old_backups(status, activebackups):
status["message"] = "Cleaning up old backups"
status["step"] = sys._getframe().f_code.co_name.replace("_", " ")
status["starttime_step"] = time.time()
restartset = get_backups_to_restart()
allbackupnames = get_list()
sia_filedata, sia_status_code = get_from_sia('renter/files')
if sia_status_code == 200:
sia_map = dict((d["siapath"], index)
for (index, d) in enumerate(sia_filedata["files"]))
else:
return False, "ERROR: Sia daemon needs to be running for any uploads."
keepfiles = []
keepset_complete = False
for backupname in allbackupnames:
keep_this_backup = False
if not keepset_complete:
# We don't have all files to keep yet, see if this is our "golden"
# backup, an active or to-restart one - otherwise, schedule removal.
backupfiles, is_finished = get_files(backupname)
if backupname in activebackups or backupname in restartset:
keep_this_backup = True
# If we have an active backup that has metadata uploaded,
# we can consider it "golden".
if (backupname in activebackups
and "metadata_uploaded" in status
and status["metadata_uploaded"]):
current_app.logger.info("Backup %s is complete!" %
backupname)
keepset_complete = True
elif backupfiles and is_finished:
files_missing = False
for bfile in backupfiles:
if (not bfile in sia_map or not sia_filedata["files"][
sia_map[bfile]]["available"]):
files_missing = True
current_app.logger.info("%s is not fully available!" %
bfile)
if not files_missing:
# Yay! A finished backup with all files available!
# Keep this and everything we already collected, but that's it.
current_app.logger.info("Backup %s is fully complete!" %
backupname)
keep_this_backup = True
keepset_complete = True
if keep_this_backup:
current_app.logger.info(
"Keeping %s backup %s" %
("finished" if is_finished else "unfinished", backupname))
# Note that extend does not return anything.
keepfiles.extend(backupfiles)
# Converting to a set eliminates duplicates.
# Convert back to list for type consistency.
keepfiles = list(set(keepfiles))
if not keep_this_backup:
# We already have all files to keep, any older backup can be discarded.
current_app.logger.info("Discard old backup %s" % backupname)
zipname = join(METADATA_BASE, "backup.%s.zip" % backupname)
dirname = join(METADATA_BASE, "backup.%s" % backupname)
if path.isfile(zipname):
os.rename(
zipname,
join(METADATA_BASE, "old.backup.%s.zip" % backupname))
else:
os.rename(dirname,
join(METADATA_BASE, "old.backup.%s" % backupname))
rinfo = _get_repair_paths()
keepsnaps = []
if keepfiles and sia_filedata["files"]:
current_app.logger.info("Removing unneeded files from Sia network")
for siafile in sia_filedata["files"]:
if siafile["siapath"] in keepfiles:
# Get snapname from local paths like
# /mnt/lowerX/data/snapshots/<snapname>/<id>/minebox_v1_<num>.dat
src_snap = rinfo[siafile["siapath"]]["RepairPath"].split(
"/")[5]
# Keep snapshots that are the source of all files to keep, as
# they could still be uploading and therefore need the source.
if not src_snap in keepsnaps:
keepsnaps.append(src_snap)
else:
siadata, sia_status_code = post_to_sia(
"renter/delete/%s" % siafile["siapath"], "")
if sia_status_code != 204:
return False, "ERROR: sia delete error %s: %s" % (
sia_status_code, siadata['message'])
current_app.logger.info(
"The following snapshots need to be kept for potential uploads: %s",
keepsnaps)
current_app.logger.info(
"Removing possibly unneeded old lower-level data snapshots")
for snap in glob(path.join(DATADIR_MASK, "snapshots", "*")):
snapname = path.basename(snap)
zipname = join(METADATA_BASE, "backup.%s.zip" % snapname)
dirname = join(METADATA_BASE, "backup.%s" % snapname)
if (not path.isfile(zipname) and not path.isdir(dirname)
and not snapname in activebackups
and not snapname in keepsnaps):
subprocess.call([BTRFS, 'subvolume', 'delete', snap])
return True, ""
def initiate_uploads(status):
current_app.logger.info('Starting uploads.')
status["message"] = "Starting uploads"
status["step"] = sys._getframe().f_code.co_name.replace("_", " ")
status["starttime_step"] = time.time()
snapname = status["snapname"]
backupname = status["backupname"]
metadir = path.join(METADATA_BASE, backupname)
bfinfo_path = path.join(metadir, INFO_FILENAME)
if path.isfile(bfinfo_path):
remove(bfinfo_path)
sia_filedata, sia_status_code = get_from_sia('renter/files')
if sia_status_code == 200:
siafiles = sia_filedata["files"]
else:
return False, "ERROR: sia daemon needs to be running for any uploads."
# We have a randomly named subdirectory containing the .dat files.
# The subdirectory matches the serial number that MineBD returns.
mbdata, mb_status_code = get_from_minebd('serialnumber')
if mb_status_code == 200:
mbdirname = mbdata["message"]
else:
return False, "ERROR: Could not get serial number from MineBD."
status["backupfileinfo"] = []
status["backupfiles"] = []
status["uploadfiles"] = []
status["backupsize"] = 0
status["uploadsize"] = 0
status["min_redundancy"] = None
status["earliest_expiration"] = None
for filepath in glob(
path.join(DATADIR_MASK, 'snapshots', snapname, mbdirname,
'*.dat')):
fileinfo = stat(filepath)
# Only use files of non-zero size.
if fileinfo.st_size:
filename = path.basename(filepath)
(froot, fext) = path.splitext(filename)
sia_fname = '%s.%s%s' % (froot, int(fileinfo.st_mtime), fext)
if sia_fname in status["backupfiles"]:
# This file is already in the list, and we probably have
# multiple lower disks, so omit this file.
continue
status["backupfiles"].append(sia_fname)
status["backupsize"] += fileinfo.st_size
if (siafiles and any(sf["siapath"] == sia_fname and sf["available"]
and sf["redundancy"] > REDUNDANCY_LIMIT
for sf in siafiles)):
current_app.logger.info(
"%s is part of the set and already uploaded." % sia_fname)
elif (siafiles
and any(sf["siapath"] == sia_fname for sf in siafiles)):
status["uploadsize"] += fileinfo.st_size
status["uploadfiles"].append(sia_fname)
current_app.logger.info(
"%s is part of the set and the upload is already in progress."
% sia_fname)
else:
status["uploadsize"] += fileinfo.st_size
status["uploadfiles"].append(sia_fname)
current_app.logger.info(
"%s has to be uploaded, starting that." % sia_fname)
siadata, sia_status_code = post_to_sia(
'renter/upload/%s' % sia_fname, {'source': filepath})
if sia_status_code != 204:
return False, ("ERROR: sia upload error %s: %s" %
(sia_status_code, siadata["message"]))
status["backupfileinfo"].append({
"siapath": sia_fname,
"size": fileinfo.st_size
})
if not status["backupfiles"]:
return False, "ERROR: The backup set has no files, that's impossible."
with open(bfinfo_path, 'w') as outfile:
json.dump(status["backupfileinfo"], outfile)
return True, ""
def _rebalance_hosting_to_ratio():
settings = get_sia_config()
folderdata = {}
if not settings["minebox_sharing"]["enabled"]:
current_app.logger.warn(
"Sharing not enabled, cannot rebalance hosting space.")
return False
siadata, sia_status_code = get_from_sia("host/storage")
if sia_status_code >= 400:
current_app.logger.error("Sia error %s: %s" %
(sia_status_code, siadata["message"]))
return False
used_space = 0
if siadata["folders"]:
for folder in siadata["folders"]:
folderdata[folder["path"]] = folder
used_space += (folder["capacity"] - folder["capacityremaining"])
success = True
for basepath in glob(HOST_DIR_BASE_MASK):
hostpath = os.path.join(basepath, HOST_DIR_NAME)
if not os.path.isdir(hostpath):
subprocess.call(
['/usr/sbin/btrfs', 'subvolume', 'create', hostpath])
# Make sure the directory is owned by the sia user and group.
uid = pwd.getpwnam("sia").pw_uid
gid = grp.getgrnam("sia").gr_gid
os.chown(hostpath, uid, gid)
hostspace = _get_btrfs_space(hostpath)
if hostspace:
# We need to add already used hosting space to free space in this
# calculation, otherwise hosted files would reduce the hosting
# capacity!
raw_size = ((hostspace["free_est"] + used_space) *
settings["minebox_sharing"]["shared_space_ratio"])
# The actual share size must be a multiple of the granularity.
share_size = int(
(raw_size // SIA_HOST_GRANULARITY) * SIA_HOST_GRANULARITY)
else:
share_size = 0
if hostpath in folderdata:
# Existing folder, (try to) resize it if necessary.
if folderdata[hostpath]["capacity"] != share_size:
current_app.logger.info(
"Resize Sia hosting space at %s to %s MB.", hostpath,
int(share_size // 2**20))
siadata, sia_status_code = post_to_sia(
"host/storage/folders/resize", {
"path": hostpath,
"newsize": share_size
})
if sia_status_code >= 400:
current_app.logger.error(
"Sia error %s: %s" %
(sia_status_code, siadata["message"]))
success = False
else:
# New folder, add it.
current_app.logger.info("Add Sia hosting space at %s with %s MB.",
hostpath, int(share_size // 2**20))
siadata, sia_status_code = post_to_sia("host/storage/folders/add",
{
"path": hostpath,
"size": share_size
})
if sia_status_code >= 400:
current_app.logger.error("Sia error %s: %s" %
(sia_status_code, siadata["message"]))
success = False
return success