def api_status(): # Doc: https://bitbucket.org/mineboxgmbh/minebox-client-tools/src/master/doc/mb-ui-gateway-api.md#markdown-header-get-status username = check_login() outdata = {} if username: outdata["logged_in"] = True outdata["user"] = username else: outdata["logged_in"] = False outdata["user"] = None mbdata, mb_status_code = get_from_minebd('status') if mb_status_code == 200: outdata["minebd_running"] = True outdata["minebd_encrypted"] = mbdata["hasEncryptionKey"] outdata["minebd_storage_mounted"] = ismount(MINEBD_STORAGE_PATH) outdata["restore_running"] = mbdata["restoreRunning"] outdata["restore_progress"] = mbdata["completedRestorePercent"] else: outdata["minebd_running"] = False outdata["minebd_encrypted"] = None outdata["minebd_storage_mounted"] = False outdata["restore_running"] = False outdata["restore_progress"] = None hasusers = False for user in pwd.getpwall(): if user.pw_uid >= 1000 and user.pw_uid < 65500 and user.pw_name != "sia": hasusers = True outdata["users_created"] = hasusers if 'DEMO' in environ: outdata["backup_type"] = "sia_demo" else: outdata["backup_type"] = "sia" consdata, cons_status_code = get_from_sia('consensus') if cons_status_code == 200: outdata["sia_daemon_running"] = True outdata["consensus_height"] = consdata["height"] outdata["consensus_synced"] = consdata["synced"] else: outdata["sia_daemon_running"] = False outdata["consensus_height"] = None outdata["consensus_synced"] = None walletdata, wallet_status_code = get_from_sia('wallet') if username and wallet_status_code == 200: outdata["wallet_unlocked"] = walletdata["unlocked"] outdata["wallet_encrypted"] = walletdata["encrypted"] outdata["wallet_confirmed_balance_sc"] = int(walletdata["confirmedsiacoinbalance"]) / H_PER_SC outdata["wallet_unconfirmed_delta_sc"] = (int(walletdata["unconfirmedincomingsiacoins"]) - int(walletdata["unconfirmedoutgoingsiacoins"])) / H_PER_SC else: outdata["wallet_unlocked"] = None outdata["wallet_encrypted"] = None outdata["wallet_confirmed_balance_sc"] = None outdata["wallet_unconfirmed_delta_sc"] = None return jsonify(outdata), 200
def update_sia_config(): settings = get_sia_config() # Renting settings siadata, sia_status_code = get_from_sia("renter") if sia_status_code >= 400: # If we can't get the current settings, no use in comparing to new ones. return False renter_params = {} # If new settings values differ by at least 10% from currently set values, # only then update Sia with the new settings. if _absdiff(settings["renter"]["allowance_funds"], int(siadata["settings"]["allowance"]["funds"])) > 0.1: renter_params["funds"] = str(settings["renter"]["allowance_funds"]) if _absdiff(settings["renter"]["allowance_period"], int(siadata["settings"]["allowance"]["period"])) > 0.1: renter_params["period"] = str(settings["renter"]["allowance_period"]) if renter_params: current_app.logger.info("Updating Sia renter/allowance settings.") siadata, sia_status_code = post_to_sia("renter", renter_params) if sia_status_code >= 400: current_app.logger.error("Sia error %s: %s" % (sia_status_code, siadata["message"])) return False # Hosting settings siadata, sia_status_code = get_from_sia('host') if sia_status_code >= 400: # If we can't get the current settings, no use in comparing to new ones. return False if (not siadata["internalsettings"]["acceptingcontracts"] and not settings["minebox_sharing"]["enabled"]): # If hosting is deactivated, pings will call setup_sia_system() # This will care about settings so we don't do anything here. return True host_params = {} if settings["minebox_sharing"]["enabled"] != siadata["internalsettings"][ "acceptingcontracts"]: host_params["acceptingcontracts"] = settings["minebox_sharing"][ "enabled"] for var in [ "mincontractprice", "mindownloadbandwidthprice", "minstorageprice", "minuploadbandwidthprice", "collateral", "collateralbudget", "maxcollateral", "maxduration" ]: if _absdiff(settings["host"][var], int( siadata["internalsettings"][var])) > 0.1: host_params[var] = str(settings["host"][var]) if host_params: current_app.logger.info("Updating Sia host settings.") siadata, sia_status_code = post_to_sia("host", host_params) if sia_status_code >= 400: current_app.logger.error("Sia error %s: %s" % (sia_status_code, siadata["message"])) return False # We're done here :) return True
def api_contracts(): # Doc: https://bitbucket.org/mineboxgmbh/minebox-client-tools/src/master/doc/mb-ui-gateway-api.md#markdown-header-get-contracts if not check_login(): return jsonify( message="Unauthorized access, please log into the main UI."), 401 siadata, sia_status_code = get_from_sia('renter/contracts') if sia_status_code >= 400: return jsonify(siadata), sia_status_code # Create a summary similar to what `siac renter contracts` presents. # We could expose the full details of a contract in a different route, e.g. /contract/<id>. contractlist = [] for contract in siadata["contracts"]: contractlist.append({ "id": contract["id"], "host": contract["netaddress"], "funds_remaining_sc": int(contract["renterfunds"]) / H_PER_SC, "funds_spent_sc": (int(contract["StorageSpending"]) + int(contract["uploadspending"]) + int(contract["downloadspending"])) / H_PER_SC, "fees_spent_sc": int(contract["fees"]) / H_PER_SC, "totalcost_sc": int(contract["totalcost"]) / H_PER_SC, "data_size": contract["size"], "height_end": contract["endheight"], "esttime_end": estimate_timestamp_for_height(contract["endheight"]), }) return jsonify(contractlist), 200
def api_consensus(): # Doc: https://bitbucket.org/mineboxgmbh/minebox-client-tools/src/master/doc/mb-ui-gateway-api.md#markdown-header-get-consensus if not check_login(): return jsonify(message="Unauthorized access, please log into the main UI."), 401 siadata, status_code = get_from_sia('consensus') # For now, just return the info from Sia directly. return jsonify(siadata), status_code
def api_wallet_status(): # Doc: https://bitbucket.org/mineboxgmbh/minebox-client-tools/src/master/doc/mb-ui-gateway-api.md#markdown-header-get-walletstatus if not check_login(): return jsonify( message="Unauthorized access, please log into the main UI."), 401 siadata, sia_status_code = get_from_sia('wallet') if sia_status_code >= 400: return jsonify(siadata), sia_status_code walletdata = { "encrypted": siadata["encrypted"], "unlocked": siadata["unlocked"], "confirmedsiacoinbalance": siadata["confirmedsiacoinbalance"], "confirmedsiacoinbalance_sc": int(siadata["confirmedsiacoinbalance"]) / H_PER_SC, "unconfirmedincomingsiacoins": siadata["unconfirmedincomingsiacoins"], "unconfirmedincomingsiacoins_sc": int(siadata["unconfirmedincomingsiacoins"]) / H_PER_SC, "unconfirmedoutgoingsiacoins": siadata["unconfirmedoutgoingsiacoins"], "unconfirmedoutgoingsiacoins_sc": int(siadata["unconfirmedoutgoingsiacoins"]) / H_PER_SC, "siacoinclaimbalance": siadata["siacoinclaimbalance"], "siacoinclaimbalance_sc": int(siadata["siacoinclaimbalance"]) / H_PER_SC, "siafundbalance": siadata["siafundbalance"], "siafundbalance_sc": int(siadata["siafundbalance"]) / H_PER_SC, } return jsonify(walletdata), 200
def api_wallet_address(): # Doc: https://bitbucket.org/mineboxgmbh/minebox-client-tools/src/master/doc/mb-ui-gateway-api.md#markdown-header-get-walletaddress if not check_login(): return jsonify(message="Unauthorized access, please log into the main UI."), 401 siadata, sia_status_code = get_from_sia('wallet/address') # Just return the info from Sia directly as it's either an error # or the address in a field called "address", so pretty straight forward. return jsonify(siadata), sia_status_code
def api_transactions(): # Doc: https://bitbucket.org/mineboxgmbh/minebox-client-tools/src/master/doc/mb-ui-gateway-api.md#markdown-header-get-transactions if not check_login(): return jsonify(message="Unauthorized access, please log into the main UI."), 401 consdata, cons_status_code = get_from_sia('consensus') if cons_status_code == 200: consensus_height = consdata["height"] else: return jsonify(consdata), cons_status_code blocks_per_day = 24 * 3600 / SEC_PER_BLOCK offset = int(request.args.get('offsetdays') or 0) * blocks_per_day endheight = int(consensus_height - offset) startheight = int(endheight - blocks_per_day) siadata, sia_status_code = get_from_sia("wallet/transactions?startheight=%s&endheight=%s" % (startheight, endheight)) if sia_status_code >= 400: return jsonify(siadata), sia_status_code # For now, just return the info from Sia directly. return jsonify(siadata), sia_status_code
def check_sia_sync(): # Check if sia is running and in sync before doing other sia actions. consdata, cons_status_code = get_from_sia('consensus') if cons_status_code == 200: if not consdata["synced"]: return False, "ERROR: sia seems not to be synced. Please try again when the consensus is synced." else: return False, "ERROR: sia daemon needs to be running before you can work with it." return True, ""
def run_sia_setup(startevent, walletdata, hostdata): # The routes have implicit Flask application context, but the thread needs an explicit one. # See http://flask.pocoo.org/docs/appcontext/#creating-an-application-context with app.app_context(): threading.current_thread().name = "sia.setup" # Tell main thread we are set up. startevent.set() # Do the initial setup of the sia system, so uploading and hosting files works. # 0) Check if sia is running and consensus in sync. # If wallet is not unlocked: # 1) Get wallet seed from MineBD. # 2) If the wallet is not encrypted yet, init the wallet with that seed. # 3) Unlock the wallet, using the seed as password. # 4) Fetch our initial allotment of siacoins from Minebox (if applicable). # 5) Set an allowance for renting, so that we can start uploading backups. # 6) Set up sia hosting. success, errmsg = check_sia_sync() if not success: app.logger.error(errmsg) app.logger.info( "Exiting sia setup because sia is not ready, will try again on next ping." ) return if not walletdata["unlocked"]: seed = get_seed() if not seed: app.logger.error( "Did not get a useful seed, cannot initialize the sia wallet." ) return if not walletdata["encrypted"]: if not init_wallet(seed): return if not unlock_wallet(seed): return if (walletdata["confirmedsiacoinbalance"] == "0" and walletdata["unconfirmedoutgoingsiacoins"] == "0" and walletdata["unconfirmedincomingsiacoins"] == "0"): # We have an empty wallet, let's try to fetch some siacoins. fetch_siacoins() # If we succeeded, we need to wait for the coins to arrive, # and if we failed, we have no balance and can't set an allowance # or host files, so in any case, we return here. return renterdata, renter_status_code = get_from_sia('renter') if renter_status_code == 200 and renterdata["settings"]["allowance"][ "funds"] == "0": # No allowance, let's set one. if not set_allowance(): return if not hostdata["internalsettings"]["acceptingcontracts"]: set_up_hosting()
def fetch_siacoins(): current_app.logger.info("Fetching base allotment of coins from Minebox.") # Fetch a wallet address to send the siacoins to. siadata, sia_status_code = get_from_sia("wallet/address") if sia_status_code >= 400: current_app.logger.error("Sia error %s: %s" % (sia_status_code, siadata["message"])) return False # Use the address from above to request siacoins from the faucet. fsdata, fs_status_code = post_to_faucetservice( "getCoins", {"address": siadata["address"]}) if fs_status_code >= 400: current_app.logger.error("Faucet error %s: %s" % (fs_status_code, fsdata["message"])) return False return True
def check_backup_prerequisites(): # Check if prerequisites are met to make backups. success, errmsg = check_sia_sync() if not success: return False, errmsg siadata, sia_status_code = get_from_sia("renter/contracts") if sia_status_code >= 400: return False, siadata["message"] if not siadata["contracts"]: return False, "No Sia renter contracts, so uploading is not possible." mbdata, mb_status_code = get_from_minebd('status') if mb_status_code >= 400: return False, mbdata["message"] if mbdata["restoreRunning"] and mbdata["completedRestorePercent"] < 100: return False, "MineBD is running an incomplete restore, so creating a backup is not possible." # Potentially check things other than sia. return True, ""
def api_contractstats(): # Doc: https://bitbucket.org/mineboxgmbh/minebox-client-tools/src/master/doc/mb-ui-gateway-api.md#markdown-header-get-contractstats if not check_login(): return jsonify( message="Unauthorized access, please log into the main UI."), 401 siadata, sia_status_code = get_from_sia('renter/contracts') if sia_status_code >= 400: return jsonify(siadata), sia_status_code # List stats about the contracts. statdata = { "contract_count": 0, "data_size": 0, "totalcost_sc": 0, "funds_remaining_sc": 0 } for contract in siadata["contracts"]: statdata["contract_count"] += 1 statdata["data_size"] += contract["size"] statdata["totalcost_sc"] += int(contract["totalcost"]) / H_PER_SC statdata["funds_remaining_sc"] += int( contract["renterfunds"]) / H_PER_SC return jsonify(statdata), 200
def api_sia_status(): # Doc: https://bitbucket.org/mineboxgmbh/minebox-client-tools/src/master/doc/mb-ui-gateway-api.md#markdown-header-get-siastatus if not check_login(): return jsonify( message="Unauthorized access, please log into the main UI."), 401 bytes_per_tb = 1e12 # not 2 ** 40 as Sia uses SI TB, see https://github.com/NebulousLabs/Sia/blob/v1.2.2/modules/host.go#L14 blocks_per_month = 30 * 24 * 3600 / SEC_PER_BLOCK sctb_per_hb = H_PER_SC / bytes_per_tb # SC / TB -> hastings / byte sctbmon_per_hbblk = sctb_per_hb / blocks_per_month # SC / TB / month -> hastings / byte / block outdata = {} consdata, cons_status_code = get_from_sia('consensus') if cons_status_code == 200: outdata["sia_daemon_running"] = True outdata["consensus"] = { "height": consdata["height"], "synced": consdata["synced"], } if consdata["synced"]: outdata["consensus"]["sync_progress"] = 100 else: outdata["consensus"]["sync_progress"] = (100 * consdata["height"] // estimate_current_height()) else: outdata["sia_daemon_running"] = False outdata["consensus"] = { "height": None, "synced": None, "sync_progress": None, } verdata, ver_status_code = get_from_sia("daemon/version") if ver_status_code == 200: outdata["sia_version"] = verdata["version"] else: outdata["sia_version"] = None walletdata, wallet_status_code = get_from_sia('wallet') if wallet_status_code == 200: outdata["wallet"] = { "unlocked": walletdata["unlocked"], "encrypted": walletdata["encrypted"], "confirmed_balance_sc": int(walletdata["confirmedsiacoinbalance"]) / H_PER_SC, "unconfirmed_delta_sc": (int(walletdata["unconfirmedincomingsiacoins"]) - int(walletdata["unconfirmedoutgoingsiacoins"])) / H_PER_SC, } else: outdata["wallet"] = { "unlocked": None, "encrypted": None, "confirmed_balance_sc": None, "unconfirmed_delta_sc": None, } siadata, sia_status_code = get_from_sia("renter/contracts") if sia_status_code == 200: outdata["renting"] = { "contracts": len(siadata["contracts"]) if siadata["contracts"] else 0 } else: outdata["renting"] = {"contracts": None} siadata, sia_status_code = get_from_sia("renter/files") if sia_status_code == 200: if siadata["files"]: outdata["renting"]["uploaded_files"] = len(siadata["files"]) upsize = 0 for fdata in siadata["files"]: upsize += fdata["filesize"] * fdata["redundancy"] outdata["renting"]["uploaded_size"] = upsize else: outdata["renting"]["uploaded_files"] = 0 outdata["renting"]["uploaded_size"] = 0 else: outdata["renting"]["uploaded_files"] = None outdata["renting"]["uploaded_size"] = None siadata, sia_status_code = get_from_sia("renter") if sia_status_code == 200: outdata["renting"]["allowance_funds_sc"] = int( siadata["settings"]["allowance"]["funds"]) / H_PER_SC outdata["renting"]["allowance_months"] = siadata["settings"][ "allowance"]["period"] / blocks_per_month outdata["renting"]["siacoins_spent"] = ( int(siadata["financialmetrics"]["contractspending"]) + int(siadata["financialmetrics"]["downloadspending"]) + int(siadata["financialmetrics"]["storagespending"]) + int(siadata["financialmetrics"]["uploadspending"])) / H_PER_SC outdata["renting"]["siacoins_unspent"] = int( siadata["financialmetrics"]["unspent"]) / H_PER_SC else: outdata["renting"]["allowance_funds_sc"] = None outdata["renting"]["allowance_months"] = None outdata["renting"]["siacoins_spent"] = None outdata["renting"]["siacoins_unspent"] = None siadata, sia_status_code = get_from_sia('host') if sia_status_code == 200: outdata["hosting"] = { "enabled": siadata["internalsettings"]["acceptingcontracts"], "maxduration_months": siadata["internalsettings"]["maxduration"] / blocks_per_month, "netaddress": siadata["internalsettings"]["netaddress"], "collateral_sc": int(siadata["internalsettings"]["collateral"]) / sctbmon_per_hbblk, "collateralbudget_sc": int(siadata["internalsettings"]["collateralbudget"]) / H_PER_SC, "maxcollateral_sc": int(siadata["internalsettings"]["maxcollateral"]) / H_PER_SC, "mincontractprice_sc": int(siadata["internalsettings"]["mincontractprice"]) / H_PER_SC, "mindownloadbandwidthprice_sc": int(siadata["internalsettings"]["mindownloadbandwidthprice"]) / sctb_per_hb, "minstorageprice_sc": int(siadata["internalsettings"]["minstorageprice"]) / sctbmon_per_hbblk, "minuploadbandwidthprice_sc": int(siadata["internalsettings"]["minuploadbandwidthprice"]) / sctb_per_hb, "connectabilitystatus": siadata["connectabilitystatus"], "workingstatus": siadata["workingstatus"], "contracts": siadata["financialmetrics"]["contractcount"], "collateral_locked_sc": int(siadata["financialmetrics"]["lockedstoragecollateral"]) / H_PER_SC, "collateral_lost_sc": int(siadata["financialmetrics"]["loststoragecollateral"]) / H_PER_SC, "collateral_risked_sc": int(siadata["financialmetrics"]["riskedstoragecollateral"]) / H_PER_SC, "revenue_sc": (int(siadata["financialmetrics"]["storagerevenue"]) + int(siadata["financialmetrics"]["downloadbandwidthrevenue"]) + int(siadata["financialmetrics"]["uploadbandwidthrevenue"])) / H_PER_SC, } else: outdata["hosting"] = { "enabled": None, } return jsonify(outdata), 200
def api_status(): # Doc: https://bitbucket.org/mineboxgmbh/minebox-client-tools/src/master/doc/mb-ui-gateway-api.md#markdown-header-get-status username = check_login() outdata = {} outdata["hostname"] = uname()[1] if username: outdata["logged_in"] = True outdata["user"] = username else: outdata["logged_in"] = False outdata["user"] = None mbdata, mb_status_code = get_from_minebd('status') if mb_status_code == 200: outdata["minebd_running"] = True outdata["minebd_encrypted"] = mbdata["hasEncryptionKey"] outdata["minebd_storage_mounted"] = ismount(MINEBD_STORAGE_PATH) ST_RDONLY = 1 # In Python >= 3.2, we could use os.ST_RDONLY directly instead. outdata["minebd_storage_mount_readonly"] = bool( os.statvfs(MINEBD_STORAGE_PATH).f_flag & ST_RDONLY) outdata["restore_running"] = mbdata["restoreRunning"] outdata["restore_progress"] = mbdata["completedRestorePercent"] else: outdata["minebd_running"] = False outdata["minebd_encrypted"] = None outdata["minebd_storage_mounted"] = False outdata["minebd_storage_mount_readonly"] = None outdata["restore_running"] = False outdata["restore_progress"] = None hasusers = False for user in pwd.getpwall(): if (user.pw_uid >= 1000 and user.pw_uid < 65500 and user.pw_name != "sia" and not user.pw_name.endswith("$")): hasusers = True outdata["users_created"] = hasusers # Only used by minebox-ui which requires minebox-rockstor if is_rockstor_system(): outdata["user_setup_complete"] = rockstor_user_setup() if 'DEMO' in environ: outdata["backup_type"] = "sia_demo" else: outdata["backup_type"] = "sia" consdata, cons_status_code = get_from_sia('consensus') if cons_status_code == 200: outdata["sia_daemon_running"] = True outdata["consensus_height"] = consdata["height"] outdata["consensus_synced"] = consdata["synced"] else: outdata["sia_daemon_running"] = False outdata["consensus_height"] = None outdata["consensus_synced"] = None walletdata, wallet_status_code = get_from_sia('wallet') if username and wallet_status_code == 200: outdata["wallet_unlocked"] = walletdata["unlocked"] outdata["wallet_encrypted"] = walletdata["encrypted"] outdata["wallet_confirmed_balance_sc"] = int( walletdata["confirmedsiacoinbalance"]) / H_PER_SC outdata["wallet_unconfirmed_delta_sc"] = ( int(walletdata["unconfirmedincomingsiacoins"]) - int(walletdata["unconfirmedoutgoingsiacoins"])) / H_PER_SC else: outdata["wallet_unlocked"] = None outdata["wallet_encrypted"] = None outdata["wallet_confirmed_balance_sc"] = None outdata["wallet_unconfirmed_delta_sc"] = None return jsonify(outdata), 200
def get_upload_status(backupfileinfo, uploadfiles, is_archived=False): sia_filedata, sia_status_code = get_from_sia("renter/files") if sia_status_code >= 400: current_app.logger.error("Error %s getting Sia files: %s", sia_status_code, sia_filedata["message"]) return False upstatus = { "filecount": 0, "backupsize": 0, "uploadsize": 0, "total_uploaded_size": 0, "uploaded_size": 0, "fully_available": True, } redundancy = [] expiration = [] # create a dict generated from the JSON response. sia_map = dict((d["siapath"], index) for (index, d) in enumerate(sia_filedata["files"])) for finfo in backupfileinfo: if not is_archived and finfo["siapath"] in sia_map: upstatus["filecount"] += 1 fdata = sia_filedata["files"][sia_map[finfo["siapath"]]] upstatus["backupsize"] += fdata["filesize"] upstatus["total_uploaded_size"] += (fdata["filesize"] * fdata["uploadprogress"] / 100.0) if fdata["siapath"] in uploadfiles: upstatus["uploadsize"] += fdata["filesize"] upstatus["uploaded_size"] += (fdata["filesize"] * fdata["uploadprogress"] / 100.0) redundancy.append(fdata["redundancy"]) expiration.append(fdata["expiration"]) if not fdata["available"]: upstatus["fully_available"] = False elif re.match(r".*\.dat$", finfo["siapath"]): upstatus["filecount"] += 1 upstatus["backupsize"] += finfo["size"] if finfo["siapath"] in uploadfiles: upstatus["uploadsize"] += finfo["size"] if not is_archived: upstatus["fully_available"] = False current_app.logger.warn("File %s not found on Sia!", finfo["siapath"]) else: current_app.logger.debug( 'File "%s" not on Sia and not matching watched names.', finfo["siapath"]) # If size is 0, we report 100% progress. # This is really needed for upload as otherwise a backup with no # new uploadfiles would never go to 100%. upstatus["uploadprogress"] = (100.0 * upstatus["uploaded_size"] / upstatus["uploadsize"] if upstatus["uploadsize"] else 100) upstatus["totalprogress"] = (100.0 * upstatus["total_uploaded_size"] / upstatus["backupsize"] if upstatus["backupsize"] else 100) upstatus["min_redundancy"] = min(redundancy) if redundancy else 0 upstatus["earliest_expiration"] = min(expiration) if expiration else 0 return upstatus
def remove_old_backups(status, activebackups): status["message"] = "Cleaning up old backups" status["step"] = sys._getframe().f_code.co_name.replace("_", " ") status["starttime_step"] = time.time() restartset = get_backups_to_restart() allbackupnames = get_list() sia_filedata, sia_status_code = get_from_sia('renter/files') if sia_status_code == 200: sia_map = dict((d["siapath"], index) for (index, d) in enumerate(sia_filedata["files"])) else: return False, "ERROR: Sia daemon needs to be running for any uploads." keepfiles = [] keepset_complete = False for backupname in allbackupnames: keep_this_backup = False if not keepset_complete: # We don't have all files to keep yet, see if this is our "golden" # backup, an active or to-restart one - otherwise, schedule removal. backupfiles, is_finished = get_files(backupname) if backupname in activebackups or backupname in restartset: keep_this_backup = True # If we have an active backup that has metadata uploaded, # we can consider it "golden". if (backupname in activebackups and "metadata_uploaded" in status and status["metadata_uploaded"]): current_app.logger.info("Backup %s is complete!" % backupname) keepset_complete = True elif backupfiles and is_finished: files_missing = False for bfile in backupfiles: if (not bfile in sia_map or not sia_filedata["files"][ sia_map[bfile]]["available"]): files_missing = True current_app.logger.info("%s is not fully available!" % bfile) if not files_missing: # Yay! A finished backup with all files available! # Keep this and everything we already collected, but that's it. current_app.logger.info("Backup %s is fully complete!" % backupname) keep_this_backup = True keepset_complete = True if keep_this_backup: current_app.logger.info( "Keeping %s backup %s" % ("finished" if is_finished else "unfinished", backupname)) # Note that extend does not return anything. keepfiles.extend(backupfiles) # Converting to a set eliminates duplicates. # Convert back to list for type consistency. keepfiles = list(set(keepfiles)) if not keep_this_backup: # We already have all files to keep, any older backup can be discarded. current_app.logger.info("Discard old backup %s" % backupname) zipname = join(METADATA_BASE, "backup.%s.zip" % backupname) dirname = join(METADATA_BASE, "backup.%s" % backupname) if path.isfile(zipname): os.rename( zipname, join(METADATA_BASE, "old.backup.%s.zip" % backupname)) else: os.rename(dirname, join(METADATA_BASE, "old.backup.%s" % backupname)) rinfo = _get_repair_paths() keepsnaps = [] if keepfiles and sia_filedata["files"]: current_app.logger.info("Removing unneeded files from Sia network") for siafile in sia_filedata["files"]: if siafile["siapath"] in keepfiles: # Get snapname from local paths like # /mnt/lowerX/data/snapshots/<snapname>/<id>/minebox_v1_<num>.dat src_snap = rinfo[siafile["siapath"]]["RepairPath"].split( "/")[5] # Keep snapshots that are the source of all files to keep, as # they could still be uploading and therefore need the source. if not src_snap in keepsnaps: keepsnaps.append(src_snap) else: siadata, sia_status_code = post_to_sia( "renter/delete/%s" % siafile["siapath"], "") if sia_status_code != 204: return False, "ERROR: sia delete error %s: %s" % ( sia_status_code, siadata['message']) current_app.logger.info( "The following snapshots need to be kept for potential uploads: %s", keepsnaps) current_app.logger.info( "Removing possibly unneeded old lower-level data snapshots") for snap in glob(path.join(DATADIR_MASK, "snapshots", "*")): snapname = path.basename(snap) zipname = join(METADATA_BASE, "backup.%s.zip" % snapname) dirname = join(METADATA_BASE, "backup.%s" % snapname) if (not path.isfile(zipname) and not path.isdir(dirname) and not snapname in activebackups and not snapname in keepsnaps): subprocess.call([BTRFS, 'subvolume', 'delete', snap]) return True, ""
def initiate_uploads(status): current_app.logger.info('Starting uploads.') status["message"] = "Starting uploads" status["step"] = sys._getframe().f_code.co_name.replace("_", " ") status["starttime_step"] = time.time() snapname = status["snapname"] backupname = status["backupname"] metadir = path.join(METADATA_BASE, backupname) bfinfo_path = path.join(metadir, INFO_FILENAME) if path.isfile(bfinfo_path): remove(bfinfo_path) sia_filedata, sia_status_code = get_from_sia('renter/files') if sia_status_code == 200: siafiles = sia_filedata["files"] else: return False, "ERROR: sia daemon needs to be running for any uploads." # We have a randomly named subdirectory containing the .dat files. # The subdirectory matches the serial number that MineBD returns. mbdata, mb_status_code = get_from_minebd('serialnumber') if mb_status_code == 200: mbdirname = mbdata["message"] else: return False, "ERROR: Could not get serial number from MineBD." status["backupfileinfo"] = [] status["backupfiles"] = [] status["uploadfiles"] = [] status["backupsize"] = 0 status["uploadsize"] = 0 status["min_redundancy"] = None status["earliest_expiration"] = None for filepath in glob( path.join(DATADIR_MASK, 'snapshots', snapname, mbdirname, '*.dat')): fileinfo = stat(filepath) # Only use files of non-zero size. if fileinfo.st_size: filename = path.basename(filepath) (froot, fext) = path.splitext(filename) sia_fname = '%s.%s%s' % (froot, int(fileinfo.st_mtime), fext) if sia_fname in status["backupfiles"]: # This file is already in the list, and we probably have # multiple lower disks, so omit this file. continue status["backupfiles"].append(sia_fname) status["backupsize"] += fileinfo.st_size if (siafiles and any(sf["siapath"] == sia_fname and sf["available"] and sf["redundancy"] > REDUNDANCY_LIMIT for sf in siafiles)): current_app.logger.info( "%s is part of the set and already uploaded." % sia_fname) elif (siafiles and any(sf["siapath"] == sia_fname for sf in siafiles)): status["uploadsize"] += fileinfo.st_size status["uploadfiles"].append(sia_fname) current_app.logger.info( "%s is part of the set and the upload is already in progress." % sia_fname) else: status["uploadsize"] += fileinfo.st_size status["uploadfiles"].append(sia_fname) current_app.logger.info( "%s has to be uploaded, starting that." % sia_fname) siadata, sia_status_code = post_to_sia( 'renter/upload/%s' % sia_fname, {'source': filepath}) if sia_status_code != 204: return False, ("ERROR: sia upload error %s: %s" % (sia_status_code, siadata["message"])) status["backupfileinfo"].append({ "siapath": sia_fname, "size": fileinfo.st_size }) if not status["backupfiles"]: return False, "ERROR: The backup set has no files, that's impossible." with open(bfinfo_path, 'w') as outfile: json.dump(status["backupfileinfo"], outfile) return True, ""
def _rebalance_hosting_to_ratio(): settings = get_sia_config() folderdata = {} if not settings["minebox_sharing"]["enabled"]: current_app.logger.warn( "Sharing not enabled, cannot rebalance hosting space.") return False siadata, sia_status_code = get_from_sia("host/storage") if sia_status_code >= 400: current_app.logger.error("Sia error %s: %s" % (sia_status_code, siadata["message"])) return False used_space = 0 if siadata["folders"]: for folder in siadata["folders"]: folderdata[folder["path"]] = folder used_space += (folder["capacity"] - folder["capacityremaining"]) success = True for basepath in glob(HOST_DIR_BASE_MASK): hostpath = os.path.join(basepath, HOST_DIR_NAME) if not os.path.isdir(hostpath): subprocess.call( ['/usr/sbin/btrfs', 'subvolume', 'create', hostpath]) # Make sure the directory is owned by the sia user and group. uid = pwd.getpwnam("sia").pw_uid gid = grp.getgrnam("sia").gr_gid os.chown(hostpath, uid, gid) hostspace = _get_btrfs_space(hostpath) if hostspace: # We need to add already used hosting space to free space in this # calculation, otherwise hosted files would reduce the hosting # capacity! raw_size = ((hostspace["free_est"] + used_space) * settings["minebox_sharing"]["shared_space_ratio"]) # The actual share size must be a multiple of the granularity. share_size = int( (raw_size // SIA_HOST_GRANULARITY) * SIA_HOST_GRANULARITY) else: share_size = 0 if hostpath in folderdata: # Existing folder, (try to) resize it if necessary. if folderdata[hostpath]["capacity"] != share_size: current_app.logger.info( "Resize Sia hosting space at %s to %s MB.", hostpath, int(share_size // 2**20)) siadata, sia_status_code = post_to_sia( "host/storage/folders/resize", { "path": hostpath, "newsize": share_size }) if sia_status_code >= 400: current_app.logger.error( "Sia error %s: %s" % (sia_status_code, siadata["message"])) success = False else: # New folder, add it. current_app.logger.info("Add Sia hosting space at %s with %s MB.", hostpath, int(share_size // 2**20)) siadata, sia_status_code = post_to_sia("host/storage/folders/add", { "path": hostpath, "size": share_size }) if sia_status_code >= 400: current_app.logger.error("Sia error %s: %s" % (sia_status_code, siadata["message"])) success = False return success
def api_wallet_transactions(): # Doc: https://bitbucket.org/mineboxgmbh/minebox-client-tools/src/master/doc/mb-ui-gateway-api.md#markdown-header-get-wallettransactions if not check_login(): return jsonify( message="Unauthorized access, please log into the main UI."), 401 # Do something similar to |siac wallet transactions|, see # https://github.com/NebulousLabs/Sia/blob/master/cmd/siac/walletcmd.go#L443 siadata, sia_status_code = get_from_sia( "wallet/transactions?startheight=%s&endheight=%s" % (0, 10000000)) if sia_status_code >= 400: return jsonify(siadata), sia_status_code showsplits = bool(strtobool(request.args.get("showsplits") or "false")) onlyconfirmed = bool( strtobool(request.args.get("onlyconfirmed") or "false")) tdata = [] alltypes = ["confirmed"] if not onlyconfirmed: alltypes.append("unconfirmed") for ttype in alltypes: for trans in siadata["{0}transactions".format(ttype)]: # Note that inputs into a transaction are outgoing currency and outputs # are incoming, actually. txn = { "type": ttype, "height": trans["confirmationheight"], "timestamp": trans["confirmationtimestamp"], "transactionid": trans["transactionid"], "incoming": {}, "outgoing": {}, "change": 0, "fundschange": 0, } for t_input in trans["inputs"]: if t_input["walletaddress"]: # Only process the rest if the address is owned by the wallet. if t_input["fundtype"] in txn["outgoing"]: txn["outgoing"][t_input["fundtype"]] += int( t_input["value"]) else: txn["outgoing"][t_input["fundtype"]] = int( t_input["value"]) if t_input["fundtype"].startswith("siafund"): txn["fundschange"] -= int(t_input["value"]) else: txn["change"] -= int(t_input["value"]) for t_output in trans["outputs"]: if t_output["walletaddress"]: # Only process the rest if the address is owned by the wallet. if t_output["fundtype"] in txn["incoming"]: txn["incoming"][t_output["fundtype"]] += int( t_output["value"]) else: txn["incoming"][t_output["fundtype"]] = int( t_output["value"]) if t_input["fundtype"].startswith("siafund"): txn["fundschange"] += int(t_output["value"]) else: txn["change"] += int(t_output["value"]) # Convert into data that can be put into JSON properly. # This also adds _sc values for anything in hastings (not siafunds). txndata = { "confirmed": txn["type"] == "confirmed", "height": txn["height"], "timestamp": txn["timestamp"], "transactionid": txn["transactionid"], "incoming": {}, "outgoing": {}, "incoming_sc": {}, "outgoing_sc": {}, "change": str(txn["change"]), "change_sc": txn["change"] / H_PER_SC, "fundschange": str(txn["fundschange"]), } for tdirection in ["outgoing", "incoming"]: for ftype in txn[tdirection]: txndata[tdirection][ftype] = str(txn[tdirection][ftype]) if not ftype.startswith("siafund"): txndata["%s_sc" % tdirection][ ftype] = txn[tdirection][ftype] / H_PER_SC # Only add transaction to display if it either has an actual change or # we want to show splits. if txn["change"] or txn["fundschange"] or showsplits: tdata.append(txndata) return jsonify(tdata), sia_status_code
def api_ping(): # This can be called to just have the service run something. # For example, we need to do this early after booting to restart backups # if needed (via @app.before_first_request). if not os.path.isfile(MACHINE_AUTH_FILE): app.logger.info( "Submit machine authentication to Minebox admin service.") success, errmsg = submit_machine_auth() if not success: app.logger.error(errmsg) # Look if we need to run some system maintenance tasks. # Do this here so it runs even if Sia and upper storage are down. # Note that in the case of updates being available for backup-service, # this results in a restart and the rest of the ping will not be executed. success, errmsg = system_maintenance() if not success: app.logger.error(errmsg) # Check for synced sia consensus as a prerequisite to everything else. success, errmsg = check_sia_sync() if not success: # Return early, we need a synced consensus to do anything. app.logger.debug(errmsg) app.logger.info( "Exiting because sia is not ready, let's check again on next ping." ) return "", 204 if not os.path.ismount(MINEBD_STORAGE_PATH): current_app.logger.info( "Upper storage is not mounted (yet), let's check again on next ping." ) return "", 204 # See if sia is fully set up and do init tasks if needed. # Setting up hosting is the last step, so if that is not active, we still # need to do something. walletdata, wallet_status_code = get_from_sia('wallet') hostdata, host_status_code = get_from_sia('host') if wallet_status_code == 200 and host_status_code == 200: if not hostdata["internalsettings"]["acceptingcontracts"]: # We need to seed the wallet, set up allowances and hosting, etc. setup_sia_system(walletdata, hostdata) elif not walletdata["unlocked"]: # We should unlock the wallet so new contracts can be made. unlock_sia_wallet() # Trigger a backup if the latest is older than 24h. timenow = int(time.time()) latestbackup = get_latest() timelatest = int(latestbackup) if latestbackup else 0 if timelatest < timenow - 24 * 3600: success, errmsg = check_backup_prerequisites() if success: bthread = start_backup_thread() # If no backup is active but the most recent one is not finished, # perform a restart of backups. active_backups = get_running_backups() if not active_backups: snapname = get_latest() if snapname: if not is_finished(snapname): restart_backups() else: # If the upload step is stuck (taking longer than 30 minutes), # we should restart the sia service. # See https://github.com/NebulousLabs/Sia/issues/1605 for tname in threadstatus: if (threadstatus[tname]["snapname"] in active_backups and threadstatus[tname]["step"] == "initiate uploads" and threadstatus[tname]["starttime_step"] < time.time() - 30 * 60): # This would return True for success but already logs errors. restart_sia() # If the list of unfinished backups is significantly larger than active # backups, we very probably have quite a few backups hanging around # that we need to cleanup but don't get to routine cleanup (which # happens only when a backup finishes). unfinished_backups = get_list() if len(unfinished_backups) > len(active_backups) + 3: app.logger.info( "We have %s unfinished backups but only %s active ones, let's clean up." % (len(unfinished_backups), len(active_backups))) start_cleanup_thread() # Update Sia config if more than 10% off. update_sia_config() # See if we need to rebalance the disk space. rebalance_diskspace() return "", 204