def daemon(): if args.daemon_type[0] == "ipfs": from broker.utils import run_ipfs_daemon run_ipfs_daemon(_is_print=True) if args.daemon_type[0] == "slurm": from broker.config import env from broker.utils import run run(["sudo", env.BASH_SCRIPTS_PATH / "run_slurm.sh"])
def get_ipfs_id() -> str: run_ipfs_daemon() try: # may create error client = ipfshttpclient.connect("/ip4/127.0.0.1/tcp/5001/http") except ipfshttpclient.exceptions.ConnectionError: log("E: Failed to establish a new connection to IPFS, please run it on the background.\n" "Please run [magenta]~/ebloc-broker/broker/_daemons/ipfs.py") sys.exit(1) except Exception as e: print_tb(e) log("E: Failed to establish a new connection to IPFS, please run it on the background.\n" "Please run [magenta]~/ebloc-broker/broker/_daemons/ipfs.py") sys.exit(1) try: return cfg.ipfs.get_ipfs_id(client) except Exception as e: print_tb(str(e)) sys.exit(1)
def pre_check(job, requester): """Pre check jobs to submit.""" try: job.check_account_status(requester) is_bin_installed("ipfs") if not is_dpkg_installed("pigz"): log("E: Install [green]pigz[/green].\nsudo apt install -y pigz") sys.exit() if not os.path.isfile(env.GPG_PASS_FILE): log(f"E: Please store your gpg password in the [magenta]{env.GPG_PASS_FILE}[/magenta]\nfile for decrypting") raise QuietExit run_ipfs_daemon() if job.storage_ids[0] == StorageID.IPFS: for storage_id in job.storage_ids[1:]: if storage_id in (StorageID.GDRIVE, StorageID.EUDAT): raise Exception( "If source code is submitted via IPFS, then data files must be submitted using IPFS or IPFS_GPG" ) except Exception as e: print_tb(e) sys.exit()
def run(self) -> bool: self.start_time = time.time() if cfg.IS_THREADING_ENABLED: self.thread_log_setup() run_ipfs_daemon() log(f"{br(get_date())} Job's source code has been sent through ", "bold cyan", end="") if self.cloudStorageID[0] == StorageID.IPFS: log("[bold green]IPFS") else: log("[bold green]IPFS_GPG") if not is_ipfs_on(): return False log(f"==> is_hash_locally_cached={cfg.ipfs.is_hash_locally_cached(self.job_key)}" ) if not os.path.isdir(self.results_folder): os.makedirs(self.results_folder) _remove(f"{self.results_folder}/{self.job_key}") try: self.check_ipfs(self.job_key) except: return False self.registered_data_hashes = [] for idx, source_code_hash in enumerate(self.code_hashes): if self.cloudStorageID[idx] == StorageID.NONE: self.registered_data_hashes.append(source_code_hash) # GOTCHA else: ipfs_hash = bytes32_to_ipfs(source_code_hash) if ipfs_hash not in self.ipfs_hashes: try: # job_key as data hash already may added to the list self.check_ipfs(ipfs_hash) except: return False initial_folder_size = calculate_size(self.results_folder) for idx, ipfs_hash in enumerate(self.ipfs_hashes): # here scripts knows that provided IPFS hashes exists online is_hashed = False log(f"## attempting to get IPFS file: {ipfs_hash} ... ", end="") if cfg.ipfs.is_hash_locally_cached(ipfs_hash): is_hashed = True log(ok("already cached")) else: log() if idx == 0: target = self.results_folder else: # "_" added before the filename in case $ ipfs get <ipfs_hash> target = f"{self.results_data_folder}/_{ipfs_hash}" mkdir(target) is_storage_paid = False # TODO: should be set before by user input cfg.ipfs.get(ipfs_hash, target, is_storage_paid) if idx > 0: # https://stackoverflow.com/a/31814223/2402577 dst_filename = os.path.join(self.results_data_folder, os.path.basename(ipfs_hash)) if os.path.exists(dst_filename): _remove(dst_filename) shutil.move(target, dst_filename) target = dst_filename if self.cloudStorageID[idx] == StorageID.IPFS_GPG: cfg.ipfs.decrypt_using_gpg(f"{target}/{ipfs_hash}", target) try: _git.initialize_check(target) except Exception as e: raise e if not is_hashed: folder_size = calculate_size(self.results_folder) self.data_transfer_in_to_download_mb += folder_size - initial_folder_size initial_folder_size = folder_size if idx == 0 and not self.check_run_sh(): self.complete_refund() return False log(f"==> data_transfer_in={self.data_transfer_in_to_download_mb} MB | " f"rounded={int(self.data_transfer_in_to_download_mb)} MB") return self.sbatch_call()
def _tools(block_continue): # noqa """Check whether the required functions are running or not. :param block_continue: Continue from given block number """ session_start_msg(env.SLURMUSER, block_continue, pid) try: if not is_internet_on(): raise Terminate("Network connection is down. Please try again") if not check_ubuntu_packages(): raise Terminate() if not env.IS_BLOXBERG: is_geth_on() else: log(":beer: Connected into [green]BLOXBERG[/green]", "bold") if not Contract.Ebb.is_orcid_verified(env.PROVIDER_ID): log(f"warning: provider [green]{env.PROVIDER_ID}[/green]'s orcid id is not authenticated yet") raise QuietExit slurm.is_on() if not is_process_on("mongod"): raise Exception("mongodb is not running in the background") # run_driver_cancel() # TODO: uncomment if env.IS_EUDAT_USE: if not env.OC_USER: raise Terminate(f"OC_USER is not set in {env.LOG_PATH.joinpath('.env')}") eudat.login(env.OC_USER, env.LOG_PATH.joinpath(".eudat_client.txt"), env.OC_CLIENT) if env.IS_GDRIVE_USE: is_program_valid(["gdrive", "version"]) if env.GDRIVE == "": raise Terminate(f"E: Path for gdrive='{env.GDRIVE}' please set a valid path in the .env file") provider_info = Contract.Ebb.get_provider_info(env.PROVIDER_ID) email = provider_info["email"] try: output, gdrive_email = gdrive.check_user(email) except Exception as e: print_tb(e) raise QuietExit from e if not output: log( f"E: provider's registered email=[magenta]{email}[/magenta] does not match\n" f" with the set gdrive's email=[magenta]{gdrive_email}[/magenta]" ) raise QuietExit log(f"==> provider_email=[magenta]{email}") if env.IS_IPFS_USE: if not os.path.isfile(env.GPG_PASS_FILE): log(f"E: Please store your gpg password in the {env.GPG_PASS_FILE}\nfile for decrypting using ipfs") raise QuietExit if not os.path.isdir(env.IPFS_REPO): log(f"E: {env.IPFS_REPO} does not exist") raise QuietExit run_ipfs_daemon() except QuietExit as e: raise e except Exception as e: print_tb(e) raise Terminate from e
def register_requester(self, yaml_fn, is_question=True): """Register or update requester into smart contract.""" yaml_fn = os.path.expanduser(yaml_fn) try: run_ipfs_daemon() client = ipfshttpclient.connect("/ip4/127.0.0.1/tcp/5001/http") except Exception as e: log("E: Run ipfs daemon to detect your ipfs_id") print_tb(e) sys.exit(1) if not os.path.exists(yaml_fn): log(f"E: yaml_fn({yaml_fn}) does not exist") raise QuietExit args = Yaml(yaml_fn) ipfs_id = cfg.ipfs.get_ipfs_id(client) email = env.GMAIL gpg_fingerprint = get_gpg_fingerprint(email) try: is_gpg_published(gpg_fingerprint) except Exception as e: raise e account = args["config"]["account"].lower() email = args["config"]["email"] federation_cloud_id = args["config"]["federation_cloud_id"] # if env.IS_BLOXBERG: # account = self.brownie_load_account().address log(f"==> registering {account} as requester") if is_byte_str_zero(account): log(f"E: account={account} is not valid, change it in [{c.pink}]~/.ebloc-broker/cfg.yaml" ) raise QuietExit if len(federation_cloud_id) >= 128: raise Exception("E: federation_cloud_id is more than 128") if len(email) >= 128: raise Exception("E: email is more than 128") if len(gpg_fingerprint) != 40: raise Exception("E: gpg_fingerprint should be 40 characters") if self.does_requester_exist(account): log(f"warning: requester {account} is already registered") requester_info = Ebb.get_requester_info(account) if (requester_info["email"] == email and requester_info["gpg_fingerprint"] == gpg_fingerprint and requester_info["ipfs_id"] == ipfs_id and requester_info["f_id"] == federation_cloud_id): log(requester_info) log("## Same requester information is provided, nothing to do") raise QuietExit log("==> [bold yellow]registered_requester_info:") log(requester_info) _requester_info = { "email": email, "federation_cloud_id": federation_cloud_id, "gpg_fingerprint": gpg_fingerprint, "ipfs_id": ipfs_id, } log("==> [bold yellow]new_requester_info:") log(_requester_info) if is_question and not question_yes_no( "#> Would you like to update requester info?"): return try: tx = self._register_requester(account, gpg_fingerprint, email, federation_cloud_id, ipfs_id) return self.tx_id(tx) except Exception as e: print_tb(e) raise e