def main(): coll = cl["eBlocBroker"]["cache"] block_number = Ebb.get_block_number() storageID = None cursor = coll.find({}) for document in cursor: # print(document) received_block_number, storage_duration = Ebb.get_job_storage_duration( env.PROVIDER_ID, document["sourceCodeHash"] ) end_block_time = received_block_number + storage_duration * cfg.BLOCK_DURATION_1_HOUR storageID = document["storageID"] if end_block_time < block_number and received_block_number != 0: if storageID in (StorageID.IPFS, StorageID.IPFS_GPG): ipfsHash = document["jobKey"] print(run(["ipfs", "pin", "rm", ipfsHash])) print(run(["ipfs", "repo", "gc"])) else: cached_file_name = ( env.PROGRAM_PATH / document["requesterID"] / "cache" / document["sourceCodeHash"] + "tar.gz" ) print(cached_file_name) _remove(cached_file_name) cached_file_name = env.PROGRAM_PATH / "cache" / document["sourceCodeHash"] + "tar.gz" print(cached_file_name) _remove(cached_file_name) print(received_block_number) coll.delete_one({"jobKey": ipfsHash})
def process_logged_job(self, idx): """Process logged job one by one.""" self.received_block = [] self.storage_duration = [] wait_until_idle_core_available() self.is_provider_received_job = True console_ruler(idx, character="-") # sourceCodeHash = binascii.hexlify(logged_job.args['sourceCodeHash'][0]).decode("utf-8")[0:32] job_key = self.logged_job.args["jobKey"] index = self.logged_job.args["index"] self.job_block_number = self.logged_job["blockNumber"] self.cloud_storage_id = self.logged_job.args["cloudStorageID"] log(f"## job_key=[magenta]{job_key}[/magenta] | index={index}") log( f"received_block_number={self.job_block_number} \n" f"transactionHash={self.logged_job['transactionHash'].hex()} | " f"log_index={self.logged_job['logIndex']} \n" f"provider={self.logged_job.args['provider']} \n" f"received={self.logged_job.args['received']}", "bold yellow", ) if self.logged_job["blockNumber"] > self.latest_block_number: self.latest_block_number = self.logged_job["blockNumber"] try: run([env.BASH_SCRIPTS_PATH / "is_str_valid.sh", job_key]) except: logging.error("E: Filename contains an invalid character") return try: job_id = 0 # main job_id self.job_info = eblocbroker_function_call( partial(self.Ebb.get_job_info, env.PROVIDER_ID, job_key, index, job_id, self.job_block_number), max_retries=10, ) cfg.Ebb.get_job_code_hashes(env.PROVIDER_ID, job_key, index, self.job_block_number) self.requester_id = self.job_info["job_owner"] self.job_info.update({"received_block": self.received_block}) self.job_info.update({"storage_duration": self.storage_duration}) self.job_info.update({"cacheType": self.logged_job.args["cacheType"]}) cfg.Ebb.analyze_data(job_key, env.PROVIDER_ID) self.job_infos.append(self.job_info) log(f"==> requester={self.requester_id}") log("==> [yellow]job_info:", "bold") log(self.job_info) except Exception as e: print_tb(e) return for job in range(1, len(self.job_info["core"])): with suppress(Exception): self.job_infos.append( # if workflow is given then add jobs into list self.Ebb.get_job_info(env.PROVIDER_ID, job_key, index, job, self.job_block_number) ) self.check_requested_job()
def is_repo(folders): for folder in folders: if not isinstance(folder, bytes): with cd(folder): if not is_initialized(folder): log(f"warning: .git does not exits in {folder}. Applying: git init ", end="") run(["git", "init", "--initial-branch=master"]) log(ok())
def daemon(): if args.daemon_type[0] == "ipfs": from broker.utils import run_ipfs_daemon run_ipfs_daemon(_is_print=True) if args.daemon_type[0] == "slurm": from broker.config import env from broker.utils import run run(["sudo", env.BASH_SCRIPTS_PATH / "run_slurm.sh"])
def run_stdout_to_file(cmd, path, mode="w") -> None: """Run command pipe output into give file.""" p, output, error = popen_communicate(cmd, stdout_fn=path, mode=mode) if p.returncode != 0 or (isinstance(error, str) and "error:" in error): _cmd = " ".join(cmd) log(f"\n{_cmd}", "red") raise Exception(f"E: scontrol error:\n{output}") # log(f"## writing into path({path}) is completed") run(["sed", "-i", "s/[ \t]*$//", path]) # remove trailing whitespaces with sed
def initialize_check(path): """Validate if .git/ folder exist within the target folder.""" with cd(path): if not is_initialized(path): try: log(f"## git_repo={path}") log("Creating an empty Git repository ", end="") run(["git", "init", "--initial-branch=master"]) log(ok()) add_all() except Exception as e: log(f"E: {e}") raise e
def decrypt_using_gpg(self, gpg_file, extract_target=None): """Decrypt compresses file using gpg. This function is specific for using on driver.ipfs to decript tar file, specific for "tar.gz" file types. cmd: gpg --verbose --output={tar_file} --pinentry-mode loopback \ --passphrase-file=f"{env.LOG_PATH}/gpg_pass.txt" \ --decrypt {gpg_file_link} """ if not os.path.isfile(f"{gpg_file}.gpg"): os.symlink(gpg_file, f"{gpg_file}.gpg") gpg_file_link = f"{gpg_file}.gpg" tar_file = f"{gpg_file}.tar.gz" cmd = [ "gpg", "--verbose", "--batch", "--yes", f"--output={tar_file}", "--pinentry-mode", "loopback", f"--passphrase-file={env.GPG_PASS_FILE}", "--decrypt", gpg_file_link, ] try: run(cmd) log(f"==> GPG decrypt {ok()}") _remove(gpg_file) os.unlink(gpg_file_link) except Exception as e: print_tb(e) raise e # finally: # os.unlink(gpg_file_link) if extract_target: try: untar(tar_file, extract_target) except Exception as e: raise Exception( "E: Could not extract the given tar file") from e finally: cmd = None _remove(f"{extract_target}/.git") _remove(tar_file)
def commit_changes(path): with cd(path): repo = git.Repo(".", search_parent_directories=True) try: output = run(["ls", "-l", ".git/refs/heads"]) except Exception as e: raise Exception("E: Problem on git.commit_changes()") from e if output == "total 0": logging.warning("There is no first commit") else: changed_files = [item.a_path for item in repo.index.diff(None)] if len(changed_files) > 0: log("==> adding changed files:") for _file in changed_files: log(_file, "bold") repo.git.add(A=True) if len(repo.index.diff("HEAD")) == 0: log(f"==> {path}\n is committed with the given changes using git" ) try: add_all(repo) except Exception as e: log(f"E: {e}") raise e
def diff_patch(path: Path, source_code_hash, index, target_path): """Apply diff patch. "git diff HEAD" for detecting all the changes: Shows all the changes between the working directory and HEAD (which includes changes in the index). This shows all the changes since the last commit, whether or not they have been staged for commit or not. """ is_file_empty = False with cd(path): log(f"==> Navigate to {path}") """TODO if not is_initialized(path): upload everything, changed files! """ repo = git.Repo(".", search_parent_directories=True) try: repo.git.config("core.fileMode", "false") # git config core.fileMode false # first ignore deleted files not to be added into git run([env.BASH_SCRIPTS_PATH / "git_ignore_deleted.sh"]) head_commit_id = repo.rev_parse("HEAD") sep = "~" # separator in between the string infos patch_name = f"patch{sep}{head_commit_id}{sep}{source_code_hash}{sep}{index}.diff" except: return False patch_upload_name = f"{patch_name}.gz" # file to be uploaded as zip patch_file = f"{target_path}/{patch_upload_name}" logging.info(f"patch_path={patch_upload_name}") try: repo.git.add(A=True) diff_and_gzip(patch_file) except: return False time.sleep(0.25) if is_gzip_file_empty(patch_file): log("==> Created patch file is empty, nothing to upload") with suppress(Exception): os.remove(patch_upload_name) os.remove(patch_file) is_file_empty = True return patch_upload_name, patch_file, is_file_empty
def apply_patch(git_folder, patch_file, is_gpg=False): """Apply git patch. output = repo.git.apply("--reject", "--whitespace=fix", "--ignore-space-change", "--ignore-whitespace", "--verbose", patch_file) __ https://stackoverflow.com/a/15375869/2402577 """ if is_gpg: cfg.ipfs.decrypt_using_gpg(patch_file) with cd(git_folder): base_name = path_leaf(patch_file) log(f"==> [magenta]{base_name}") # folder_name = base_name_split[2] # # base_name_split = base_name.split("_") # git_hash = base_name_split[1] # run(["git", "checkout", git_hash]) # run(["git", "reset", "--hard"]) # run(["git", "clean", "-f"]) # echo "\n" >> patch_file.txt seems like fixing it # # with open(patch_file, "a") as myfile: # myfile.write("\n") cmd = [ "git", "apply", "--reject", "--whitespace=fix", "--ignore-space-change", "--ignore-whitespace", "--verbose", patch_file, ] # ,is_quiet=True, cmd_summary = cmd.copy() cmd_summary.insert(3, "--summary") output = run(cmd_summary) log(output) output = run(cmd) log(output)
def connect_into_web3() -> None: """Connect into private network using web3. Note that you should create only one RPC Provider per process, as it recycles underlying TCP/IP network connections between your process and Ethereum node """ web3_ipc_path = env.DATADIR.joinpath("geth.ipc") for _ in range(5): _connect_into_web3() if not cfg.w3.isConnected(): try: if env.IS_GETH_TUNNEL: raise Exception( "web3ConnectError: try tunnelling: ssh -f -N -L 8545:localhost:8545 username@<ip>" ) if not env.IS_BLOXBERG: is_geth_on() else: log("E: web3 is not connected into [green]BLOXBERG[/green]" ) except QuietExit: pass except Exception as e: print_tb(e) sys.exit(1) if not env.IS_GETH_TUNNEL and not env.IS_BLOXBERG: log( "E: If web3 is not connected please start geth server and give permission \n" "to /private/geth.ipc file doing: ", end="", ) log(f"sudo chown $(logname) {web3_ipc_path}", "green") log(f"#> Running `sudo chown $(whoami) {web3_ipc_path}`") run(["sudo", "chown", env.WHOAMI, web3_ipc_path]) else: break else: terminate(is_traceback=False)
def new_contract_function_call(self, code_hash): """Call contract function with new brownie object. This part is not needed when `{from: }` is removed from the getter in the smart contract """ filename = call.__file__ data = ("func", self.job.provider, self.job.requester, code_hash) output = run(["python", filename, *[str(arg) for arg in data]]) # GOTCHA output = output.split("\n") received_storage_deposit = float(output[0]) job_storage_duration = make_tuple(output[1]) return received_storage_deposit, job_storage_duration
def is_hash_locally_cached(self, ipfs_hash: str, ipfs_refs_local=None) -> bool: """Return true if hash locally cached. Run `ipfs --offline refs -r` or `ipfs --offline block stat` etc even if your normal daemon is running. With that you can check if something is available locally or no. """ if not ipfs_refs_local: ipfs_refs_local = run(["ipfs", "refs", "local"]).split("\n") for _hash in ipfs_refs_local: if ipfs_hash == _hash: return True return False
def get_only_ipfs_hash(self, path, is_hidden=True) -> str: """Get only chunk and hash of a given path, do not write to disk. Args: path: Path of a folder or file Return string that contains the ouput of the run commad. """ if os.path.isdir(path): cmd = ["ipfs", "add", "-r", "--quieter", "--only-hash", path] if is_hidden: # include files that are hidden such as .git/ # Only takes effect on recursive add cmd.insert(3, "--hidden") elif os.path.isfile(path): cmd = ["ipfs", "add", "--quieter", "--only-hash", path] else: raise Exception("Requested path does not exist") try: return _try(lambda: run(cmd)) except Exception as e: raise e
def gpg_encrypt(self, user_gpg_finderprint, target): is_delete = False if os.path.isdir(target): try: *_, encrypt_target = compress_folder(target) encrypted_file_target = f"{encrypt_target}.gpg" is_delete = True except Exception as e: print_tb(e) sys.exit(1) else: if not os.path.isfile(target): logging.error(f"{target} does not exist") sys.exit(1) else: encrypt_target = target encrypted_file_target = f"{target}.gpg" is_delete = True if os.path.isfile(encrypted_file_target): log(f"## gpg_file: {encrypted_file_target} is already created") return encrypted_file_target for attempt in range(5): try: cmd = [ "gpg", "--keyserver", "hkps://keyserver.ubuntu.com", "--recv-key", user_gpg_finderprint ] log(f"{br(attempt)} cmd: [magenta]{' '.join(cmd)}", "bold") run( cmd ) # this may not work if it is requested too much in a short time break except Exception as e: log(f"warning: {e}") time.sleep(30) try: cmd = [ "gpg", "--batch", "--yes", "--recipient", user_gpg_finderprint, "--trust-model", "always", "--output", encrypted_file_target, "--encrypt", encrypt_target, ] run(cmd) log(f"==> gpg_file=[magenta]{encrypted_file_target}") return encrypted_file_target except Exception as e: print_tb(e) if "encryption failed: Unusable public key" in str(e): log("#> Check solution: https://stackoverflow.com/a/34132924/2402577" ) finally: if is_delete: _remove(encrypt_target)
if __name__ == "__main__": import call from broker.utils import run filename = call.__file__ func_name = "getStorageInfo" data = ( "", "0xD118b6EF83ccF11b34331F1E7285542dDf70Bc49", "0xD118b6EF83ccF11b34331F1E7285542dDf70Bc49", "QmdxhbcHJr3r4yeJNdvRabLVJ78VT2DTNMkBD7LN2MzeqJ", ) output = run(["python", filename, *[str(arg) for arg in data]]) print(output) # # in case filename is an executable you don't need "python" before `filename`: # try: # # for breakpoint remove , stdout=subprocess.PIPE # p = subprocess.Popen(args=["python", filename, *[str(arg) for arg in data]], stdout=subprocess.PIPE) # output, error = p.communicate() # p.wait() # time.sleep(1) # if output: # output = output.strip().decode("utf-8") # print(output) # except: # breakpoint() # DEBUG # pass
def pin(self, ipfs_hash: str) -> bool: return run(["ipfs", "pin", "add", ipfs_hash])
def run_driver(given_bn): """Run the main driver script for eblocbroker on the background.""" # dummy sudo command to get the password when session starts for only to # create users and submit the slurm job under another user run(["sudo", "printf", "hello"]) kill_process_by_name("gpg-agent") config.logging = setup_logger(_log.DRIVER_LOG) # driver_cancel_process = None try: from broker.imports import connect connect() Ebb: "Contract.Contract" = cfg.Ebb driver = Driver() except Exception as e: raise Terminate from e if not env.PROVIDER_ID: raise Terminate(f"PROVIDER_ID is None in {env.LOG_PATH}/.env") if not env.WHOAMI or not env.EBLOCPATH or not env.PROVIDER_ID: raise Terminate(f"Please run: {env.BASH_SCRIPTS_PATH}/folder_setup.sh") if not env.SLURMUSER: raise Terminate(f"SLURMUSER is not set in {env.LOG_PATH}/.env") try: deployed_block_number = Ebb.get_deployed_block_number() except Exception as e: raise e if not env.config["block_continue"]: env.config["block_continue"] = deployed_block_number if given_bn > 0: block_number_saved = int(given_bn) else: block_number_saved = env.config["block_continue"] if not isinstance(env.config["block_continue"], int): log("E: block_continue variable is empty or contains an invalid character") if not question_yes_no("#> Would you like to read from the contract's deployed block number?"): terminate() block_number_saved = deployed_block_number if deployed_block_number: env.config["block_continue"] = deployed_block_number else: raise Terminate(f"deployed_block_number={deployed_block_number} is invalid") _tools(block_number_saved) try: Ebb.is_contract_exists() except: terminate( "Contract address does not exist on the blockchain, is the blockchain sync?\n" f"block_number={Ebb.get_block_number()}", is_traceback=False, ) if cfg.IS_THREADING_ENABLED: log(f"## is_threading={cfg.IS_THREADING_ENABLED}") Ebb.is_eth_account_locked(env.PROVIDER_ID) log(f"==> whoami={env.WHOAMI}") log(f"==> log_file={_log.DRIVER_LOG}") log(f"==> rootdir={os.getcwd()}") log(f"==> is_web3_connected={Ebb.is_web3_connected()}") if not Ebb.does_provider_exist(env.PROVIDER_ID): # updated since cluster is not registered env.config["block_continue"] = Ebb.get_block_number() terminate( textwrap.fill( f"Your Ethereum address {env.PROVIDER_ID} " "does not match with any provider in eBlocBroker. Please register your " "provider using your Ethereum Address in to the eBlocBroker. You can " "use eblocbroker/register_provider.py script to register your provider." ), is_traceback=False, ) if not Ebb.is_orcid_verified(env.PROVIDER_ID): raise QuietExit(f"provider's ({env.PROVIDER_ID}) ORCID is not verified") blk_read = block_number_saved balance_temp = Ebb.get_balance(env.PROVIDER_ID) eth_balance = Ebb.eth_balance(env.PROVIDER_ID) log(f"==> deployed_block_number={deployed_block_number}") log(f"==> account_balance={eth_balance} gwei | {cfg.w3.fromWei(eth_balance, 'ether')} eth") log(f"==> Ebb_balance={balance_temp}") while True: wait_until_idle_core_available() time.sleep(0.2) if not str(blk_read).isdigit(): raise Terminate(f"block_read_from={blk_read}") balance = Ebb.get_balance(env.PROVIDER_ID) if cfg.IS_THREADING_ENABLED: _squeue() console_ruler() if isinstance(balance, int): value = int(balance) - int(balance_temp) if value > 0: log(f"==> Since Driver started provider_gained_wei={value}") current_bn = Ebb.get_block_number() log(f" * {get_date()} waiting new job to come since block_number={blk_read}") log(f"==> current_block={current_bn} | sync_from={blk_read}") flag = True while current_bn < int(blk_read): current_bn = Ebb.get_block_number() if flag: log(f"## Waiting block number to be updated, it remains constant at {current_bn}") flag = False time.sleep(2) log(f"#> [bold yellow]Passed incremented block number... Watching from block_number=[cyan]{blk_read}") blk_read = str(blk_read) # reading events' block number has been updated slurm.pending_jobs_check() try: driver.logged_jobs_to_process = Ebb.run_log_job(blk_read, env.PROVIDER_ID) driver.process_logged_jobs() if len(driver.logged_jobs_to_process) > 0 and driver.latest_block_number > 0: # updates the latest read block number blk_read = driver.latest_block_number + 1 env.config["block_continue"] = blk_read if not driver.is_provider_received_job: blk_read = env.config["block_continue"] = current_bn except Exception as e: log() log(f"E: {e}") if "Filter not found" in str(e) or "Read timed out" in str(e): # HTTPSConnectionPool(host='core.bloxberg.org', port=443): Read timed out. (read timeout=10) log("## sleeping for 60 seconds...", end="") time.sleep(60) log(ok()) else: print_tb(e)