def is_geth_on(): """Check whether geth runs on the background.""" process_name = f"geth@{env.RPC_PORT}" if not is_process_on(process_name, "Geth", process_count=0): log(f"E: geth is not running on the background, {process_name}. Please run:") log("sudo ~/eBlocPOA/server.sh", "bold yellow") raise QuietExit
def main(): check_gdrive_user() console_ruler(f"NEW_TEST {Ebb.get_block_number()}") log(f" * {datetime.now().strftime('%Y-%m-%d %H:%M')}") if not is_process_on("mongod", "mongod"): raise Exception("mongodb is not running in the background") storage_ids = ["eudat", "gdrive", "ipfs"] ipfs_ids = ["ipfs_gpg", "ipfs"] # for provider_address in provider_addresses: # pre_submit(storage_ids, provider_address) benchmarks = ["nas", "cppr"] test_dir = Path.home() / "ebloc-broker" / "broker" / "test_setup" / "nas" nas_yaml_fn = test_dir / "job_nas.yaml" cppr_yam_fn = test_dir / "job_cppr.yaml" yaml_cfg = None counter = 0 for _ in range(60): for _ in range(2): # submitted as batch is faster for idx, provider_address in enumerate(provider_addresses): # yaml_cfg["config"]["data"]["data3"]["storage_id"] = random.choice(storage_ids) storage_id = (idx + counter) % len(storage_ids) selected_benchmark = random.choice(benchmarks) storage = storage_ids[storage_id] if storage == "ipfs": storage = random.choice(ipfs_ids) if selected_benchmark == "nas": log( f" * Submitting job from NAS Benchmark to [green]{provider_address}", "blue") yaml_cfg = Yaml(nas_yaml_fn) benchmark_name = create_nas_job_script() elif selected_benchmark == "cppr": log( f" * Submitting job with cppr datasets to [green]{provider_address}", "blue") yaml_cfg = Yaml(cppr_yam_fn) hash_small_data, hash_med_data = create_cppr_job_script() yaml_cfg["config"]["data"]["data1"][ "hash"] = hash_small_data yaml_cfg["config"]["data"]["data2"]["hash"] = hash_med_data yaml_cfg["config"]["data"]["data3"]["storage_id"] = storage small_datasets = Path.home( ) / "test_eblocbroker" / "dataset_zip" / "small" dirs = [ d for d in os.listdir(small_datasets) if os.path.isdir(os.path.join(small_datasets, d)) ] dir_name = random.choice(dirs) yaml_cfg["config"]["data"]["data3"]["path"] = str( small_datasets / dir_name) yaml_cfg["config"]["source_code"]["storage_id"] = storage yaml_cfg["config"]["provider_address"] = provider_address try: submit_base = SubmitBase(yaml_cfg.path) submission_date = _date() submission_timestamp = _timestamp() requester_address = random.choice(users).lower() yaml_cfg["config"]["requester_address"] = requester_address log(f"requester={requester_address}", "bold") tx_hash = submit_base.submit(is_pass=True) log(f"tx_hash={tx_hash}", "bold") tx_receipt = get_tx_status(tx_hash, is_silent=True) if tx_receipt["status"] == 1: processed_logs = Ebb._eBlocBroker.events.LogJob( ).processReceipt(tx_receipt, errors=DISCARD) job_result = vars(processed_logs[0].args) job_result["submit_date"] = submission_date job_result["submit_timestamp"] = submission_timestamp job_result["tx_hash"] = tx_hash if selected_benchmark == "nas": job_result[ "submitted_job_kind"] = f"{selected_benchmark}_{benchmark_name}" elif selected_benchmark == "cppr": job_result[ "submitted_job_kind"] = f"{selected_benchmark}_{hash_small_data}_{hash_med_data}" ebb_mongo.add_item(tx_hash, job_result) log(job_result) countdown(seconds=5, is_silent=True) except Exception as e: print_tb(e) counter += 1 sleep_time = randint(200, 400) countdown(sleep_time)
def is_ganache_on(port) -> bool: """Check whether Ganache CLI runs on the background.""" return is_process_on("node.*[g]anache-cli", "Ganache CLI", process_count=0, port=port)
def is_driver_on(process_count=0, is_print=True): """Check whether driver runs on the background.""" if is_process_on("python.*[D]river", "Driver", process_count, is_print=is_print): log(f"## Track output using:\n[blue]tail -f {_log.DRIVER_LOG}") raise QuietExit
def is_ipfs_on(is_print=False) -> bool: """Check whether ipfs runs on the background.""" return is_process_on("[i]pfs\ daemon", "IPFS", process_count=0, is_print=is_print)
def _tools(block_continue): # noqa """Check whether the required functions are running or not. :param block_continue: Continue from given block number """ session_start_msg(env.SLURMUSER, block_continue, pid) try: if not is_internet_on(): raise Terminate("Network connection is down. Please try again") if not check_ubuntu_packages(): raise Terminate() if not env.IS_BLOXBERG: is_geth_on() else: log(":beer: Connected into [green]BLOXBERG[/green]", "bold") if not Contract.Ebb.is_orcid_verified(env.PROVIDER_ID): log(f"warning: provider [green]{env.PROVIDER_ID}[/green]'s orcid id is not authenticated yet") raise QuietExit slurm.is_on() if not is_process_on("mongod"): raise Exception("mongodb is not running in the background") # run_driver_cancel() # TODO: uncomment if env.IS_EUDAT_USE: if not env.OC_USER: raise Terminate(f"OC_USER is not set in {env.LOG_PATH.joinpath('.env')}") eudat.login(env.OC_USER, env.LOG_PATH.joinpath(".eudat_client.txt"), env.OC_CLIENT) if env.IS_GDRIVE_USE: is_program_valid(["gdrive", "version"]) if env.GDRIVE == "": raise Terminate(f"E: Path for gdrive='{env.GDRIVE}' please set a valid path in the .env file") provider_info = Contract.Ebb.get_provider_info(env.PROVIDER_ID) email = provider_info["email"] try: output, gdrive_email = gdrive.check_user(email) except Exception as e: print_tb(e) raise QuietExit from e if not output: log( f"E: provider's registered email=[magenta]{email}[/magenta] does not match\n" f" with the set gdrive's email=[magenta]{gdrive_email}[/magenta]" ) raise QuietExit log(f"==> provider_email=[magenta]{email}") if env.IS_IPFS_USE: if not os.path.isfile(env.GPG_PASS_FILE): log(f"E: Please store your gpg password in the {env.GPG_PASS_FILE}\nfile for decrypting using ipfs") raise QuietExit if not os.path.isdir(env.IPFS_REPO): log(f"E: {env.IPFS_REPO} does not exist") raise QuietExit run_ipfs_daemon() except QuietExit as e: raise e except Exception as e: print_tb(e) raise Terminate from e
def run_driver_cancel(): """Run driver_cancel daemon on the background.""" if not is_process_on("python.*[d]river_cancel", "driver_cancel"): # Running driver_cancel.py on the background if it is not already config.driver_cancel_process = subprocess.Popen( ["python3", "driver_cancel.py"])