def _update_data_price(): Ebb = cfg.Ebb if not Ebb.does_provider_exist(env.PROVIDER_ID): log(f"warning: Provider {env.PROVIDER_ID} is not registered.\n") raise QuietExit source_code_hash = "b6aaf03752dc68d625fc57b451faa2bf" new_data_price = 21 commitment_block_duration = 600 source_code_hash_bytes = cfg.w3.toBytes(text=source_code_hash) try: (price, _commitment_block_duration) = cfg.Ebb.get_registered_data_prices( env.PROVIDER_ID, source_code_hash_bytes, 0 ) if price == new_data_price and _commitment_block_duration == commitment_block_duration: log(f"## data([green]{source_code_hash}[/green]) already registerered with the given values") raise QuietExit except: raise QuietExit try: tx = Ebb.update_data_price(source_code_hash_bytes, new_data_price, commitment_block_duration) get_tx_status(Ebb.tx_id(tx)) except QuietExit: pass except Exception as e: print_tb(e)
def git_diff_patch_and_upload(self, source: Path, name, storage_class, is_job_key): if is_job_key: log(f"==> base_patch={self.patch_folder}") log(f"==> sourcecode_patch={name}") else: log(f"==> datafile_patch={name}") try: if storage_class is Ipfs or storage_class is IpfsGPG: target_path = self.patch_folder_ipfs else: target_path = self.patch_folder self.patch_upload_name, self.patch_file, is_file_empty = _git.diff_patch( source, name, self.index, target_path) if not is_file_empty: try: storage_class.upload(self, name, is_job_key) except Exception as e: print_tb(e) raise e except Exception as e: raise Exception( "E: Problem on the git_diff_patch_and_upload() function" ) from e
def is_contract_exists(): try: Ebb = cfg.Ebb except Exception as e: print_tb(e) log(f"==> is_contract_exists={Ebb.is_contract_exists()}", "bold")
def get_data_init(self, key, _id, is_job_key=False): try: cmd = ["gdrive", "info", "--bytes", key, "-c", env.GDRIVE_METADATA] _p, gdrive_output, *_ = popen_communicate(cmd) if _p.returncode != 0: raise Exception(gdrive_output) except Exception as e: raise e mime_type = gdrive.get_file_info(gdrive_output, _type="Mime") folder_name = gdrive.get_file_info(gdrive_output, _type="Name") log(f"==> mime_type=[magenta]{mime_type}") if is_job_key: # key for the sourceCode tar.gz file is obtained try: self.data_transfer_in_to_download, self.job_key_list, key = gdrive.size( key, mime_type, folder_name, gdrive_output, self.results_folder_prev, self.code_hashes, self.job_infos[0]["is_cached"], ) except Exception as e: print_tb(e) raise e return mime_type, folder_name
def eudat_submit(job: Job, is_pass=False, required_confs=1): log("==> Submitting source code through [blue]EUDAT[/blue]") Ebb = cfg.Ebb requester = Ebb.w3.toChecksumAddress(job.requester_addr) oc_client = "059ab6ba-4030-48bb-b81b-12115f531296" connect() try: job.check_account_status(requester) except Exception as e: print_tb(e) raise e login(oc_client, env.LOG_PATH.joinpath(".eudat_client.txt"), env.OC_CLIENT) if len(sys.argv) == 3: provider = str(sys.argv[1]) tar_hash = sys.argv[2] log(f"==> provided_hash={tar_hash}") else: provider = Ebb.w3.toChecksumAddress(job.provider_addr) job.folders_to_share = job.paths check_link_folders(job.data_paths, job.registered_data_files, is_pass=is_pass) return submit(provider, requester, job, required_confs=required_confs)
def add_all(repo=None): """Add all into git.""" try: if not repo: repo = git.Repo(".", search_parent_directories=True) log("all files in the entire working tree are updated in the Git repository ", end="") repo.git.add(A=True) log(ok()) try: #: git diff HEAD --name-only | wc -l changed_file_len = len( repo.index.diff("HEAD", ignore_blank_lines=True, ignore_space_at_eol=True)) except: # if it is the first commit HEAD might not exist changed_file_len = len( repo.git.diff("--cached", "--ignore-blank-lines", "--ignore-space-at-eol", "--name-only").split("\n")) if changed_file_len > 0: log("Record changes to the repository ", end="") repo.git.commit("-m", "update") log(ok()) except Exception as e: print_tb(e) raise e
def upload_driver(self): self.clean_before_upload() try: storage_class = self.get_cloud_storage_class(0) self.git_diff_patch_and_upload(self.results_folder, self.job_key, storage_class, is_job_key=True) except Exception as e: raise e for idx, name in enumerate(self.code_hashes_to_process[1:], 1): # starting from 1st index for data files source = self.results_data_folder / name try: if not self.storage_ids[idx] == StorageID.NONE: storage_class = self.get_cloud_storage_class(idx) self.git_diff_patch_and_upload(source, name, storage_class, is_job_key=False) else: pass except Exception as e: print_tb(e) raise e if not is_dir_empty(self.patch_folder_ipfs): # it will upload files after all the patchings are completed # in case any file is created via ipfs self._ipfs_add_folder(self.patch_folder_ipfs)
def submit_slurm_job(self, job_core_num, sbatch_file_path): """Slurm submits job. * Real mode -n is used. * For Emulator-mode -N use 'sbatch -c' * cmd: sudo su - $requester_id -c "cd $results_folder && firejail --noprofile \ sbatch -c$job_core_num $results_folder/${job_key}*${index}.sh --mail-type=ALL """ for _attempt in range(10): try: cmd = f'sbatch -n {job_core_num} "{sbatch_file_path}" --mail-type=ALL' with cd(self.results_folder): try: job_id = _run_as_sudo(env.SLURMUSER, cmd, shell=True) return job_id except Exception as e: if "Invalid account" in str(e): remove_user(env.SLURMUSER) add_user_to_slurm(env.SLURMUSER) job_id = _run_as_sudo(env.SLURMUSER, cmd, shell=True) time.sleep( 1) # wait 1 second for slurm idle core to be updated except Exception as e: print_tb(e) slurm.remove_user(self.requester_id) slurm.add_user_to_slurm(self.requester_id) raise Exception("E: sbatch could not submit the job")
def _dump_dict_to_file(filename, job_keys): try: log("==> meta_data.json file is updated in the parent folder") dump_dict_to_file(filename, job_keys) except Exception as e: print_tb(e) raise e
def main(): pidfile = env.DRIVER_DAEMON_LOCK d = DriverDaemon(pidfile=pidfile) # pidfile = daemon.pidfile.PIDLockFile(env.DRIVER_DAEMON_LOCK) # logfile = os.path.join(os.getcwd(), "sleepy.log") # pidfile = os.path.join(os.getcwd(), "sleepy.pid") # logging.basicConfig(filename=logfile, level=logging.DEBUG) # d = Daemon_base(pidfile, env.EBLOCPATH, cmd) if len(sys.argv) == 2: if sys.argv[1] in ["start", "s"]: try: is_driver_on() d.start() except QuietExit: pass except Exception as e: print_tb(str(e)) sys.exit(1) elif sys.argv[1] in ["terminate", "t"]: d.terminate() elif sys.argv[1] in ["reload", "r"]: d.restart() else: print("Unknown command") sys.exit(2) sys.exit(0) else: print("usage: %s [s]tart|[t]erminate|[r]eload" % sys.argv[0]) sys.exit(2)
def process_payment_tx(self): try: tx_hash = eblocbroker_function_call( lambda: Ebb.process_payment( self.job_key, self.index, self.job_id, self.elapsed_time, self.result_ipfs_hash, self.storage_ids, self.end_time_stamp, self.data_transfer_in, self.data_transfer_out, self.job_info["core"], self.job_info["run_time"], self.received_block_number, ), max_retries=10, ) except Exception as e: print_tb(e) sys.exit(1) log(f"==> process_payment {self.job_key} {self.index}") return tx_hash
def sbatch_call(self): link = Link(self.results_data_folder, self.results_data_link) try: if len(self.registered_data_hashes) > 0: # in case there there mounted folder first umount them in order # to give folder permission link.umount(self.registered_data_hashes) # folder permissions should be applied before linking the # folders in case there is a read-only folder. file permission for # the requester's foders should be reset give_rwe_access(self.requester_id, self.results_folder_prev) give_rwe_access(env.WHOAMI, self.results_folder_prev) # give_rwe_access(self.requester_id, self.requester_home) # give_rwe_access(env.WHOAMI, self.requester_home) if calculate_size(self.results_data_folder, _type="bytes") > 0: link.link_folders() if len(self.registered_data_hashes) > 0: link.registered_data(self.registered_data_hashes) self._sbatch_call() except Exception as e: print_tb(f"E: Failed to call _sbatch_call() function. {e}") raise e
def get_data_key_ids(results_folder_prev) -> bool: filename = f"{results_folder_prev}/meta_data.json" log(f"==> meta_data_path={filename}") try: meta_data = read_json(filename) except Exception as e: print_tb(e) return meta_data
def initialize(self): with suppress(Exception): eudat.login(env.OC_USER, env.LOG_PATH.joinpath(".eudat_client.txt"), env.OC_CLIENT) try: self.get_shared_tokens() except Exception as e: print_tb(e) raise e
def submit_gdrive(job: Job, is_pass=False, required_confs=1): log("==> Submitting source code through [blue]GDRIVE[/blue]") pre_check() Ebb = cfg.Ebb job.folders_to_share = job.paths check_link_folders(job.data_paths, job.registered_data_files, is_pass=is_pass) _git.generate_git_repo(job.folders_to_share) job.clean_before_submit() requester = Ebb.w3.toChecksumAddress(job.requester_addr) provider = Ebb.w3.toChecksumAddress(job.provider_addr) job = gdrive.submit(provider, requester, job) for folder_to_share in job.folders_to_share: if isinstance(folder_to_share, bytes): code_hash = folder_to_share job.code_hashes.append(code_hash) job.code_hashes_str.append(code_hash.decode("utf-8")) else: tar_hash = job.foldername_tar_hash[folder_to_share] #: required to send string as bytes == str_data.encode('utf-8') code_hash = Ebb.w3.toBytes(text=tar_hash) job.code_hashes.append(code_hash) job.code_hashes_str.append(code_hash.decode("utf-8")) tar_hash = job.foldername_tar_hash[job.folders_to_share[0]] key = job.keys[tar_hash] job.price, *_ = job.cost(provider, requester) try: tx_hash = Ebb.submit_job(provider, key, job, requester=requester, required_confs=required_confs) tx_receipt = get_tx_status(tx_hash) if tx_receipt["status"] == 1: processed_logs = Ebb._eBlocBroker.events.LogJob().processReceipt( tx_receipt, errors=DISCARD) log(vars(processed_logs[0].args)) try: log(f"{ok()} [bold]job_index={processed_logs[0].args['index']}" ) except IndexError: log(f"E: Tx({tx_hash}) is reverted") except QuietExit: pass except Exception as e: print_tb(e) log() for k, v in job.tar_hashes.items(): log(f"{k} [blue]=>[/blue] {v}") return tx_hash
def _ipfs_add_folder(self, folder_path): try: self.result_ipfs_hash = cfg.ipfs.add(folder_path) log(f"==> result_ipfs_hash={self.result_ipfs_hash}") cfg.ipfs.pin(self.result_ipfs_hash) data_transfer_out = cfg.ipfs.get_cumulative_size( self.result_ipfs_hash) except Exception as e: print_tb(e) raise e data_transfer_out = byte_to_mb(data_transfer_out) self.data_transfer_out += data_transfer_out
def main(): provider = env.PROVIDER_ID pre_check_data(provider) Ebb = cfg.Ebb data_hash = b"f13d75bc60898f0823566347e380a34d" try: if is_data_registered(provider, data_hash): tx = Ebb.remove_registered_data(data_hash) get_tx_status(Ebb.tx_id(tx)) else: log(f"## data({data_hash}) is alread deleted or not registered") except Exception as e: print_tb(e)
def share_single_folder(folder_name, f_id) -> bool: try: # folder_names = os.listdir(env.OWNCLOUD_PATH) # fID = '*****@*****.**' if not config.oc.is_shared(folder_name): config.oc.share_file_with_user(folder_name, f_id, remote_user=True, perms=31) log(f"sharing with [yellow]{f_id}[/yellow] {ok()}", "bold") return True log("## Requester folder is already shared") return True except Exception as e: print_tb(e) return False
def pre_data_check(self, key): if self.data_transfer_in_to_download > self.data_transfer_in_requested: # TODO: full refund raise Exception( "Requested size to download the source_code and data files is greater than the given amount" ) try: cmd = ["gdrive", "info", "--bytes", key, "-c", env.GDRIVE_METADATA] return subprocess_call(cmd, 1) except Exception as e: # TODO: gdrive list --query "sharedWithMe" print_tb(e) raise e
def main(): Ebb = cfg.Ebb is_write_to_file = False if len(sys.argv) == 2: if sys.argv[1] in ("1", "True", "true"): is_write_to_file = True try: output = Ebb.get_block_number() if is_write_to_file: env.config["block_continue"] = output else: log(f"block_number={output}", "bold") except Exception as e: print_tb(e)
def run(self) -> bool: self.start_time = time.time() if cfg.IS_THREADING_ENABLED: self.thread_log_setup() log(f"## Keep track from: tail -f {self.drivers_log_path}") try: log(f" * log_path={self.drivers_log_path}") self._run() # self.thread_log_setup() return True except Exception as e: print_tb(f"{self.job_key}_{self.index} {e}") sys.exit(1) finally: time.sleep(0.25)
def complete_refund(self) -> str: """Complete refund back to the requester.""" try: tx_hash = self.Ebb.refund( self.logged_job.args["provider"], env.PROVIDER_ID, self.job_key, self.index, self.job_id, self.cores, self.run_time, ) log(f"==> refund tx_hash={tx_hash}") return tx_hash except Exception as e: print_tb(e) raise e
def search_token(self, f_id, share_list, folder_name, is_silent=False) -> bool: """Search for the share_token from the shared folder.""" share_key = f"{folder_name}_{self.requester_id[:16]}" if not is_silent: log(f"## searching share tokens for the related source_code_folder={folder_name}" ) for idx in range(len(share_list) - 1, -1, -1): # starts iterating from last item to the first one input_folder_name = share_list[idx]["name"] input_folder_name = input_folder_name[ 1:] # removes '/' at the beginning share_id = share_list[idx]["id"] # input_owner = share_list[i]['owner'] input_user = f"{share_list[idx]['user']}@b2drop.eudat.eu" if input_folder_name == share_key and input_user == f_id: self.share_token = str(share_list[idx]["share_token"]) self.share_id[share_key] = { "share_id": int(share_id), "share_token": self.share_token, } if Ebb.mongo_broker.add_item_share_id(share_key, share_id, self.share_token): # adding into mongoDB for future uses log(f"#> Added into mongoDB {ok()}") else: logging.error( "E: Something is wrong, not added into mongoDB") log(f"==> name={folder_name} | share_id={share_id} | share_token={self.share_token} {ok()}" ) try: config.oc.accept_remote_share(int(share_id)) log(f"## share_id={share_id} is accepted") except Exception as e: print_tb(e) self.accept_flag += 1 return True return False
def _withdraw(): Ebb = cfg.Ebb if len(sys.argv) == 2: account = str(sys.argv[1]) else: log("## provide an ethereum account as an argument") sys.exit(1) try: balance = Ebb.get_balance(account) if balance > 0: log(f"account_balance={balance}", "bold") get_tx_status(Ebb.withdraw(account)) else: log("warning: account balance is empty nothing to do") except QuietExit: pass except Exception as e: print_tb(e)
def upload(self, key, is_job_key): """Upload generated result into gdrive. :param key: key of the shared gdrive file :returns: True if upload is successful """ try: if not is_job_key: meta_data = gdrive.get_data_key_ids(self.results_folder_prev) key = meta_data[key] cmd = [ env.GDRIVE, "info", "--bytes", key, "-c", env.GDRIVE_METADATA ] gdrive_info = subprocess_call(cmd, 5, sleep_time=30) except Exception as e: raise Exception( f"{WHERE(1)} E: {key} does not have a match. meta_data={meta_data}. {e}" ) from e mime_type = gdrive.get_file_info(gdrive_info, "Mime") log(f"mime_type={mime_type}") self.data_transfer_out += calculate_size(self.patch_file) log(f"data_transfer_out={self.data_transfer_out} MB =>" f" rounded={int(self.data_transfer_out)} MB") if "folder" in mime_type: cmd = [ env.GDRIVE, "upload", "--parent", key, self.patch_file, "-c", env.GDRIVE_METADATA ] elif "gzip" in mime_type or "/zip" in mime_type: cmd = [ env.GDRIVE, "update", key, self.patch_file, "-c", env.GDRIVE_METADATA ] else: raise Exception("Files could not be uploaded") try: log(subprocess_call(cmd, 5)) except Exception as e: print_tb(e) raise Exception("E: gdrive could not upload the file") from e
def remove_source_code(self): """Client's initial downloaded files are removed.""" timestamp_file = f"{self.results_folder_prev}/timestamp.txt" try: cmd = [ "find", self.results_folder, "-type", "f", "!", "-newer", timestamp_file ] files_to_remove = run(cmd) if files_to_remove: log(f"## Files to be removed: \n{files_to_remove}\n") except Exception as e: print_tb(e) sys.exit() run([ "find", self.results_folder, "-type", "f", "!", "-newer", timestamp_file, "-delete" ])
def scontrol_update(self, job_core_num, sbatch_file_path, time_limit): """Prevent scontrol update locked exception. scontrol generates: Job update not available right now, the DB index is being set, try again in a bit for job 5. """ try: _slurm_job_id = self.submit_slurm_job(job_core_num, sbatch_file_path) slurm_job_id = _slurm_job_id.split()[3] cmd = [ "scontrol", "update", f"jobid={slurm_job_id}", f"TimeLimit={time_limit}" ] subprocess_call(cmd, attempt=10, sleep_time=10) return slurm_job_id except Exception as e: print_tb(e) raise e
def submit(provider, requester, job, required_confs=1): try: tx_hash = _submit(provider, requester, job, required_confs) if required_confs >= 1: tx_receipt = get_tx_status(tx_hash) if tx_receipt["status"] == 1: processed_logs = Ebb._eBlocBroker.events.LogJob().processReceipt(tx_receipt, errors=DISCARD) log(vars(processed_logs[0].args)) try: log(f"{ok()} [bold]job_index={processed_logs[0].args['index']}") except IndexError: log(f"E: Tx({tx_hash}) is reverted") else: log(f"tx_hash={tx_hash}", "bold") except QuietExit: pass except Exception as e: print_tb(e) return tx_hash
def run(self) -> bool: self.start_time = time.time() if cfg.IS_THREADING_ENABLED: self.thread_log_setup() log(f"{br(get_date())} job's source code has been sent through Google Drive", "bold cyan") # self.get_data_init(key=self.job_key, _id=0, is_job_key=True) try: if os.path.isdir(self.results_folder): # attempt to download the source code target = self.get_data(key=self.job_key, _id=0, is_job_key=True) if not os.path.isdir(f"{target}/.git"): log(f"warning: .git folder does not exist within {target}") _git.generate_git_repo(target) except Exception as e: print_tb(e) return False if not self.check_run_sh(): self.complete_refund() for idx, source_code_hash in enumerate(self.code_hashes): if self.cloudStorageID[idx] == StorageID.NONE: if isinstance(source_code_hash, bytes): self.registered_data_hashes.append(source_code_hash.decode("utf-8")) else: self.registered_data_hashes.append(source_code_hash) for idx, (_, value) in enumerate(self.job_key_list.items()): try: target = self.get_data(value, idx + 1) if not os.path.isdir(f"{target}/.git"): log(f"warning: .git folder does not exist within {target}") _git.generate_git_repo(target) except: return False return self.sbatch_call()
def attemp_get_job_info(self): is_print = True sleep_time = 30 for attempt in range(10): # log(self.job_info) if self.job_info["stateCode"] == state.code["RUNNING"]: # it will come here eventually, when setJob() is deployed. Wait # until does values updated on the blockchain log("## job has been started") return if self.job_info["stateCode"] == state.code["COMPLETED"]: # detects an error on the slurm side log("warning: job is already completed and its money is received" ) self.get_job_info() raise QuietExit try: self.job_info = Ebb.get_job_info(env.PROVIDER_ID, self.job_key, self.index, self.job_id, self.received_block_number, is_print) is_print = False except Exception as e: print_tb(e) # sys.exit(1) # sleep here so this loop is not keeping CPU busy due to # start_code tx may deploy late into the blockchain. log( f"==> {br(attempt)} start_code tx of the job is not obtained yet, " f"waiting for {sleep_time} seconds to pass...", end="", ) sleep(sleep_time) log(ok()) log("E: failed all the attempts, abort") sys.exit(1)