def deposit_storage(eth_address, is_provider=False): """Deposit storage balance. :param str eth_address: Ethereum address of the provider :param bool is_provider: Checks it the caller provider """ from_block = Ebb.get_deployed_block_number() if is_provider: event_filter = Ebb._eBlocBroker.events.LogJob.createFilter( fromBlock=int(from_block), argument_filters={"provider": eth_address}, toBlock="latest", ) else: # should be owner of the job event_filter = Ebb._eBlocBroker.events.LogJob.createFilter( fromBlock=int(from_block), argument_filters={"owner": eth_address}, toBlock="latest", ) for job in enumerate(event_filter.get_all_entries()): job_info = job[1].args flag_check = [] for idx, code_hash in enumerate(job_info["sourceCodeHash"]): main_cloud_storage_id = job_info["cloudStorageID"][idx] if main_cloud_storage_id in (StorageID.IPFS, StorageID.IPFS_GPG): _hash = bytes32_to_ipfs(code_hash) _type = "ipfs_hash" else: _hash = cfg.w3.toText(code_hash) _type = "md5sum" log(br(f"{idx}, {_type}"), "bold cyan", end="") if len(code_hash) <= 32: log(f" {_hash} bytes={code_hash}", "bold") else: log(f" {_hash}\n\t{code_hash}", "bold") provider = Ebb.w3.toChecksumAddress(job_info["provider"]) if is_provider and eth_address.lower() == provider.lower(): data_owner = Ebb.w3.toChecksumAddress(job_info["owner"]) deposit, output = Ebb.get_storage_info(provider, data_owner, code_hash) flag_check.append(output[3]) log(f"deposit={deposit}, {output}", "bold") if deposit > 0 and not any( flag_check): # if not any(i for i in flag_check): is_verified_list = [True, True] tx = Ebb._data_received( job_info["jobKey"], job_info["index"], job_info["sourceCodeHash"], job_info["cacheType"], is_verified_list, ) get_tx_status(Ebb.tx_id(tx)) else: log("warning: already all data files are are verifid")
def set_code_hashes_to_process(self): for idx, source_code_hash in enumerate(self.code_hashes): if self.storage_ids[idx] in [StorageID.IPFS, StorageID.IPFS_GPG]: ipfs_hash = bytes32_to_ipfs(source_code_hash) self.code_hashes_to_process.append(ipfs_hash) else: self.code_hashes_to_process.append( cfg.w3.toText(source_code_hash))
def set_storage_cost(self): """Calculate the cache cost.""" self.storage_cost = 0 self.cache_cost = 0 self.data_transfer_in_sum = 0 for idx, source_code_hash in enumerate(self.job.code_hashes): if self.is_brownie: ds = self.create_data_storage(source_code_hash) else: ds = self.create_data_storage(self.job.code_hashes_str[idx]) if ds.received_block + ds.storage_duration < self.w3.eth.block_number: # storage time is completed ds.received_storage_deposit = 0 try: _source_code_hash = source_code_hash.decode("utf-8") except: _source_code_hash = bytes32_to_ipfs(source_code_hash) log(f"==> is_private{br(_source_code_hash, 'blue')}={ds.is_private}" ) # print(received_block + storage_duration >= self.w3.eth.block_number) # if ds.received_storage_deposit > 0 or if (ds.received_storage_deposit > 0 and ds.received_block + ds.storage_duration >= self.w3.eth.block_number) or ( ds.received_block + ds.storage_duration >= self.w3.eth.block_number and not ds.is_private and ds.is_verified_used): log(f"==> For {bytes32_to_ipfs(source_code_hash)} cost of storage is not paid" ) else: if self.job.data_prices_set_block_numbers[idx] > 0: # if true, registered data's price should be considered for storage output = self.ebb.getRegisteredDataPrice( self.job.provider, source_code_hash, self.job.data_prices_set_block_numbers[idx], ) data_price = output[0] self.storage_cost += data_price break # if not ds.received_storage_deposit and (received_block + storage_duration < w3.eth.block_number): if not ds.received_storage_deposit: self.data_transfer_in_sum += self.job.data_transfer_ins[ idx] if self.job.storage_hours[idx] > 0: self.storage_cost += (self.price_storage * self.job.data_transfer_ins[idx] * self.job.storage_hours[idx]) else: self.cache_cost += self.price_cache * self.job.data_transfer_ins[ idx] self.data_transfer_in_cost = self.price_data_transfer * self.data_transfer_in_sum self.data_transfer_out_cost = self.price_data_transfer * self.job.data_transfer_out self.data_transfer_cost = self.data_transfer_in_cost + self.data_transfer_out_cost
def get_job_info_print(self, provider, job_key, index, received_block_number): Ebb = cfg.Ebb elapsed_time = 0 result_ipfs_hash = "" if self.job_info["result_ipfs_hash"] != empty_bytes32 and self.job_info[ "result_ipfs_hash"] != "": result_ipfs_hash = bytes32_to_ipfs(self.job_info["result_ipfs_hash"]) if self.job_info["completion_time"]: elapsed_time = int(self.job_info["completion_time"]) - int( self.job_info["start_time"]) if isinstance(self.job_info, dict): log(f"==> state_code={state.inv_code[self.job_info['stateCode']]}({self.job_info['stateCode']})" ) log(self.job_info) if result_ipfs_hash: log(f"==> result_ipfs_hash={result_ipfs_hash}") Ebb.get_job_code_hashes(provider, job_key, index, received_block_number) if self.job_info["code_hashes"]: log("code_hashes:", "bold blue") for idx, code_hash in enumerate(self.job_info["code_hashes"]): main_cloud_storage_id = self.job_info["cloudStorageID"][idx] if main_cloud_storage_id in (StorageID.IPFS, StorageID.IPFS_GPG): _hash = bytes32_to_ipfs(code_hash) _type = "ipfs_hash" else: _hash = cfg.w3.toText(code_hash) _type = "md5sum" log(br(f"{idx}, {_type}"), "bold cyan", end="") if len(code_hash) <= 32: log(f" {_hash} bytes={code_hash}", "bold") else: log(f" {_hash}\n\t{code_hash}", "bold") log() self.analyze_data(job_key, provider) else: print(self.job_info) assert elapsed_time >= 0, "elapsed_time is negative"
def __init__(self, **kwargs) -> None: self.Ebb = cfg.Ebb self.thread_name = uuid.uuid4( ).hex # https://stackoverflow.com/a/44992275/2402577 self.requester_id = kwargs.pop("requester_id") self.job_infos = kwargs.pop("job_infos") self.logged_job = kwargs.pop("logged_job") self.is_cached = kwargs.pop("is_cached") self.job_key = self.logged_job.args["jobKey"] self.index = self.logged_job.args["index"] self.cores = self.logged_job.args["core"] self.run_time = self.logged_job.args["runTime"] self.job_id = 0 self.cache_type = self.logged_job.args["cacheType"] self.data_transfer_in_requested = self.job_infos[0]["data_transfer_in"] self.data_transfer_in_to_download_mb = 0 # total size in MB to download self.code_hashes: List[bytes] = self.logged_job.args["sourceCodeHash"] self.code_hashes_str: List[str] = [ bytes32_to_ipfs(_hash) for _hash in self.code_hashes ] self.registered_data_hashes = [] # noqa self.job_key_list: List[str] = [] self.md5sum_dict: Dict[str, str] = {} self.folder_path_to_download: Dict[str, Path] = {} self.folder_type_dict: Dict[str, str] = {} self.PROGRAM_PATH = Path(env.PROGRAM_PATH) self.cloudStorageID = self.logged_job.args["cloudStorageID"] self.requester_home = self.PROGRAM_PATH / self.requester_id self.results_folder_prev = self.requester_home / f"{self.job_key}_{self.index}" self.results_folder = self.results_folder_prev / "JOB_TO_RUN" self.run_path = self.results_folder / "run.sh" self.results_data_folder = self.results_folder_prev / "data" self.results_data_link = self.results_folder_prev / "data_link" self.private_dir = self.requester_home / "cache" self.public_dir = self.PROGRAM_PATH / "cache" self.patch_folder = self.results_folder_prev / "patch" self.drivers_log_path = f"{env.LOG_PATH}/drivers_output/{self.job_key}_{self.index}.log" self.start_time = None self.mc = None self.data_transfer_in_to_download: int = 0 _log.thread_log_files[self.thread_name] = self.drivers_log_path try: mkdir(self.private_dir) except PermissionError: give_rwe_access(env.SLURMUSER, self.requester_home) mkdir(self.private_dir) mkdir(self.public_dir) mkdir(self.results_folder) mkdir(self.results_data_folder) mkdir(self.results_data_link) mkdir(self.patch_folder)
def analyze_data(self, key, provider=None): """Obtain information related to source-code data.""" current_block_number = cfg.Ebb.get_block_number() self.received_block = [] self.storage_duration = [] self.job_info["is_cached"] = {} for idx, code_hash in enumerate(self.job_info["code_hashes"]): if self.job_info["cloudStorageID"][idx] in (StorageID.IPFS, StorageID.IPFS_GPG): source_code_hash = source_code_hash_str = bytes32_to_ipfs( code_hash) if idx == 0 and key != source_code_hash: log(f"E: IPFS hash does not match with the given source_code_hash.\n\t{key} != {source_code_hash}" ) continue else: source_code_hash = code_hash source_code_hash_str = cfg.w3.toText(code_hash) received_storage_deposit = cfg.Ebb.get_received_storage_deposit( provider, self.job_info["job_owner"], source_code_hash) job_storage_duration = cfg.Ebb.get_job_storage_duration( provider, source_code_hash) ds = DataStorage(job_storage_duration) ds.received_storage_deposit = received_storage_deposit self.received_block.append(ds.received_block) self.storage_duration.append(ds.storage_duration) self.job_info["is_cached"][ source_code_hash_str] = False # FIXME double check # if remaining time to cache is 0, then caching is requested for the # related source_code_hash if ds.received_block + ds.storage_duration >= current_block_number: if ds.received_block < current_block_number: self.job_info["is_cached"][source_code_hash_str] = True elif ds.received_block == current_block_number: if source_code_hash in self.job_info["is_cached"]: self.job_info["is_cached"][source_code_hash_str] = True else: # for the first job should be False since it is # requested for cache for the first time self.job_info["is_cached"][source_code_hash_str] = False log(f" * source_code_hash{br(idx)}=[green]{source_code_hash_str}") log(f"==> received_block={ds.received_block}") log(f"==> storage_duration{br(self.job_info['received_block_number'])}={ds.storage_duration}" ) log(f"==> cloud_storage_id{br(idx)}={StorageID(self.job_info['cloudStorageID'][idx]).name}" ) log(f"==> cached_type={CacheType(self.job_info['cacheType'][idx]).name}" ) log(f"==> is_cached={self.job_info['is_cached'][source_code_hash_str]}" )
def run(self) -> bool: self.start_time = time.time() if cfg.IS_THREADING_ENABLED: self.thread_log_setup() run_ipfs_daemon() log(f"{br(get_date())} Job's source code has been sent through ", "bold cyan", end="") if self.cloudStorageID[0] == StorageID.IPFS: log("[bold green]IPFS") else: log("[bold green]IPFS_GPG") if not is_ipfs_on(): return False log(f"==> is_hash_locally_cached={cfg.ipfs.is_hash_locally_cached(self.job_key)}" ) if not os.path.isdir(self.results_folder): os.makedirs(self.results_folder) _remove(f"{self.results_folder}/{self.job_key}") try: self.check_ipfs(self.job_key) except: return False self.registered_data_hashes = [] for idx, source_code_hash in enumerate(self.code_hashes): if self.cloudStorageID[idx] == StorageID.NONE: self.registered_data_hashes.append(source_code_hash) # GOTCHA else: ipfs_hash = bytes32_to_ipfs(source_code_hash) if ipfs_hash not in self.ipfs_hashes: try: # job_key as data hash already may added to the list self.check_ipfs(ipfs_hash) except: return False initial_folder_size = calculate_size(self.results_folder) for idx, ipfs_hash in enumerate(self.ipfs_hashes): # here scripts knows that provided IPFS hashes exists online is_hashed = False log(f"## attempting to get IPFS file: {ipfs_hash} ... ", end="") if cfg.ipfs.is_hash_locally_cached(ipfs_hash): is_hashed = True log(ok("already cached")) else: log() if idx == 0: target = self.results_folder else: # "_" added before the filename in case $ ipfs get <ipfs_hash> target = f"{self.results_data_folder}/_{ipfs_hash}" mkdir(target) is_storage_paid = False # TODO: should be set before by user input cfg.ipfs.get(ipfs_hash, target, is_storage_paid) if idx > 0: # https://stackoverflow.com/a/31814223/2402577 dst_filename = os.path.join(self.results_data_folder, os.path.basename(ipfs_hash)) if os.path.exists(dst_filename): _remove(dst_filename) shutil.move(target, dst_filename) target = dst_filename if self.cloudStorageID[idx] == StorageID.IPFS_GPG: cfg.ipfs.decrypt_using_gpg(f"{target}/{ipfs_hash}", target) try: _git.initialize_check(target) except Exception as e: raise e if not is_hashed: folder_size = calculate_size(self.results_folder) self.data_transfer_in_to_download_mb += folder_size - initial_folder_size initial_folder_size = folder_size if idx == 0 and not self.check_run_sh(): self.complete_refund() return False log(f"==> data_transfer_in={self.data_transfer_in_to_download_mb} MB | " f"rounded={int(self.data_transfer_in_to_download_mb)} MB") return self.sbatch_call()