def deposit_storage(eth_address, is_provider=False):
"""Deposit storage balance.
:param str eth_address: Ethereum address of the provider
:param bool is_provider: Checks it the caller provider
"""
from_block = Ebb.get_deployed_block_number()
if is_provider:
event_filter = Ebb._eBlocBroker.events.LogJob.createFilter(
fromBlock=int(from_block),
argument_filters={"provider": eth_address},
toBlock="latest",
)
else: # should be owner of the job
event_filter = Ebb._eBlocBroker.events.LogJob.createFilter(
fromBlock=int(from_block),
argument_filters={"owner": eth_address},
toBlock="latest",
)
for job in enumerate(event_filter.get_all_entries()):
job_info = job[1].args
flag_check = []
for idx, code_hash in enumerate(job_info["sourceCodeHash"]):
main_cloud_storage_id = job_info["cloudStorageID"][idx]
if main_cloud_storage_id in (StorageID.IPFS, StorageID.IPFS_GPG):
_hash = bytes32_to_ipfs(code_hash)
_type = "ipfs_hash"
else:
_hash = cfg.w3.toText(code_hash)
_type = "md5sum"
log(br(f"{idx}, {_type}"), "bold cyan", end="")
if len(code_hash) <= 32:
log(f" {_hash} bytes={code_hash}", "bold")
else:
log(f" {_hash}\n\t{code_hash}", "bold")
provider = Ebb.w3.toChecksumAddress(job_info["provider"])
if is_provider and eth_address.lower() == provider.lower():
data_owner = Ebb.w3.toChecksumAddress(job_info["owner"])
deposit, output = Ebb.get_storage_info(provider, data_owner,
code_hash)
flag_check.append(output[3])
log(f"deposit={deposit}, {output}", "bold")
if deposit > 0 and not any(
flag_check): # if not any(i for i in flag_check):
is_verified_list = [True, True]
tx = Ebb._data_received(
job_info["jobKey"],
job_info["index"],
job_info["sourceCodeHash"],
job_info["cacheType"],
is_verified_list,
)
get_tx_status(Ebb.tx_id(tx))
else:
log("warning: already all data files are are verifid")
def set_code_hashes_to_process(self):
for idx, source_code_hash in enumerate(self.code_hashes):
if self.storage_ids[idx] in [StorageID.IPFS, StorageID.IPFS_GPG]:
ipfs_hash = bytes32_to_ipfs(source_code_hash)
self.code_hashes_to_process.append(ipfs_hash)
else:
self.code_hashes_to_process.append(
cfg.w3.toText(source_code_hash))
def set_storage_cost(self):
"""Calculate the cache cost."""
self.storage_cost = 0
self.cache_cost = 0
self.data_transfer_in_sum = 0
for idx, source_code_hash in enumerate(self.job.code_hashes):
if self.is_brownie:
ds = self.create_data_storage(source_code_hash)
else:
ds = self.create_data_storage(self.job.code_hashes_str[idx])
if ds.received_block + ds.storage_duration < self.w3.eth.block_number:
# storage time is completed
ds.received_storage_deposit = 0
try:
_source_code_hash = source_code_hash.decode("utf-8")
except:
_source_code_hash = bytes32_to_ipfs(source_code_hash)
log(f"==> is_private{br(_source_code_hash, 'blue')}={ds.is_private}"
)
# print(received_block + storage_duration >= self.w3.eth.block_number)
# if ds.received_storage_deposit > 0 or
if (ds.received_storage_deposit > 0 and ds.received_block +
ds.storage_duration >= self.w3.eth.block_number) or (
ds.received_block + ds.storage_duration >=
self.w3.eth.block_number and not ds.is_private
and ds.is_verified_used):
log(f"==> For {bytes32_to_ipfs(source_code_hash)} cost of storage is not paid"
)
else:
if self.job.data_prices_set_block_numbers[idx] > 0:
# if true, registered data's price should be considered for storage
output = self.ebb.getRegisteredDataPrice(
self.job.provider,
source_code_hash,
self.job.data_prices_set_block_numbers[idx],
)
data_price = output[0]
self.storage_cost += data_price
break
# if not ds.received_storage_deposit and (received_block + storage_duration < w3.eth.block_number):
if not ds.received_storage_deposit:
self.data_transfer_in_sum += self.job.data_transfer_ins[
idx]
if self.job.storage_hours[idx] > 0:
self.storage_cost += (self.price_storage *
self.job.data_transfer_ins[idx] *
self.job.storage_hours[idx])
else:
self.cache_cost += self.price_cache * self.job.data_transfer_ins[
idx]
self.data_transfer_in_cost = self.price_data_transfer * self.data_transfer_in_sum
self.data_transfer_out_cost = self.price_data_transfer * self.job.data_transfer_out
self.data_transfer_cost = self.data_transfer_in_cost + self.data_transfer_out_cost
def get_job_info_print(self, provider, job_key, index, received_block_number):
Ebb = cfg.Ebb
elapsed_time = 0
result_ipfs_hash = ""
if self.job_info["result_ipfs_hash"] != empty_bytes32 and self.job_info[
"result_ipfs_hash"] != "":
result_ipfs_hash = bytes32_to_ipfs(self.job_info["result_ipfs_hash"])
if self.job_info["completion_time"]:
elapsed_time = int(self.job_info["completion_time"]) - int(
self.job_info["start_time"])
if isinstance(self.job_info, dict):
log(f"==> state_code={state.inv_code[self.job_info['stateCode']]}({self.job_info['stateCode']})"
)
log(self.job_info)
if result_ipfs_hash:
log(f"==> result_ipfs_hash={result_ipfs_hash}")
Ebb.get_job_code_hashes(provider, job_key, index,
received_block_number)
if self.job_info["code_hashes"]:
log("code_hashes:", "bold blue")
for idx, code_hash in enumerate(self.job_info["code_hashes"]):
main_cloud_storage_id = self.job_info["cloudStorageID"][idx]
if main_cloud_storage_id in (StorageID.IPFS,
StorageID.IPFS_GPG):
_hash = bytes32_to_ipfs(code_hash)
_type = "ipfs_hash"
else:
_hash = cfg.w3.toText(code_hash)
_type = "md5sum"
log(br(f"{idx}, {_type}"), "bold cyan", end="")
if len(code_hash) <= 32:
log(f" {_hash} bytes={code_hash}", "bold")
else:
log(f" {_hash}\n\t{code_hash}", "bold")
log()
self.analyze_data(job_key, provider)
else:
print(self.job_info)
assert elapsed_time >= 0, "elapsed_time is negative"
def __init__(self, **kwargs) -> None:
self.Ebb = cfg.Ebb
self.thread_name = uuid.uuid4(
).hex # https://stackoverflow.com/a/44992275/2402577
self.requester_id = kwargs.pop("requester_id")
self.job_infos = kwargs.pop("job_infos")
self.logged_job = kwargs.pop("logged_job")
self.is_cached = kwargs.pop("is_cached")
self.job_key = self.logged_job.args["jobKey"]
self.index = self.logged_job.args["index"]
self.cores = self.logged_job.args["core"]
self.run_time = self.logged_job.args["runTime"]
self.job_id = 0
self.cache_type = self.logged_job.args["cacheType"]
self.data_transfer_in_requested = self.job_infos[0]["data_transfer_in"]
self.data_transfer_in_to_download_mb = 0 # total size in MB to download
self.code_hashes: List[bytes] = self.logged_job.args["sourceCodeHash"]
self.code_hashes_str: List[str] = [
bytes32_to_ipfs(_hash) for _hash in self.code_hashes
]
self.registered_data_hashes = [] # noqa
self.job_key_list: List[str] = []
self.md5sum_dict: Dict[str, str] = {}
self.folder_path_to_download: Dict[str, Path] = {}
self.folder_type_dict: Dict[str, str] = {}
self.PROGRAM_PATH = Path(env.PROGRAM_PATH)
self.cloudStorageID = self.logged_job.args["cloudStorageID"]
self.requester_home = self.PROGRAM_PATH / self.requester_id
self.results_folder_prev = self.requester_home / f"{self.job_key}_{self.index}"
self.results_folder = self.results_folder_prev / "JOB_TO_RUN"
self.run_path = self.results_folder / "run.sh"
self.results_data_folder = self.results_folder_prev / "data"
self.results_data_link = self.results_folder_prev / "data_link"
self.private_dir = self.requester_home / "cache"
self.public_dir = self.PROGRAM_PATH / "cache"
self.patch_folder = self.results_folder_prev / "patch"
self.drivers_log_path = f"{env.LOG_PATH}/drivers_output/{self.job_key}_{self.index}.log"
self.start_time = None
self.mc = None
self.data_transfer_in_to_download: int = 0
_log.thread_log_files[self.thread_name] = self.drivers_log_path
try:
mkdir(self.private_dir)
except PermissionError:
give_rwe_access(env.SLURMUSER, self.requester_home)
mkdir(self.private_dir)
mkdir(self.public_dir)
mkdir(self.results_folder)
mkdir(self.results_data_folder)
mkdir(self.results_data_link)
mkdir(self.patch_folder)
def analyze_data(self, key, provider=None):
"""Obtain information related to source-code data."""
current_block_number = cfg.Ebb.get_block_number()
self.received_block = []
self.storage_duration = []
self.job_info["is_cached"] = {}
for idx, code_hash in enumerate(self.job_info["code_hashes"]):
if self.job_info["cloudStorageID"][idx] in (StorageID.IPFS,
StorageID.IPFS_GPG):
source_code_hash = source_code_hash_str = bytes32_to_ipfs(
code_hash)
if idx == 0 and key != source_code_hash:
log(f"E: IPFS hash does not match with the given source_code_hash.\n\t{key} != {source_code_hash}"
)
continue
else:
source_code_hash = code_hash
source_code_hash_str = cfg.w3.toText(code_hash)
received_storage_deposit = cfg.Ebb.get_received_storage_deposit(
provider, self.job_info["job_owner"], source_code_hash)
job_storage_duration = cfg.Ebb.get_job_storage_duration(
provider, source_code_hash)
ds = DataStorage(job_storage_duration)
ds.received_storage_deposit = received_storage_deposit
self.received_block.append(ds.received_block)
self.storage_duration.append(ds.storage_duration)
self.job_info["is_cached"][
source_code_hash_str] = False # FIXME double check
# if remaining time to cache is 0, then caching is requested for the
# related source_code_hash
if ds.received_block + ds.storage_duration >= current_block_number:
if ds.received_block < current_block_number:
self.job_info["is_cached"][source_code_hash_str] = True
elif ds.received_block == current_block_number:
if source_code_hash in self.job_info["is_cached"]:
self.job_info["is_cached"][source_code_hash_str] = True
else:
# for the first job should be False since it is
# requested for cache for the first time
self.job_info["is_cached"][source_code_hash_str] = False
log(f" * source_code_hash{br(idx)}=[green]{source_code_hash_str}")
log(f"==> received_block={ds.received_block}")
log(f"==> storage_duration{br(self.job_info['received_block_number'])}={ds.storage_duration}"
)
log(f"==> cloud_storage_id{br(idx)}={StorageID(self.job_info['cloudStorageID'][idx]).name}"
)
log(f"==> cached_type={CacheType(self.job_info['cacheType'][idx]).name}"
)
log(f"==> is_cached={self.job_info['is_cached'][source_code_hash_str]}"
)
def run(self) -> bool:
self.start_time = time.time()
if cfg.IS_THREADING_ENABLED:
self.thread_log_setup()
run_ipfs_daemon()
log(f"{br(get_date())} Job's source code has been sent through ",
"bold cyan",
end="")
if self.cloudStorageID[0] == StorageID.IPFS:
log("[bold green]IPFS")
else:
log("[bold green]IPFS_GPG")
if not is_ipfs_on():
return False
log(f"==> is_hash_locally_cached={cfg.ipfs.is_hash_locally_cached(self.job_key)}"
)
if not os.path.isdir(self.results_folder):
os.makedirs(self.results_folder)
_remove(f"{self.results_folder}/{self.job_key}")
try:
self.check_ipfs(self.job_key)
except:
return False
self.registered_data_hashes = []
for idx, source_code_hash in enumerate(self.code_hashes):
if self.cloudStorageID[idx] == StorageID.NONE:
self.registered_data_hashes.append(source_code_hash) # GOTCHA
else:
ipfs_hash = bytes32_to_ipfs(source_code_hash)
if ipfs_hash not in self.ipfs_hashes:
try: # job_key as data hash already may added to the list
self.check_ipfs(ipfs_hash)
except:
return False
initial_folder_size = calculate_size(self.results_folder)
for idx, ipfs_hash in enumerate(self.ipfs_hashes):
# here scripts knows that provided IPFS hashes exists online
is_hashed = False
log(f"## attempting to get IPFS file: {ipfs_hash} ... ", end="")
if cfg.ipfs.is_hash_locally_cached(ipfs_hash):
is_hashed = True
log(ok("already cached"))
else:
log()
if idx == 0:
target = self.results_folder
else:
# "_" added before the filename in case $ ipfs get <ipfs_hash>
target = f"{self.results_data_folder}/_{ipfs_hash}"
mkdir(target)
is_storage_paid = False # TODO: should be set before by user input
cfg.ipfs.get(ipfs_hash, target, is_storage_paid)
if idx > 0:
# https://stackoverflow.com/a/31814223/2402577
dst_filename = os.path.join(self.results_data_folder,
os.path.basename(ipfs_hash))
if os.path.exists(dst_filename):
_remove(dst_filename)
shutil.move(target, dst_filename)
target = dst_filename
if self.cloudStorageID[idx] == StorageID.IPFS_GPG:
cfg.ipfs.decrypt_using_gpg(f"{target}/{ipfs_hash}", target)
try:
_git.initialize_check(target)
except Exception as e:
raise e
if not is_hashed:
folder_size = calculate_size(self.results_folder)
self.data_transfer_in_to_download_mb += folder_size - initial_folder_size
initial_folder_size = folder_size
if idx == 0 and not self.check_run_sh():
self.complete_refund()
return False
log(f"==> data_transfer_in={self.data_transfer_in_to_download_mb} MB | "
f"rounded={int(self.data_transfer_in_to_download_mb)} MB")
return self.sbatch_call()