def _update_data_price():
Ebb = cfg.Ebb
if not Ebb.does_provider_exist(env.PROVIDER_ID):
log(f"warning: Provider {env.PROVIDER_ID} is not registered.\n")
raise QuietExit
source_code_hash = "b6aaf03752dc68d625fc57b451faa2bf"
new_data_price = 21
commitment_block_duration = 600
source_code_hash_bytes = cfg.w3.toBytes(text=source_code_hash)
try:
(price, _commitment_block_duration) = cfg.Ebb.get_registered_data_prices(
env.PROVIDER_ID, source_code_hash_bytes, 0
)
if price == new_data_price and _commitment_block_duration == commitment_block_duration:
log(f"## data([green]{source_code_hash}[/green]) already registerered with the given values")
raise QuietExit
except:
raise QuietExit
try:
tx = Ebb.update_data_price(source_code_hash_bytes, new_data_price, commitment_block_duration)
get_tx_status(Ebb.tx_id(tx))
except QuietExit:
pass
except Exception as e:
print_tb(e)
def deposit_storage(eth_address, is_provider=False):
"""Deposit storage balance.
:param str eth_address: Ethereum address of the provider
:param bool is_provider: Checks it the caller provider
"""
from_block = Ebb.get_deployed_block_number()
if is_provider:
event_filter = Ebb._eBlocBroker.events.LogJob.createFilter(
fromBlock=int(from_block),
argument_filters={"provider": eth_address},
toBlock="latest",
)
else: # should be owner of the job
event_filter = Ebb._eBlocBroker.events.LogJob.createFilter(
fromBlock=int(from_block),
argument_filters={"owner": eth_address},
toBlock="latest",
)
for job in enumerate(event_filter.get_all_entries()):
job_info = job[1].args
flag_check = []
for idx, code_hash in enumerate(job_info["sourceCodeHash"]):
main_cloud_storage_id = job_info["cloudStorageID"][idx]
if main_cloud_storage_id in (StorageID.IPFS, StorageID.IPFS_GPG):
_hash = bytes32_to_ipfs(code_hash)
_type = "ipfs_hash"
else:
_hash = cfg.w3.toText(code_hash)
_type = "md5sum"
log(br(f"{idx}, {_type}"), "bold cyan", end="")
if len(code_hash) <= 32:
log(f" {_hash} bytes={code_hash}", "bold")
else:
log(f" {_hash}\n\t{code_hash}", "bold")
provider = Ebb.w3.toChecksumAddress(job_info["provider"])
if is_provider and eth_address.lower() == provider.lower():
data_owner = Ebb.w3.toChecksumAddress(job_info["owner"])
deposit, output = Ebb.get_storage_info(provider, data_owner,
code_hash)
flag_check.append(output[3])
log(f"deposit={deposit}, {output}", "bold")
if deposit > 0 and not any(
flag_check): # if not any(i for i in flag_check):
is_verified_list = [True, True]
tx = Ebb._data_received(
job_info["jobKey"],
job_info["index"],
job_info["sourceCodeHash"],
job_info["cacheType"],
is_verified_list,
)
get_tx_status(Ebb.tx_id(tx))
else:
log("warning: already all data files are are verifid")
def main():
owner_address = Ebb.get_owner()
for user in users:
try:
tx_hash = Ebb.authenticate_orc_id(user, "0000-0001-7642-0552", owner_address)
if tx_hash:
get_tx_status(tx_hash)
except Exception as e:
print_tb(e)
def main():
yaml_fn = "~/ebloc-broker/broker/test_setup/requester.yaml"
for user in users:
yaml_user = Yaml(yaml_fn)
yaml_user["config"]["account"] = user
with suppress(Exception):
tx_hash = Ebb.register_requester(yaml_fn, is_question=False)
if tx_hash:
get_tx_status(tx_hash)
def main():
provider = env.PROVIDER_ID
pre_check_data(provider)
Ebb = cfg.Ebb
data_hash = b"f13d75bc60898f0823566347e380a34d"
try:
if is_data_registered(provider, data_hash):
tx = Ebb.remove_registered_data(data_hash)
get_tx_status(Ebb.tx_id(tx))
else:
log(f"## data({data_hash}) is alread deleted or not registered")
except Exception as e:
print_tb(e)
def pre_submit(storage_ids, provider_address):
is_pass = True
required_confs = 0
yaml_fn = Path.home(
) / "ebloc-broker" / "broker" / "test_setup" / "nas" / "job_nas.yaml"
yaml_cfg = Yaml(yaml_fn)
yaml_cfg["config"]["provider_address"] = provider_address
for storage_id in storage_ids:
yaml_cfg["config"]["source_code"]["storage_id"] = storage_id
benchmark_name = create_nas_job_script(is_small=True)
submit_base = SubmitBase(yaml_cfg.path)
tx_hash = submit_base.submit(is_pass, required_confs)
if required_confs >= 1:
tx_receipt = get_tx_status(tx_hash, is_silent=True)
if tx_receipt["status"] == 1:
processed_logs = Ebb._eBlocBroker.events.LogJob(
).processReceipt(tx_receipt, errors=DISCARD)
try:
if processed_logs:
job_result = vars(processed_logs[0].args)
job_result["tx_hash"] = tx_hash
job_result[
"submitted_job_kind"] = f"nas_{benchmark_name}"
log(job_result)
except IndexError:
log(f"E: Tx({tx_hash}) is reverted")
def register_requester(yaml_fn):
"""Return provider info.
:param str yaml_fn: Full file path of Yaml file that contains the requester info
"""
t1.join()
try:
tx_hash = Ebb.register_requester(yaml_fn)
if tx_hash:
get_tx_status(tx_hash)
else:
log()
except QuietExit:
pass
except Exception as e:
print_tb(e)
def main():
Ebb = cfg.Ebb
if len(sys.argv) == 3:
address = str(sys.argv[1])
orc_id = str(sys.argv[2])
else:
log("E: Please provide the address and its orc_id as argument")
log("[bold] ./authenticate_orc_id.py <address> <orc_id>", "mangenta")
sys.exit(1)
try:
owner_address = Ebb.get_owner()
tx_hash = Ebb.authenticate_orc_id(address, orc_id, owner_address)
if tx_hash:
get_tx_status(tx_hash)
except Exception as e:
print_tb(e)
def _withdraw():
Ebb = cfg.Ebb
if len(sys.argv) == 2:
account = str(sys.argv[1])
else:
log("## provide an ethereum account as an argument")
sys.exit(1)
try:
balance = Ebb.get_balance(account)
if balance > 0:
log(f"account_balance={balance}", "bold")
get_tx_status(Ebb.withdraw(account))
else:
log("warning: account balance is empty nothing to do")
except QuietExit:
pass
except Exception as e:
print_tb(e)
def submit_gdrive(job: Job, is_pass=False, required_confs=1):
log("==> Submitting source code through [blue]GDRIVE[/blue]")
pre_check()
Ebb = cfg.Ebb
job.folders_to_share = job.paths
check_link_folders(job.data_paths,
job.registered_data_files,
is_pass=is_pass)
_git.generate_git_repo(job.folders_to_share)
job.clean_before_submit()
requester = Ebb.w3.toChecksumAddress(job.requester_addr)
provider = Ebb.w3.toChecksumAddress(job.provider_addr)
job = gdrive.submit(provider, requester, job)
for folder_to_share in job.folders_to_share:
if isinstance(folder_to_share, bytes):
code_hash = folder_to_share
job.code_hashes.append(code_hash)
job.code_hashes_str.append(code_hash.decode("utf-8"))
else:
tar_hash = job.foldername_tar_hash[folder_to_share]
#: required to send string as bytes == str_data.encode('utf-8')
code_hash = Ebb.w3.toBytes(text=tar_hash)
job.code_hashes.append(code_hash)
job.code_hashes_str.append(code_hash.decode("utf-8"))
tar_hash = job.foldername_tar_hash[job.folders_to_share[0]]
key = job.keys[tar_hash]
job.price, *_ = job.cost(provider, requester)
try:
tx_hash = Ebb.submit_job(provider,
key,
job,
requester=requester,
required_confs=required_confs)
tx_receipt = get_tx_status(tx_hash)
if tx_receipt["status"] == 1:
processed_logs = Ebb._eBlocBroker.events.LogJob().processReceipt(
tx_receipt, errors=DISCARD)
log(vars(processed_logs[0].args))
try:
log(f"{ok()} [bold]job_index={processed_logs[0].args['index']}"
)
except IndexError:
log(f"E: Tx({tx_hash}) is reverted")
except QuietExit:
pass
except Exception as e:
print_tb(e)
log()
for k, v in job.tar_hashes.items():
log(f"{k} [blue]=>[/blue] {v}")
return tx_hash
def _register_data(source_code_hash, data_price, commitment_blk_duration):
Ebb = cfg.Ebb
is_exit = False
price = None
if not Ebb.does_provider_exist(env.PROVIDER_ID):
log(f"warning: provider [green]{env.PROVIDER_ID}[/green] is not registered")
raise QuietExit
if not Ebb.is_orcid_verified(env.PROVIDER_ID):
log(f"warning: provider [green]{env.PROVIDER_ID}[/green]'s orcid id is not authenticated yet")
raise QuietExit
source_code_hash_bytes = cfg.w3.toBytes(text=source_code_hash)
try:
(price, _commitment_blk_duration) = cfg.Ebb.get_registered_data_prices(
env.PROVIDER_ID, source_code_hash_bytes, 0
)
log(
f"## data([green]{source_code_hash}[/green]) is already registerered.\n"
"Use [blue]./update_data_price.py[/blue] to update its price"
)
is_exit = True
except:
pass
if is_exit:
raise QuietExit
if price == data_price and _commitment_blk_duration == commitment_blk_duration:
log(f"## data([green]{source_code_hash}[/green]) already registerered with the given values")
raise QuietExit
try:
tx = Ebb.register_data(source_code_hash_bytes, data_price, commitment_blk_duration)
get_tx_status(Ebb.tx_id(tx))
except QuietExit as e:
raise e
except Exception as e:
print_tb(e)
def analyze_tx_status(self, tx_hash) -> bool:
try:
tx_receipt = get_tx_status(tx_hash)
try:
if not self.Ebb:
log("warning: self.Ebb is empty object")
processed_logs = self.Ebb.eBlocBroker.events.LogJob(
).processReceipt(tx_receipt, errors=self.w3.DISCARD)
log(vars(processed_logs[0].args))
log(f"==> job_index={processed_logs[0].args['index']}")
except IndexError:
log("E: Transaction is reverted")
return True
except Exception as e:
print_tb(e)
return False
def submit(provider, requester, job, required_confs=1):
try:
tx_hash = _submit(provider, requester, job, required_confs)
if required_confs >= 1:
tx_receipt = get_tx_status(tx_hash)
if tx_receipt["status"] == 1:
processed_logs = Ebb._eBlocBroker.events.LogJob().processReceipt(tx_receipt, errors=DISCARD)
log(vars(processed_logs[0].args))
try:
log(f"{ok()} [bold]job_index={processed_logs[0].args['index']}")
except IndexError:
log(f"E: Tx({tx_hash}) is reverted")
else:
log(f"tx_hash={tx_hash}", "bold")
except QuietExit:
pass
except Exception as e:
print_tb(e)
return tx_hash
if not available_core:
raise Exception("Please enter positive value for the available core number")
if not commitment_blk:
raise Exception("Please enter positive value for the commitment block number")
try:
tx = self._update_provider_prices(available_core, commitment_blk, prices)
return self.tx_id(tx)
except Exception as e:
print_tb(e)
raise e
if __name__ == "__main__":
Ebb = cfg.Ebb
available_core = 4
commitment_blk = 600
price_core_min = 100
price_data_transfer = 1
price_storage = 1
price_cache = 1
prices = [price_core_min, price_data_transfer, price_storage, price_cache]
try:
tx_hash = Ebb.update_provider_prices(available_core, commitment_blk, prices)
receipt = get_tx_status(tx_hash)
except Exception as e:
print_tb(e)
sys.exit(1)
def submit_ipfs(job: Job, is_pass=False, required_confs=1):
Ebb = cfg.Ebb
requester = Ebb.w3.toChecksumAddress(job.requester_addr)
provider = Ebb.w3.toChecksumAddress(job.provider_addr)
pre_check(job, requester)
log("==> Attemptting to submit a job")
main_storage_id = job.storage_ids[0]
job.folders_to_share = job.paths
check_link_folders(job.data_paths, job.registered_data_files, is_pass=is_pass)
if main_storage_id == StorageID.IPFS:
log("==> Submitting source code through [blue]IPFS[/blue]")
elif main_storage_id == StorageID.IPFS_GPG:
log("==> Submitting source code through [blue]IPFS_GPG[/blue]")
else:
log("E: Please provide IPFS or IPFS_GPG storage type for the source code")
sys.exit(1)
targets = []
try:
provider_info = Ebb.get_provider_info(provider)
except Exception as e:
print_tb(e)
sys.exit(1)
for idx, folder in enumerate(job.folders_to_share):
if isinstance(folder, Path):
target = folder
if job.storage_ids[idx] == StorageID.IPFS_GPG:
provider_gpg_finderprint = provider_info["gpg_fingerprint"]
if not provider_gpg_finderprint:
log("E: Provider did not register any GPG fingerprint")
sys.exit(1)
log(f"==> provider_gpg_finderprint={provider_gpg_finderprint}")
try:
# target is updated
target = cfg.ipfs.gpg_encrypt(provider_gpg_finderprint, target)
log(f"==> gpg_file={target}")
except Exception as e:
print_tb(e)
sys.exit(1)
try:
ipfs_hash = cfg.ipfs.add(target)
# ipfs_hash = ipfs.add(folder, True) # True includes .git/
run(["ipfs", "refs", ipfs_hash])
except Exception as e:
print_tb(e)
sys.exit(1)
if idx == 0:
key = ipfs_hash
job.code_hashes.append(ipfs_to_bytes32(ipfs_hash))
job.code_hashes_str.append(ipfs_hash)
log(f"==> ipfs_hash={ipfs_hash} | md5sum={generate_md5sum(target)}")
if main_storage_id == StorageID.IPFS_GPG:
# created gpg file will be removed since its already in ipfs
targets.append(target)
else:
code_hash = folder
if isinstance(code_hash, bytes):
job.code_hashes.append(code_hash)
job.code_hashes_str.append(code_hash.decode("utf-8"))
# TODO: if its ipfs
# if isinstance(code_hash, bytes):
# code_hash = code_hash.decode("utf-8")
# if len(code_hash) == 32:
# value = cfg.w3.toBytes(text=code_hash)
# job.code_hashes.append(value)
# job.code_hashes_str.append(value.decode("utf-8"))
# else:
# job.code_hashes.append(ipfs_to_bytes32(code_hash))
# job.code_hashes_str.append(code_hash)
# if idx != len(job.folders_to_share) - 1:
# log("-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-", "cyan")
# requester inputs for testing purpose
job.price, *_ = job.cost(provider, requester)
try:
tx_hash = Ebb.submit_job(provider, key, job, requester=requester, required_confs=required_confs)
if required_confs >= 1:
tx_receipt = get_tx_status(tx_hash)
if tx_receipt["status"] == 1:
processed_logs = Ebb._eBlocBroker.events.LogJob().processReceipt(tx_receipt, errors=DISCARD)
try:
if processed_logs:
log("job_info:", "bold yellow")
log(vars(processed_logs[0].args))
for target in targets:
if ".tar.gz.gpg" in str(target):
_remove(target)
except IndexError:
log(f"E: Tx={tx_hash} is reverted")
else:
pass
except QuietExit:
pass
except Exception as e:
print_tb(e)
return tx_hash
def register_provider_wrapper(yaml_fn):
"""Register provider."""
yaml_fn = os.path.expanduser(yaml_fn)
if not os.path.exists(yaml_fn):
log(f"E: yaml_fn({yaml_fn}) does not exist")
raise QuietExit
args = Yaml(yaml_fn)
# @b2drop.eudat.eu
federation_cloud_id = args["cfg"]["oc_user"]
email = args["cfg"]["gmail"]
available_core = args["cfg"]["provider"]["available_core"]
commitment_blk = args["cfg"]["provider"]["prices"]["commitment_blk"]
price_core_min = args["cfg"]["provider"]["prices"]["price_core_min"]
price_data_transfer = args["cfg"]["provider"]["prices"][
"price_data_transfer"]
price_storage = args["cfg"]["provider"]["prices"]["price_storage"]
price_cache = args["cfg"]["provider"]["prices"]["price_cache"]
exit_flag = False
if not federation_cloud_id:
log(f"E: [blue]federation_cloud_id[/blue] is empty in [magenta]{yaml_fn}"
)
exit_flag = True
if not available_core:
log(f"E: [blue]available_core[/blue] is empty in [magenta]{yaml_fn}")
exit_flag = True
if not commitment_blk:
log(f"E: [blue]commitment_blk[/blue] is empty in [magenta]{yaml_fn}")
exit_flag = True
if not price_core_min:
log(f"E: [blue]price_core_min[/blue] is empty in [magenta]{yaml_fn}")
exit_flag = True
if not price_data_transfer:
log(f"E: [blue]price_data_transfer[/blue] is empty in [magenta]{yaml_fn}"
)
exit_flag = True
if not price_storage:
log(f"E: [blue]price_storage[/blue] is empty in [magenta]{yaml_fn}")
exit_flag = True
if not price_cache:
log(f"E: [blue]price_cache[/blue] is empty in [magenta]{yaml_fn}")
exit_flag = True
if not email:
log(f"E: [blue]email[/blue] is empty in [magenta]{yaml_fn}")
exit_flag = True
if exit_flag:
sys.exit(1)
ipfs_id = get_ipfs_id()
ip_address = get_ip()
if ip_address not in ipfs_id:
# public IP should exists in the ipfs id
ipfs_address = re.sub("ip4.*?tcp",
f"ip4/{ip_address}/tcp",
ipfs_id,
flags=re.DOTALL)
log(f"==> ipfs_address={ipfs_address}")
else:
ipfs_address = ipfs_id
try:
email = env.GMAIL
gpg_fingerprint = get_gpg_fingerprint(email)
is_gpg_published(gpg_fingerprint)
except Exception as e:
raise e
if not email:
log("E: Please provide a valid e-mail")
sys.exit(1)
prices = [price_core_min, price_data_transfer, price_storage, price_cache]
args = (gpg_fingerprint, email, federation_cloud_id, ipfs_address,
available_core, prices, commitment_blk)
kwargs = {
"email": email,
"federation_cloud_id": federation_cloud_id,
"commitment_blk": commitment_blk,
}
try:
tx_hash = Ebb._register_provider(*args, **kwargs)
if tx_hash:
get_tx_status(tx_hash)
else:
log()
except QuietExit:
pass
except Exception as e:
raise e
def _get_tx_status(self, tx_hash):
get_tx_status(tx_hash)
def main():
check_gdrive_user()
console_ruler(f"NEW_TEST {Ebb.get_block_number()}")
log(f" * {datetime.now().strftime('%Y-%m-%d %H:%M')}")
if not is_process_on("mongod", "mongod"):
raise Exception("mongodb is not running in the background")
storage_ids = ["eudat", "gdrive", "ipfs"]
ipfs_ids = ["ipfs_gpg", "ipfs"]
# for provider_address in provider_addresses:
# pre_submit(storage_ids, provider_address)
benchmarks = ["nas", "cppr"]
test_dir = Path.home() / "ebloc-broker" / "broker" / "test_setup" / "nas"
nas_yaml_fn = test_dir / "job_nas.yaml"
cppr_yam_fn = test_dir / "job_cppr.yaml"
yaml_cfg = None
counter = 0
for _ in range(60):
for _ in range(2): # submitted as batch is faster
for idx, provider_address in enumerate(provider_addresses):
# yaml_cfg["config"]["data"]["data3"]["storage_id"] = random.choice(storage_ids)
storage_id = (idx + counter) % len(storage_ids)
selected_benchmark = random.choice(benchmarks)
storage = storage_ids[storage_id]
if storage == "ipfs":
storage = random.choice(ipfs_ids)
if selected_benchmark == "nas":
log(
f" * Submitting job from NAS Benchmark to [green]{provider_address}",
"blue")
yaml_cfg = Yaml(nas_yaml_fn)
benchmark_name = create_nas_job_script()
elif selected_benchmark == "cppr":
log(
f" * Submitting job with cppr datasets to [green]{provider_address}",
"blue")
yaml_cfg = Yaml(cppr_yam_fn)
hash_small_data, hash_med_data = create_cppr_job_script()
yaml_cfg["config"]["data"]["data1"][
"hash"] = hash_small_data
yaml_cfg["config"]["data"]["data2"]["hash"] = hash_med_data
yaml_cfg["config"]["data"]["data3"]["storage_id"] = storage
small_datasets = Path.home(
) / "test_eblocbroker" / "dataset_zip" / "small"
dirs = [
d for d in os.listdir(small_datasets)
if os.path.isdir(os.path.join(small_datasets, d))
]
dir_name = random.choice(dirs)
yaml_cfg["config"]["data"]["data3"]["path"] = str(
small_datasets / dir_name)
yaml_cfg["config"]["source_code"]["storage_id"] = storage
yaml_cfg["config"]["provider_address"] = provider_address
try:
submit_base = SubmitBase(yaml_cfg.path)
submission_date = _date()
submission_timestamp = _timestamp()
requester_address = random.choice(users).lower()
yaml_cfg["config"]["requester_address"] = requester_address
log(f"requester={requester_address}", "bold")
tx_hash = submit_base.submit(is_pass=True)
log(f"tx_hash={tx_hash}", "bold")
tx_receipt = get_tx_status(tx_hash, is_silent=True)
if tx_receipt["status"] == 1:
processed_logs = Ebb._eBlocBroker.events.LogJob(
).processReceipt(tx_receipt, errors=DISCARD)
job_result = vars(processed_logs[0].args)
job_result["submit_date"] = submission_date
job_result["submit_timestamp"] = submission_timestamp
job_result["tx_hash"] = tx_hash
if selected_benchmark == "nas":
job_result[
"submitted_job_kind"] = f"{selected_benchmark}_{benchmark_name}"
elif selected_benchmark == "cppr":
job_result[
"submitted_job_kind"] = f"{selected_benchmark}_{hash_small_data}_{hash_med_data}"
ebb_mongo.add_item(tx_hash, job_result)
log(job_result)
countdown(seconds=5, is_silent=True)
except Exception as e:
print_tb(e)
counter += 1
sleep_time = randint(200, 400)
countdown(sleep_time)
"email": email,
"federation_cloud_id": federation_cloud_id,
"gpg_fingerprint": gpg_fingerprint,
"ipfs_id": ipfs_id,
}
log("==> [bold yellow]new_requester_info:")
log(_requester_info)
if is_question and not question_yes_no(
"#> Would you like to update requester info?"):
return
try:
tx = self._register_requester(account, gpg_fingerprint, email,
federation_cloud_id, ipfs_id)
return self.tx_id(tx)
except Exception as e:
print_tb(e)
raise e
if __name__ == "__main__":
try:
yaml_fn = "~/ebloc-broker/broker/yaml_files/register_requester.yaml"
tx_hash = Ebb.register_requester(yaml_fn)
if tx_hash:
get_tx_status(tx_hash)
except QuietExit:
pass
except Exception as e:
print_tb(e)
