def process_logged_job(self, idx):
"""Process logged job one by one."""
self.received_block = []
self.storage_duration = []
wait_until_idle_core_available()
self.is_provider_received_job = True
console_ruler(idx, character="-")
# sourceCodeHash = binascii.hexlify(logged_job.args['sourceCodeHash'][0]).decode("utf-8")[0:32]
job_key = self.logged_job.args["jobKey"]
index = self.logged_job.args["index"]
self.job_block_number = self.logged_job["blockNumber"]
self.cloud_storage_id = self.logged_job.args["cloudStorageID"]
log(f"## job_key=[magenta]{job_key}[/magenta] | index={index}")
log(
f"received_block_number={self.job_block_number} \n"
f"transactionHash={self.logged_job['transactionHash'].hex()} | "
f"log_index={self.logged_job['logIndex']} \n"
f"provider={self.logged_job.args['provider']} \n"
f"received={self.logged_job.args['received']}",
"bold yellow",
)
if self.logged_job["blockNumber"] > self.latest_block_number:
self.latest_block_number = self.logged_job["blockNumber"]
try:
run([env.BASH_SCRIPTS_PATH / "is_str_valid.sh", job_key])
except:
logging.error("E: Filename contains an invalid character")
return
try:
job_id = 0 # main job_id
self.job_info = eblocbroker_function_call(
partial(self.Ebb.get_job_info, env.PROVIDER_ID, job_key, index, job_id, self.job_block_number),
max_retries=10,
)
cfg.Ebb.get_job_code_hashes(env.PROVIDER_ID, job_key, index, self.job_block_number)
self.requester_id = self.job_info["job_owner"]
self.job_info.update({"received_block": self.received_block})
self.job_info.update({"storage_duration": self.storage_duration})
self.job_info.update({"cacheType": self.logged_job.args["cacheType"]})
cfg.Ebb.analyze_data(job_key, env.PROVIDER_ID)
self.job_infos.append(self.job_info)
log(f"==> requester={self.requester_id}")
log("==> [yellow]job_info:", "bold")
log(self.job_info)
except Exception as e:
print_tb(e)
return
for job in range(1, len(self.job_info["core"])):
with suppress(Exception):
self.job_infos.append( # if workflow is given then add jobs into list
self.Ebb.get_job_info(env.PROVIDER_ID, job_key, index, job, self.job_block_number)
)
self.check_requested_job()
def main(args):
given_bn = 0
try:
if args.bn:
given_bn = args.bn
elif args.latest:
given_bn = cfg.Ebb.get_block_number()
if args.is_thread is False:
cfg.IS_THREADING_ENABLED = False
console_ruler("provider session starts")
log(f" * {datetime.now().strftime('%Y-%m-%d %H:%M')}")
with launch_ipdb_on_exception():
# if an exception is raised, launch ipdb
_main(given_bn)
except KeyboardInterrupt:
sys.exit(1)
def handle_event(logged_jobs):
for job in logged_jobs:
cloud_storage_id = job.args["cloudStorageID"]
"""
if StorageID.IPFS == cloud_storage_id or cloudStorageID.IPFS_GPG == cloud_storage_id:
jobKey = bytes32_to_ipfs(logged_jobs[i].args['jobKey'])
else:
jobKey = logged_jobs[i].args['jobKey']
"""
log(f"transaction_hash={job['transactionHash'].hex()} | log_index={job['logIndex']}"
)
log(f"block_number={job['blockNumber']}")
log(f"provider={job.args['provider']}")
log(f"job_key={job.args['jobKey']}")
log(f"index={job.args['index']}")
log(f"cloud_storage_id={StorageID(cloud_storage_id).name}")
log(f"cache_type={CacheType(job.args['cacheType'].name)}")
log(f"received={job.args['received']}")
for value in job.args["sourceCodeHash"]:
sourceCodeHash = job.args["sourceCodeHash"][value]
log(f"source_code_hash{br(value)} => {bytes32_to_ipfs(sourceCodeHash)}"
)
console_ruler()
def submit_receipt(index,
cores,
start_time,
completion_time,
elapsed_time,
is_print=True):
text = f"{start_time},{completion_time}"
log(f"==> {br(text)} cores={cores}")
job = Job()
job.code_hashes = [b"8b3e98abb65d0c1aceea8d606fc55403"]
job.key = job.code_hashes[0]
job.index = index
job._id = 0
job.cores = cores
job.run_time = [1]
job.data_transfer_ins = [1]
job.data_transfer_out = 1
job.storage_ids = [StorageID.EUDAT.value]
job.cache_types = [CacheType.PUBLIC.value]
job.storage_hours = [0]
job.data_prices_set_block_numbers = [0]
job_price, _cost = job.cost(provider, requester)
provider_price_block_number = ebb.getProviderSetBlockNumbers(provider)[-1]
args = [
provider,
provider_price_block_number,
job.storage_ids,
job.cache_types,
job.data_prices_set_block_numbers,
job.cores,
job.run_time,
job.data_transfer_out,
]
tx = ebb.submitJob(
job.key,
job.data_transfer_ins,
args,
job.storage_hours,
job.code_hashes,
{
"from": requester,
"value": web3.toWei(job_price, "wei")
},
)
tx = ebb.setJobStatusRunning(job.key, job.index, job._id, start_time,
{"from": provider})
rpc.sleep(60)
mine(5)
data_transfer_in = 0
data_transfer_out = 0
args = [
job.index, job._id, completion_time, data_transfer_in,
data_transfer_out, job.cores, [1], True
]
tx = ebb.processPayment(job.key, args, elapsed_time, "",
{"from": provider})
if is_print:
log(f"==> process_payment received_gas_used={tx.__dict__['gas_used']}")
# received_sum = tx.events["LogProcessPayment"]["receivedWei"]
# refunded_sum = tx.events["LogProcessPayment"]["refundedWei"]
# withdraw(provider, received_sum)
# withdraw(requester, refunded_sum)
check_list(is_print)
if is_print:
console_ruler(character="-=")
return tx
def main():
check_gdrive_user()
console_ruler(f"NEW_TEST {Ebb.get_block_number()}")
log(f" * {datetime.now().strftime('%Y-%m-%d %H:%M')}")
if not is_process_on("mongod", "mongod"):
raise Exception("mongodb is not running in the background")
storage_ids = ["eudat", "gdrive", "ipfs"]
ipfs_ids = ["ipfs_gpg", "ipfs"]
# for provider_address in provider_addresses:
# pre_submit(storage_ids, provider_address)
benchmarks = ["nas", "cppr"]
test_dir = Path.home() / "ebloc-broker" / "broker" / "test_setup" / "nas"
nas_yaml_fn = test_dir / "job_nas.yaml"
cppr_yam_fn = test_dir / "job_cppr.yaml"
yaml_cfg = None
counter = 0
for _ in range(60):
for _ in range(2): # submitted as batch is faster
for idx, provider_address in enumerate(provider_addresses):
# yaml_cfg["config"]["data"]["data3"]["storage_id"] = random.choice(storage_ids)
storage_id = (idx + counter) % len(storage_ids)
selected_benchmark = random.choice(benchmarks)
storage = storage_ids[storage_id]
if storage == "ipfs":
storage = random.choice(ipfs_ids)
if selected_benchmark == "nas":
log(
f" * Submitting job from NAS Benchmark to [green]{provider_address}",
"blue")
yaml_cfg = Yaml(nas_yaml_fn)
benchmark_name = create_nas_job_script()
elif selected_benchmark == "cppr":
log(
f" * Submitting job with cppr datasets to [green]{provider_address}",
"blue")
yaml_cfg = Yaml(cppr_yam_fn)
hash_small_data, hash_med_data = create_cppr_job_script()
yaml_cfg["config"]["data"]["data1"][
"hash"] = hash_small_data
yaml_cfg["config"]["data"]["data2"]["hash"] = hash_med_data
yaml_cfg["config"]["data"]["data3"]["storage_id"] = storage
small_datasets = Path.home(
) / "test_eblocbroker" / "dataset_zip" / "small"
dirs = [
d for d in os.listdir(small_datasets)
if os.path.isdir(os.path.join(small_datasets, d))
]
dir_name = random.choice(dirs)
yaml_cfg["config"]["data"]["data3"]["path"] = str(
small_datasets / dir_name)
yaml_cfg["config"]["source_code"]["storage_id"] = storage
yaml_cfg["config"]["provider_address"] = provider_address
try:
submit_base = SubmitBase(yaml_cfg.path)
submission_date = _date()
submission_timestamp = _timestamp()
requester_address = random.choice(users).lower()
yaml_cfg["config"]["requester_address"] = requester_address
log(f"requester={requester_address}", "bold")
tx_hash = submit_base.submit(is_pass=True)
log(f"tx_hash={tx_hash}", "bold")
tx_receipt = get_tx_status(tx_hash, is_silent=True)
if tx_receipt["status"] == 1:
processed_logs = Ebb._eBlocBroker.events.LogJob(
).processReceipt(tx_receipt, errors=DISCARD)
job_result = vars(processed_logs[0].args)
job_result["submit_date"] = submission_date
job_result["submit_timestamp"] = submission_timestamp
job_result["tx_hash"] = tx_hash
if selected_benchmark == "nas":
job_result[
"submitted_job_kind"] = f"{selected_benchmark}_{benchmark_name}"
elif selected_benchmark == "cppr":
job_result[
"submitted_job_kind"] = f"{selected_benchmark}_{hash_small_data}_{hash_med_data}"
ebb_mongo.add_item(tx_hash, job_result)
log(job_result)
countdown(seconds=5, is_silent=True)
except Exception as e:
print_tb(e)
counter += 1
sleep_time = randint(200, 400)
countdown(sleep_time)
def run_driver(given_bn):
"""Run the main driver script for eblocbroker on the background."""
# dummy sudo command to get the password when session starts for only to
# create users and submit the slurm job under another user
run(["sudo", "printf", "hello"])
kill_process_by_name("gpg-agent")
config.logging = setup_logger(_log.DRIVER_LOG)
# driver_cancel_process = None
try:
from broker.imports import connect
connect()
Ebb: "Contract.Contract" = cfg.Ebb
driver = Driver()
except Exception as e:
raise Terminate from e
if not env.PROVIDER_ID:
raise Terminate(f"PROVIDER_ID is None in {env.LOG_PATH}/.env")
if not env.WHOAMI or not env.EBLOCPATH or not env.PROVIDER_ID:
raise Terminate(f"Please run: {env.BASH_SCRIPTS_PATH}/folder_setup.sh")
if not env.SLURMUSER:
raise Terminate(f"SLURMUSER is not set in {env.LOG_PATH}/.env")
try:
deployed_block_number = Ebb.get_deployed_block_number()
except Exception as e:
raise e
if not env.config["block_continue"]:
env.config["block_continue"] = deployed_block_number
if given_bn > 0:
block_number_saved = int(given_bn)
else:
block_number_saved = env.config["block_continue"]
if not isinstance(env.config["block_continue"], int):
log("E: block_continue variable is empty or contains an invalid character")
if not question_yes_no("#> Would you like to read from the contract's deployed block number?"):
terminate()
block_number_saved = deployed_block_number
if deployed_block_number:
env.config["block_continue"] = deployed_block_number
else:
raise Terminate(f"deployed_block_number={deployed_block_number} is invalid")
_tools(block_number_saved)
try:
Ebb.is_contract_exists()
except:
terminate(
"Contract address does not exist on the blockchain, is the blockchain sync?\n"
f"block_number={Ebb.get_block_number()}",
is_traceback=False,
)
if cfg.IS_THREADING_ENABLED:
log(f"## is_threading={cfg.IS_THREADING_ENABLED}")
Ebb.is_eth_account_locked(env.PROVIDER_ID)
log(f"==> whoami={env.WHOAMI}")
log(f"==> log_file={_log.DRIVER_LOG}")
log(f"==> rootdir={os.getcwd()}")
log(f"==> is_web3_connected={Ebb.is_web3_connected()}")
if not Ebb.does_provider_exist(env.PROVIDER_ID):
# updated since cluster is not registered
env.config["block_continue"] = Ebb.get_block_number()
terminate(
textwrap.fill(
f"Your Ethereum address {env.PROVIDER_ID} "
"does not match with any provider in eBlocBroker. Please register your "
"provider using your Ethereum Address in to the eBlocBroker. You can "
"use eblocbroker/register_provider.py script to register your provider."
),
is_traceback=False,
)
if not Ebb.is_orcid_verified(env.PROVIDER_ID):
raise QuietExit(f"provider's ({env.PROVIDER_ID}) ORCID is not verified")
blk_read = block_number_saved
balance_temp = Ebb.get_balance(env.PROVIDER_ID)
eth_balance = Ebb.eth_balance(env.PROVIDER_ID)
log(f"==> deployed_block_number={deployed_block_number}")
log(f"==> account_balance={eth_balance} gwei | {cfg.w3.fromWei(eth_balance, 'ether')} eth")
log(f"==> Ebb_balance={balance_temp}")
while True:
wait_until_idle_core_available()
time.sleep(0.2)
if not str(blk_read).isdigit():
raise Terminate(f"block_read_from={blk_read}")
balance = Ebb.get_balance(env.PROVIDER_ID)
if cfg.IS_THREADING_ENABLED:
_squeue()
console_ruler()
if isinstance(balance, int):
value = int(balance) - int(balance_temp)
if value > 0:
log(f"==> Since Driver started provider_gained_wei={value}")
current_bn = Ebb.get_block_number()
log(f" * {get_date()} waiting new job to come since block_number={blk_read}")
log(f"==> current_block={current_bn} | sync_from={blk_read}")
flag = True
while current_bn < int(blk_read):
current_bn = Ebb.get_block_number()
if flag:
log(f"## Waiting block number to be updated, it remains constant at {current_bn}")
flag = False
time.sleep(2)
log(f"#> [bold yellow]Passed incremented block number... Watching from block_number=[cyan]{blk_read}")
blk_read = str(blk_read) # reading events' block number has been updated
slurm.pending_jobs_check()
try:
driver.logged_jobs_to_process = Ebb.run_log_job(blk_read, env.PROVIDER_ID)
driver.process_logged_jobs()
if len(driver.logged_jobs_to_process) > 0 and driver.latest_block_number > 0:
# updates the latest read block number
blk_read = driver.latest_block_number + 1
env.config["block_continue"] = blk_read
if not driver.is_provider_received_job:
blk_read = env.config["block_continue"] = current_bn
except Exception as e:
log()
log(f"E: {e}")
if "Filter not found" in str(e) or "Read timed out" in str(e):
# HTTPSConnectionPool(host='core.bloxberg.org', port=443): Read timed out. (read timeout=10)
log("## sleeping for 60 seconds...", end="")
time.sleep(60)
log(ok())
else:
print_tb(e)
def test_workflow():
job = Job()
provider = accounts[0]
requester = accounts[1]
register_provider()
register_requester(requester)
job_key = "QmQv4AAL8DZNxZeK3jfJGJi63v1msLMZGan7vSsCDXzZud"
code_hash = ipfs_to_bytes32(job_key)
with brownie.reverts():
ebb.updataDataPrice(code_hash, 20, 100, {"from": provider})
ebb.registerData(code_hash, 20, cfg.BLOCK_DURATION_1_HOUR,
{"from": provider})
ebb.removeRegisteredData(
code_hash,
{"from": provider}) # should submitJob fail if it is not removed
code_hash1 = "0x68b8d8218e730fc2957bcb12119cb204"
# "web3.toBytes(hexstr=ipfs_to_bytes32("QmWmyoMoctfbAaiEs2G46gpeUmhqFRDW6KWo64y5r581Ve"))
ebb.registerData(code_hash1, 20, cfg.BLOCK_DURATION_1_HOUR,
{"from": provider})
mine(6)
with brownie.reverts():
ebb.registerData(code_hash1, 20, 1000, {"from": provider})
ebb.updataDataPrice(code_hash1, 250, cfg.BLOCK_DURATION_1_HOUR + 1,
{"from": provider})
data_block_numbers = ebb.getRegisteredDataBlockNumbers(
provider, code_hash1)
log(f"get_registered_data_block_numbers={data_block_numbers[1]}", "bold")
get_block_number()
data_prices = ebb.getRegisteredDataPrice(provider, code_hash1, 0)
log(f"register_data_price={data_prices}", "bold")
assert data_prices[0] == 20
res = ebb.getRegisteredDataPrice(provider, code_hash1,
data_block_numbers[1])
log(f"register_data_price={res}", "bold")
assert res[0] == 250
mine(cfg.BLOCK_DURATION_1_HOUR - 9)
res = ebb.getRegisteredDataPrice(provider, code_hash1, 0)
log(f"register_data_price={res}", "bold")
assert res[0] == 20
mine(1)
res = ebb.getRegisteredDataPrice(provider, code_hash1, 0)
log(f"register_data_price={res}", "bold")
assert res[0] == 250
job.code_hashes = [code_hash,
code_hash1] # Hashed of the data file in array
job.storage_hours = [0, 0]
job.data_transfer_ins = [100, 0]
job.data_transfer_out = 100
# job.data_prices_set_block_numbers = [0, 253] # TODO: check this ex 253 exists or not
job.data_prices_set_block_numbers = [
0, data_block_numbers[1]
] # TODO: check this ex 253 exists or not
check_price_keys(job.data_prices_set_block_numbers, provider, code_hash1)
job.cores = [2, 4, 2]
job.run_time = [10, 15, 20]
job.storage_ids = [StorageID.IPFS.value, StorageID.NONE.value]
job.cache_types = [CacheType.PUBLIC.value, CacheType.PUBLIC.value]
args = [
provider,
ebb.getProviderSetBlockNumbers(accounts[0])[-1],
job.storage_ids,
job.cache_types,
job.data_prices_set_block_numbers,
job.cores,
job.run_time,
job.data_transfer_out,
]
job_price, _cost = job.cost(provider, requester)
tx = ebb.submitJob( # first submit
job_key,
job.data_transfer_ins,
args,
job.storage_hours,
job.code_hashes,
{
"from": requester,
"value": web3.toWei(job_price, "wei")
},
)
for idx in range(0, 3):
log(ebb.getJobInfo(provider, job_key, 0, idx))
console_ruler(character="-=")
assert (
tx.events["LogRegisteredDataRequestToUse"][0]["registeredDataHash"] ==
"0x0000000000000000000000000000000068b8d8218e730fc2957bcb12119cb204"
), "registered data should be used"
with brownie.reverts():
log(ebb.getJobInfo(provider, job_key, 1, 2))
log(ebb.getJobInfo(provider, job_key, 0, 3))
# setJobStatus for the workflow:
index = 0
job_id = 0
start_time = 10
tx = ebb.setJobStatusRunning(job_key, index, job_id, start_time,
{"from": accounts[0]})
index = 0
job_id = 1
start_time = 20
tx = ebb.setJobStatusRunning(job_key, index, job_id, start_time,
{"from": accounts[0]})
# process_payment for the workflow
index = 0
job_id = 0
execution_time = 10
data_transfer = [100, 0]
end_time = 20
result_ipfs_hash = ipfs_to_bytes32(
"QmWmyoMoctfbAaiEs2G46gpeUmhqFRDW6KWo64y5r581Ve")
received_sums = []
refunded_sums = []
received_sum = 0
refunded_sum = 0
args = [
index, job_id, end_time, data_transfer[0], data_transfer[1], job.cores,
job.run_time, False
]
tx = ebb.processPayment(job_key, args, execution_time, result_ipfs_hash,
{"from": accounts[0]})
# log(tx.events['LogProcessPayment'])
received_sums.append(tx.events["LogProcessPayment"]["receivedWei"])
refunded_sums.append(tx.events["LogProcessPayment"]["refundedWei"])
received_sum += tx.events["LogProcessPayment"]["receivedWei"]
refunded_sum += tx.events["LogProcessPayment"]["refundedWei"]
log(f"received_sum={received_sum} | refunded_sum={refunded_sum} | job_price={job_price}"
)
# -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
index = 0
job_id = 1
execution_time = 15
data_transfer = [0, 0]
end_time = 39
result_ipfs_hash = ipfs_to_bytes32(
"QmWmyoMoctfbAaiEs2G46gpeUmhqFRDW6KWo64y5r581Ve")
args = [
index, job_id, end_time, data_transfer[0], data_transfer[1], job.cores,
job.run_time, False
]
tx = ebb.processPayment(job_key, args, execution_time, result_ipfs_hash,
{"from": accounts[0]})
received_sums.append(tx.events["LogProcessPayment"]["receivedWei"])
refunded_sums.append(tx.events["LogProcessPayment"]["refundedWei"])
received_sum += tx.events["LogProcessPayment"]["receivedWei"]
refunded_sum += tx.events["LogProcessPayment"]["refundedWei"]
log(f"received_sum={received_sum} | refunded_sum={refunded_sum} | job_price={job_price}"
)
# -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
index = 0
job_id = 2
execution_time = 20
data_transfer = [0, 100]
end_time = 39
result_ipfs_hash = ipfs_to_bytes32(
"QmWmyoMoctfbAaiEs2G46gpeUmhqFRDW6KWo64y5r581Ve")
with brownie.reverts(
): # processPayment should revert, setRunning is not called for the job=2
args = [
index,
job_id,
end_time,
data_transfer[0],
data_transfer[1],
job.cores,
job.run_time,
False,
]
tx = ebb.processPayment(job_key, args, execution_time,
result_ipfs_hash, {"from": accounts[0]})
index = 0
job_id = 2
start_time = 20
tx = ebb.setJobStatusRunning(job_key, index, job_id, start_time,
{"from": accounts[0]})
args = [
index, job_id, end_time, data_transfer[0], data_transfer[1], job.cores,
job.run_time, True
]
tx = ebb.processPayment(job_key, args, execution_time, result_ipfs_hash,
{"from": accounts[0]})
# log(tx.events['LogProcessPayment'])
received_sums.append(tx.events["LogProcessPayment"]["receivedWei"])
refunded_sums.append(tx.events["LogProcessPayment"]["refundedWei"])
received_sum += tx.events["LogProcessPayment"]["receivedWei"]
refunded_sum += tx.events["LogProcessPayment"]["refundedWei"]
log(f"received_sum={received_sum} | refunded_sum={refunded_sum} | job_price={job_price}"
)
log(received_sums)
log(refunded_sums)
assert job_price - _cost["storage"] == received_sum + refunded_sum
withdraw(accounts[0], received_sum)
withdraw(requester, refunded_sum)
def test_storage_refund():
job = Job()
provider = accounts[0]
requester = accounts[1]
register_provider()
register_requester(requester)
job_key = "QmQv4AAL8DZNxZeK3jfJGJi63v1msLMZGan7vSsCDXzZud"
job.code_hashes.append(ipfs_to_bytes32(job_key))
job.storage_hours.append(1)
job_key_2 = "QmVqtWxuBdZQdLnLce6XCBMuqoazAcbmuxoJHQbfbuqDu2"
job.code_hashes.append(ipfs_to_bytes32(job_key_2))
job.storage_hours.append(1)
job.data_transfer_ins = [100, 100]
job.data_transfer_out = 100
job.data_prices_set_block_numbers = [0, 0]
job.cores = [2]
job.run_time = [10]
job.provider_price_block_number = ebb.getProviderSetBlockNumbers(
accounts[0])[-1]
job.storage_ids = [StorageID.EUDAT.value, StorageID.IPFS.value]
job.cache_types = [CacheType.PRIVATE.value, CacheType.PUBLIC.value]
# provider's registered data won't be used
job.data_prices_set_block_numbers = [0, 0]
job_price, _cost = job.cost(provider, requester)
job_price += 1 # for test 1 wei extra is paid
args = [
provider,
job.provider_price_block_number,
job.storage_ids,
job.cache_types,
job.data_prices_set_block_numbers,
job.cores,
job.run_time,
job.data_transfer_out,
]
tx = ebb.submitJob(
job_key,
job.data_transfer_ins,
args,
job.storage_hours,
job.code_hashes,
{
"from": requester,
"value": web3.toWei(job_price, "wei")
},
)
refunded = tx.events["LogJob"]["refunded"]
log(f"==> job_index={tx.events['LogJob']['index']}")
log(f"refunded={refunded}", "bold")
log(tx.events["LogJob"]["jobKey"])
assert requester == tx.events["LogJob"]["owner"]
withdraw(requester, refunded) # check for extra payment is checked
index = 0
job_id = 0
tx = ebb.refund(provider, job_key, index, job_id, job.cores, job.run_time,
{"from": provider})
log(ebb.getJobInfo(provider, job_key, index, job_id))
refundedWei = tx.events["LogRefundRequest"]["refundedWei"]
log(f"refunded_wei={refundedWei}", "bold")
withdraw(requester, refundedWei)
# VM Exception while processing transaction: invalid opcode
with brownie.reverts():
ebb.getJobInfo(provider, job_key, 5, job_id)
storage_cost_sum = 0
for code_hash in job.code_hashes:
_storage_cost_sum, *_ = ebb.getStorageInfo(provider, requester,
code_hash)
storage_cost_sum += _storage_cost_sum
assert _cost["storage"] == storage_cost_sum
assert _cost["computational"] + _cost["data_transfer"] + _cost[
"cache"] == refundedWei
mine(cfg.BLOCK_DURATION_1_HOUR)
tx = ebb.refundStorageDeposit(provider, requester, job.code_hashes[0], {
"from": requester,
"gas": 4500000
})
refundedWei = tx.events["LogDepositStorage"]["payment"]
log(f"refunded_wei={refundedWei}", "bold")
withdraw(requester, refundedWei)
with brownie.reverts():
tx = ebb.refundStorageDeposit(provider, requester, job.code_hashes[0],
{
"from": requester,
"gas": 4500000
})
tx = ebb.refundStorageDeposit(provider, requester, job.code_hashes[1], {
"from": requester,
"gas": 4500000
})
refundedWei = tx.events["LogDepositStorage"]["payment"]
paid_address = tx.events["LogDepositStorage"]["paidAddress"]
withdraw(requester, refundedWei)
with brownie.reverts():
tx = ebb.refundStorageDeposit(provider, requester, job.code_hashes[0],
{
"from": requester,
"gas": 4500000
})
assert requester == paid_address
assert ebb.balanceOf(provider) == 0
console_ruler("same job submitted after full refund", color="blue")
tx = ebb.submitJob(
job_key,
job.data_transfer_ins,
args,
job.storage_hours,
job.code_hashes,
{
"from": requester,
"value": web3.toWei(job_price, "wei")
},
)
log(f"job_index={tx.events['LogJob']['index']}", "bold")
log(tx.events["LogJob"]["jobKey"])
index = 1
job_id = 0
tx = ebb.refund(provider, job_key, index, job_id, job.cores, job.run_time,
{"from": provider})
log(ebb.getJobInfo(provider, job_key, index, job_id))
refundedWei = tx.events["LogRefundRequest"]["refundedWei"]
log(f"refunded_wei={refundedWei}", "bold")
assert _cost["computational"] + _cost["data_transfer"] + _cost[
"cache"] == refundedWei
storage_cost_sum = 0
storage_payment = []
for code_hash in job.code_hashes:
deposit, *_ = ebb.getStorageInfo(provider, requester, code_hash)
storage_payment.append(deposit)
job.is_verified = [True, True]
ebb.dataReceived( # called by the provider
job_key, index, job.code_hashes, job.cache_types, job.is_verified, {
"from": provider,
"gas": 4500000
})
for code_hash in job.code_hashes:
*_, output = ebb.getStorageInfo(provider, cfg.ZERO_ADDRESS, code_hash)
log(output, "bold")
with brownie.reverts(
): # refundStorageDeposit should revert, because it is already used by the provider
for code_hash in job.code_hashes:
tx = ebb.refundStorageDeposit(provider, requester, code_hash, {
"from": requester,
"gas": 4500000
})
tx = ebb.depositStorage(requester, job.code_hashes[0], {
"from": provider,
"gas": 4500000
})
mine(cfg.BLOCK_DURATION_1_HOUR)
# after deadline (1 hr) is completed to store the data, provider could obtain the money
for idx, code_hash in enumerate(job.code_hashes):
tx = ebb.depositStorage(requester, code_hash, {
"from": provider,
"gas": 4500000
})
amount = tx.events["LogDepositStorage"]["payment"]
withdraw(provider, amount)
assert storage_payment[idx] == amount
def test_submit_job():
job = Job()
provider = accounts[0]
requester = accounts[1]
register_provider()
register_requester(requester)
fname = f"{cwd}/files/test.txt"
# fname = f"{cwd}/files/test_.txt"
log(f"==> registered_provider_addresses={ebb.getProviders()}")
provider_price_info = ebb.getProviderInfo(accounts[0], 0)
# block_read_from = provider_price_info[0]
_provider_price_info = provider_price_info[1]
# availableCoreNum = _provider_price_info[0]
# commitmentBlockDuration = _provider_price_info[1]
price_core_min = _provider_price_info[2]
# price_data_transfer = _provider_price_info[3]
# price_storage = _provider_price_info[4]
# price_cache = _provider_price_info[5]
log(f"provider_available_core={available_core}")
log(f"provider_price_core_min={price_core_min}")
log(provider_price_info)
job_price_sum = 0
job_id = 0
index = 0
with open(fname) as f:
for line in f:
arguments = line.rstrip("\n").split(" ")
storage_hour = 1
core_min = int(arguments[1]) - int(arguments[0])
core = int(arguments[2])
job.cores = [core]
job.run_time = [core_min]
# time.sleep(1)
# rpc.mine(int(arguments[0]))
job_key = "QmQv4AAL8DZNxZeK3jfJGJi63v1msLMZGan7vSsCDXzZud"
data_key = "QmQv4AAL8DZNxZeK3jfJGJi63v1msLMZGan7vSsCDXzZud"
code_hash = ipfs_to_bytes32(data_key)
# log("Client Balance before: " + str(web3.eth.balanceOf(account)))
# log("Contract Balance before: " + str(web3.eth.balanceOf(accounts[0])))
job.code_hashes = [code_hash]
job.storage_hours = [storage_hour]
job.data_transfer_ins = [100]
job.data_transfer_out = 100
job.data_prices_set_block_numbers = [0]
job.storage_ids = [StorageID.IPFS.value]
job.cache_types = [CacheType.PUBLIC.value]
args = [
provider,
ebb.getProviderSetBlockNumbers(accounts[0])[-1],
job.storage_ids,
job.cache_types,
job.data_prices_set_block_numbers,
job.cores,
job.run_time,
job.data_transfer_out,
]
# log(code_hashes[0])
job_price, _cost = job.cost(provider, requester)
job_price_sum += job_price
data_transfer_ins = [100]
job_key = job.storage_hours[0]
tx = ebb.submitJob(
job_key,
data_transfer_ins,
args,
job.storage_hours,
job.code_hashes,
{
"from": requester,
"value": web3.toWei(job_price, "wei")
},
)
# log('submitJob => GasUsed:' + str(tx.__dict__['gas_used']) + '| blockNumber=' + str(tx.block_number))
log(f"job_index={tx.events['LogJob']['index']}", "bold")
# log("Contract Balance after: " + str(web3.eth.balanceOf(accouts[0])))
# log("Client Balance after: " + str(web3.eth.balanceOf(accounts[8])))
# sys.stdout.write('jobInfo: ')
# sys.stdout.flush()
log(ebb.getJobInfo(provider, job_key, index, job_id))
index += 1
log(f"total_paid={job_price_sum}")
# log(block_read_from)
# rpc.mine(100)
# log(web3.eth.blockNumber)
job_id = 0
with open(fname) as f:
for index, line in enumerate(f):
arguments = line.rstrip("\n").split(" ")
tx = ebb.setJobStatusRunning(job_key, index, job_id,
int(arguments[0]),
{"from": accounts[0]})
if index == 0:
with brownie.reverts():
tx = ebb.setJobStatusRunning(job_key, index, job_id,
int(arguments[0]) + 1,
{"from": accounts[0]})
console_ruler()
result_ipfs_hash = ipfs_to_bytes32(
"QmWmyoMoctfbAaiEs2G46gpeUmhqFRDW6KWo64y5r581Ve")
with open(fname) as f:
for index, line in enumerate(f):
arguments = line.rstrip("\n").split(" ")
if index == 0:
data_transfer_in_sum = 90
job.data_transfer_out = 100
else:
data_transfer_in_sum = 0
job.data_transfer_out = 100
core_min = int(arguments[1]) - int(arguments[0])
core = int(arguments[2])
job.cores = [core]
job.run_time = [core_min]
log(f"contract_balance={ebb.getContractBalance()}", "bold")
job_id = 0
execution_time = int(arguments[1]) - int(arguments[0])
end_time = int(arguments[1])
args = [
index,
job_id,
end_time,
data_transfer_in_sum,
job.data_transfer_out,
job.cores,
job.run_time,
True,
]
tx = ebb.processPayment(job_key, args, execution_time,
result_ipfs_hash, {"from": accounts[0]})
# code_hashes
received = tx.events["LogProcessPayment"]["receivedWei"]
refunded = tx.events["LogProcessPayment"]["refundedWei"]
withdraw(accounts[0], received)
withdraw(requester, refunded)
log(f"received={received} | refunded={refunded}", "bold")
log(f"contract_balance={ebb.getContractBalance()}", "bold")
for idx in range(0, ebb.getProviderReceiptSize(provider)):
# prints finalize version of the linked list
log(ebb.getProviderReceiptNode(provider, idx))
console_ruler()
log(f"==> storage_duration for job={job_key}")
*_, job_storage_info = ebb.getStorageInfo(provider, cfg.ZERO_ADDRESS,
code_hash)
ds = DataStorage(job_storage_info)
log(f"receivedBlockNumber={ds.received_block} |"
f"storage_duration(block numbers)={ds.storage_duration} | "
f"is_private={ds.is_private} |"
f"is_verified_Used={ds.is_verified_used}")
received_storage_deposit, *_ = ebb.getStorageInfo(provider, requester,
code_hash)
log(f"received_storage_deposit={received_storage_deposit}")
console_ruler("DONE")
