def main():
coll = cl["eBlocBroker"]["cache"]
block_number = Ebb.get_block_number()
storageID = None
cursor = coll.find({})
for document in cursor:
# print(document)
received_block_number, storage_duration = Ebb.get_job_storage_duration(
env.PROVIDER_ID, document["sourceCodeHash"]
)
end_block_time = received_block_number + storage_duration * cfg.BLOCK_DURATION_1_HOUR
storageID = document["storageID"]
if end_block_time < block_number and received_block_number != 0:
if storageID in (StorageID.IPFS, StorageID.IPFS_GPG):
ipfsHash = document["jobKey"]
print(run(["ipfs", "pin", "rm", ipfsHash]))
print(run(["ipfs", "repo", "gc"]))
else:
cached_file_name = (
env.PROGRAM_PATH / document["requesterID"] / "cache" / document["sourceCodeHash"] + "tar.gz"
)
print(cached_file_name)
_remove(cached_file_name)
cached_file_name = env.PROGRAM_PATH / "cache" / document["sourceCodeHash"] + "tar.gz"
print(cached_file_name)
_remove(cached_file_name)
print(received_block_number)
coll.delete_one({"jobKey": ipfsHash})
def process_logged_job(self, idx):
"""Process logged job one by one."""
self.received_block = []
self.storage_duration = []
wait_until_idle_core_available()
self.is_provider_received_job = True
console_ruler(idx, character="-")
# sourceCodeHash = binascii.hexlify(logged_job.args['sourceCodeHash'][0]).decode("utf-8")[0:32]
job_key = self.logged_job.args["jobKey"]
index = self.logged_job.args["index"]
self.job_block_number = self.logged_job["blockNumber"]
self.cloud_storage_id = self.logged_job.args["cloudStorageID"]
log(f"## job_key=[magenta]{job_key}[/magenta] | index={index}")
log(
f"received_block_number={self.job_block_number} \n"
f"transactionHash={self.logged_job['transactionHash'].hex()} | "
f"log_index={self.logged_job['logIndex']} \n"
f"provider={self.logged_job.args['provider']} \n"
f"received={self.logged_job.args['received']}",
"bold yellow",
)
if self.logged_job["blockNumber"] > self.latest_block_number:
self.latest_block_number = self.logged_job["blockNumber"]
try:
run([env.BASH_SCRIPTS_PATH / "is_str_valid.sh", job_key])
except:
logging.error("E: Filename contains an invalid character")
return
try:
job_id = 0 # main job_id
self.job_info = eblocbroker_function_call(
partial(self.Ebb.get_job_info, env.PROVIDER_ID, job_key, index, job_id, self.job_block_number),
max_retries=10,
)
cfg.Ebb.get_job_code_hashes(env.PROVIDER_ID, job_key, index, self.job_block_number)
self.requester_id = self.job_info["job_owner"]
self.job_info.update({"received_block": self.received_block})
self.job_info.update({"storage_duration": self.storage_duration})
self.job_info.update({"cacheType": self.logged_job.args["cacheType"]})
cfg.Ebb.analyze_data(job_key, env.PROVIDER_ID)
self.job_infos.append(self.job_info)
log(f"==> requester={self.requester_id}")
log("==> [yellow]job_info:", "bold")
log(self.job_info)
except Exception as e:
print_tb(e)
return
for job in range(1, len(self.job_info["core"])):
with suppress(Exception):
self.job_infos.append( # if workflow is given then add jobs into list
self.Ebb.get_job_info(env.PROVIDER_ID, job_key, index, job, self.job_block_number)
)
self.check_requested_job()
def is_repo(folders):
for folder in folders:
if not isinstance(folder, bytes):
with cd(folder):
if not is_initialized(folder):
log(f"warning: .git does not exits in {folder}. Applying: git init ",
end="")
run(["git", "init", "--initial-branch=master"])
log(ok())
def daemon():
if args.daemon_type[0] == "ipfs":
from broker.utils import run_ipfs_daemon
run_ipfs_daemon(_is_print=True)
if args.daemon_type[0] == "slurm":
from broker.config import env
from broker.utils import run
run(["sudo", env.BASH_SCRIPTS_PATH / "run_slurm.sh"])
def run_stdout_to_file(cmd, path, mode="w") -> None:
"""Run command pipe output into give file."""
p, output, error = popen_communicate(cmd, stdout_fn=path, mode=mode)
if p.returncode != 0 or (isinstance(error, str) and "error:" in error):
_cmd = " ".join(cmd)
log(f"\n{_cmd}", "red")
raise Exception(f"E: scontrol error:\n{output}")
# log(f"## writing into path({path}) is completed")
run(["sed", "-i", "s/[ \t]*$//",
path]) # remove trailing whitespaces with sed
def initialize_check(path):
"""Validate if .git/ folder exist within the target folder."""
with cd(path):
if not is_initialized(path):
try:
log(f"## git_repo={path}")
log("Creating an empty Git repository ", end="")
run(["git", "init", "--initial-branch=master"])
log(ok())
add_all()
except Exception as e:
log(f"E: {e}")
raise e
def decrypt_using_gpg(self, gpg_file, extract_target=None):
"""Decrypt compresses file using gpg.
This function is specific for using on driver.ipfs to decript tar file,
specific for "tar.gz" file types.
cmd:
gpg --verbose --output={tar_file} --pinentry-mode loopback \
--passphrase-file=f"{env.LOG_PATH}/gpg_pass.txt" \
--decrypt {gpg_file_link}
"""
if not os.path.isfile(f"{gpg_file}.gpg"):
os.symlink(gpg_file, f"{gpg_file}.gpg")
gpg_file_link = f"{gpg_file}.gpg"
tar_file = f"{gpg_file}.tar.gz"
cmd = [
"gpg",
"--verbose",
"--batch",
"--yes",
f"--output={tar_file}",
"--pinentry-mode",
"loopback",
f"--passphrase-file={env.GPG_PASS_FILE}",
"--decrypt",
gpg_file_link,
]
try:
run(cmd)
log(f"==> GPG decrypt {ok()}")
_remove(gpg_file)
os.unlink(gpg_file_link)
except Exception as e:
print_tb(e)
raise e
# finally:
# os.unlink(gpg_file_link)
if extract_target:
try:
untar(tar_file, extract_target)
except Exception as e:
raise Exception(
"E: Could not extract the given tar file") from e
finally:
cmd = None
_remove(f"{extract_target}/.git")
_remove(tar_file)
def commit_changes(path):
with cd(path):
repo = git.Repo(".", search_parent_directories=True)
try:
output = run(["ls", "-l", ".git/refs/heads"])
except Exception as e:
raise Exception("E: Problem on git.commit_changes()") from e
if output == "total 0":
logging.warning("There is no first commit")
else:
changed_files = [item.a_path for item in repo.index.diff(None)]
if len(changed_files) > 0:
log("==> adding changed files:")
for _file in changed_files:
log(_file, "bold")
repo.git.add(A=True)
if len(repo.index.diff("HEAD")) == 0:
log(f"==> {path}\n is committed with the given changes using git"
)
try:
add_all(repo)
except Exception as e:
log(f"E: {e}")
raise e
def diff_patch(path: Path, source_code_hash, index, target_path):
"""Apply diff patch.
"git diff HEAD" for detecting all the changes:
Shows all the changes between the working directory and HEAD (which includes changes in the index).
This shows all the changes since the last commit, whether or not they have been staged for commit
or not.
"""
is_file_empty = False
with cd(path):
log(f"==> Navigate to {path}")
"""TODO
if not is_initialized(path):
upload everything, changed files!
"""
repo = git.Repo(".", search_parent_directories=True)
try:
repo.git.config("core.fileMode",
"false") # git config core.fileMode false
# first ignore deleted files not to be added into git
run([env.BASH_SCRIPTS_PATH / "git_ignore_deleted.sh"])
head_commit_id = repo.rev_parse("HEAD")
sep = "~" # separator in between the string infos
patch_name = f"patch{sep}{head_commit_id}{sep}{source_code_hash}{sep}{index}.diff"
except:
return False
patch_upload_name = f"{patch_name}.gz" # file to be uploaded as zip
patch_file = f"{target_path}/{patch_upload_name}"
logging.info(f"patch_path={patch_upload_name}")
try:
repo.git.add(A=True)
diff_and_gzip(patch_file)
except:
return False
time.sleep(0.25)
if is_gzip_file_empty(patch_file):
log("==> Created patch file is empty, nothing to upload")
with suppress(Exception):
os.remove(patch_upload_name)
os.remove(patch_file)
is_file_empty = True
return patch_upload_name, patch_file, is_file_empty
def apply_patch(git_folder, patch_file, is_gpg=False):
"""Apply git patch.
output = repo.git.apply("--reject", "--whitespace=fix",
"--ignore-space-change", "--ignore-whitespace", "--verbose", patch_file)
__ https://stackoverflow.com/a/15375869/2402577
"""
if is_gpg:
cfg.ipfs.decrypt_using_gpg(patch_file)
with cd(git_folder):
base_name = path_leaf(patch_file)
log(f"==> [magenta]{base_name}")
# folder_name = base_name_split[2]
#
# base_name_split = base_name.split("_")
# git_hash = base_name_split[1]
# run(["git", "checkout", git_hash])
# run(["git", "reset", "--hard"])
# run(["git", "clean", "-f"])
# echo "\n" >> patch_file.txt seems like fixing it
#
# with open(patch_file, "a") as myfile:
# myfile.write("\n")
cmd = [
"git",
"apply",
"--reject",
"--whitespace=fix",
"--ignore-space-change",
"--ignore-whitespace",
"--verbose",
patch_file,
] # ,is_quiet=True,
cmd_summary = cmd.copy()
cmd_summary.insert(3, "--summary")
output = run(cmd_summary)
log(output)
output = run(cmd)
log(output)
def connect_into_web3() -> None:
"""Connect into private network using web3.
Note that you should create only one RPC Provider per process, as it
recycles underlying TCP/IP network connections between your process and
Ethereum node
"""
web3_ipc_path = env.DATADIR.joinpath("geth.ipc")
for _ in range(5):
_connect_into_web3()
if not cfg.w3.isConnected():
try:
if env.IS_GETH_TUNNEL:
raise Exception(
"web3ConnectError: try tunnelling: ssh -f -N -L 8545:localhost:8545 username@<ip>"
)
if not env.IS_BLOXBERG:
is_geth_on()
else:
log("E: web3 is not connected into [green]BLOXBERG[/green]"
)
except QuietExit:
pass
except Exception as e:
print_tb(e)
sys.exit(1)
if not env.IS_GETH_TUNNEL and not env.IS_BLOXBERG:
log(
"E: If web3 is not connected please start geth server and give permission \n"
"to /private/geth.ipc file doing: ",
end="",
)
log(f"sudo chown $(logname) {web3_ipc_path}", "green")
log(f"#> Running `sudo chown $(whoami) {web3_ipc_path}`")
run(["sudo", "chown", env.WHOAMI, web3_ipc_path])
else:
break
else:
terminate(is_traceback=False)
def new_contract_function_call(self, code_hash):
"""Call contract function with new brownie object.
This part is not needed when `{from: }` is removed from the getter in the smart contract
"""
filename = call.__file__
data = ("func", self.job.provider, self.job.requester, code_hash)
output = run(["python", filename,
*[str(arg) for arg in data]]) # GOTCHA
output = output.split("\n")
received_storage_deposit = float(output[0])
job_storage_duration = make_tuple(output[1])
return received_storage_deposit, job_storage_duration
def is_hash_locally_cached(self,
ipfs_hash: str,
ipfs_refs_local=None) -> bool:
"""Return true if hash locally cached.
Run `ipfs --offline refs -r` or `ipfs --offline block stat` etc even if your normal daemon is running.
With that you can check if something is available locally or no.
"""
if not ipfs_refs_local:
ipfs_refs_local = run(["ipfs", "refs", "local"]).split("\n")
for _hash in ipfs_refs_local:
if ipfs_hash == _hash:
return True
return False
def get_only_ipfs_hash(self, path, is_hidden=True) -> str:
"""Get only chunk and hash of a given path, do not write to disk.
Args:
path: Path of a folder or file
Return string that contains the ouput of the run commad.
"""
if os.path.isdir(path):
cmd = ["ipfs", "add", "-r", "--quieter", "--only-hash", path]
if is_hidden:
# include files that are hidden such as .git/
# Only takes effect on recursive add
cmd.insert(3, "--hidden")
elif os.path.isfile(path):
cmd = ["ipfs", "add", "--quieter", "--only-hash", path]
else:
raise Exception("Requested path does not exist")
try:
return _try(lambda: run(cmd))
except Exception as e:
raise e
def gpg_encrypt(self, user_gpg_finderprint, target):
is_delete = False
if os.path.isdir(target):
try:
*_, encrypt_target = compress_folder(target)
encrypted_file_target = f"{encrypt_target}.gpg"
is_delete = True
except Exception as e:
print_tb(e)
sys.exit(1)
else:
if not os.path.isfile(target):
logging.error(f"{target} does not exist")
sys.exit(1)
else:
encrypt_target = target
encrypted_file_target = f"{target}.gpg"
is_delete = True
if os.path.isfile(encrypted_file_target):
log(f"## gpg_file: {encrypted_file_target} is already created")
return encrypted_file_target
for attempt in range(5):
try:
cmd = [
"gpg", "--keyserver", "hkps://keyserver.ubuntu.com",
"--recv-key", user_gpg_finderprint
]
log(f"{br(attempt)} cmd: [magenta]{' '.join(cmd)}", "bold")
run(
cmd
) # this may not work if it is requested too much in a short time
break
except Exception as e:
log(f"warning: {e}")
time.sleep(30)
try:
cmd = [
"gpg",
"--batch",
"--yes",
"--recipient",
user_gpg_finderprint,
"--trust-model",
"always",
"--output",
encrypted_file_target,
"--encrypt",
encrypt_target,
]
run(cmd)
log(f"==> gpg_file=[magenta]{encrypted_file_target}")
return encrypted_file_target
except Exception as e:
print_tb(e)
if "encryption failed: Unusable public key" in str(e):
log("#> Check solution: https://stackoverflow.com/a/34132924/2402577"
)
finally:
if is_delete:
_remove(encrypt_target)
if __name__ == "__main__":
import call
from broker.utils import run
filename = call.__file__
func_name = "getStorageInfo"
data = (
"",
"0xD118b6EF83ccF11b34331F1E7285542dDf70Bc49",
"0xD118b6EF83ccF11b34331F1E7285542dDf70Bc49",
"QmdxhbcHJr3r4yeJNdvRabLVJ78VT2DTNMkBD7LN2MzeqJ",
)
output = run(["python", filename, *[str(arg) for arg in data]])
print(output)
# # in case filename is an executable you don't need "python" before `filename`:
# try:
# # for breakpoint remove , stdout=subprocess.PIPE
# p = subprocess.Popen(args=["python", filename, *[str(arg) for arg in data]], stdout=subprocess.PIPE)
# output, error = p.communicate()
# p.wait()
# time.sleep(1)
# if output:
# output = output.strip().decode("utf-8")
# print(output)
# except:
# breakpoint() # DEBUG
# pass
def pin(self, ipfs_hash: str) -> bool:
return run(["ipfs", "pin", "add", ipfs_hash])
def run_driver(given_bn):
"""Run the main driver script for eblocbroker on the background."""
# dummy sudo command to get the password when session starts for only to
# create users and submit the slurm job under another user
run(["sudo", "printf", "hello"])
kill_process_by_name("gpg-agent")
config.logging = setup_logger(_log.DRIVER_LOG)
# driver_cancel_process = None
try:
from broker.imports import connect
connect()
Ebb: "Contract.Contract" = cfg.Ebb
driver = Driver()
except Exception as e:
raise Terminate from e
if not env.PROVIDER_ID:
raise Terminate(f"PROVIDER_ID is None in {env.LOG_PATH}/.env")
if not env.WHOAMI or not env.EBLOCPATH or not env.PROVIDER_ID:
raise Terminate(f"Please run: {env.BASH_SCRIPTS_PATH}/folder_setup.sh")
if not env.SLURMUSER:
raise Terminate(f"SLURMUSER is not set in {env.LOG_PATH}/.env")
try:
deployed_block_number = Ebb.get_deployed_block_number()
except Exception as e:
raise e
if not env.config["block_continue"]:
env.config["block_continue"] = deployed_block_number
if given_bn > 0:
block_number_saved = int(given_bn)
else:
block_number_saved = env.config["block_continue"]
if not isinstance(env.config["block_continue"], int):
log("E: block_continue variable is empty or contains an invalid character")
if not question_yes_no("#> Would you like to read from the contract's deployed block number?"):
terminate()
block_number_saved = deployed_block_number
if deployed_block_number:
env.config["block_continue"] = deployed_block_number
else:
raise Terminate(f"deployed_block_number={deployed_block_number} is invalid")
_tools(block_number_saved)
try:
Ebb.is_contract_exists()
except:
terminate(
"Contract address does not exist on the blockchain, is the blockchain sync?\n"
f"block_number={Ebb.get_block_number()}",
is_traceback=False,
)
if cfg.IS_THREADING_ENABLED:
log(f"## is_threading={cfg.IS_THREADING_ENABLED}")
Ebb.is_eth_account_locked(env.PROVIDER_ID)
log(f"==> whoami={env.WHOAMI}")
log(f"==> log_file={_log.DRIVER_LOG}")
log(f"==> rootdir={os.getcwd()}")
log(f"==> is_web3_connected={Ebb.is_web3_connected()}")
if not Ebb.does_provider_exist(env.PROVIDER_ID):
# updated since cluster is not registered
env.config["block_continue"] = Ebb.get_block_number()
terminate(
textwrap.fill(
f"Your Ethereum address {env.PROVIDER_ID} "
"does not match with any provider in eBlocBroker. Please register your "
"provider using your Ethereum Address in to the eBlocBroker. You can "
"use eblocbroker/register_provider.py script to register your provider."
),
is_traceback=False,
)
if not Ebb.is_orcid_verified(env.PROVIDER_ID):
raise QuietExit(f"provider's ({env.PROVIDER_ID}) ORCID is not verified")
blk_read = block_number_saved
balance_temp = Ebb.get_balance(env.PROVIDER_ID)
eth_balance = Ebb.eth_balance(env.PROVIDER_ID)
log(f"==> deployed_block_number={deployed_block_number}")
log(f"==> account_balance={eth_balance} gwei | {cfg.w3.fromWei(eth_balance, 'ether')} eth")
log(f"==> Ebb_balance={balance_temp}")
while True:
wait_until_idle_core_available()
time.sleep(0.2)
if not str(blk_read).isdigit():
raise Terminate(f"block_read_from={blk_read}")
balance = Ebb.get_balance(env.PROVIDER_ID)
if cfg.IS_THREADING_ENABLED:
_squeue()
console_ruler()
if isinstance(balance, int):
value = int(balance) - int(balance_temp)
if value > 0:
log(f"==> Since Driver started provider_gained_wei={value}")
current_bn = Ebb.get_block_number()
log(f" * {get_date()} waiting new job to come since block_number={blk_read}")
log(f"==> current_block={current_bn} | sync_from={blk_read}")
flag = True
while current_bn < int(blk_read):
current_bn = Ebb.get_block_number()
if flag:
log(f"## Waiting block number to be updated, it remains constant at {current_bn}")
flag = False
time.sleep(2)
log(f"#> [bold yellow]Passed incremented block number... Watching from block_number=[cyan]{blk_read}")
blk_read = str(blk_read) # reading events' block number has been updated
slurm.pending_jobs_check()
try:
driver.logged_jobs_to_process = Ebb.run_log_job(blk_read, env.PROVIDER_ID)
driver.process_logged_jobs()
if len(driver.logged_jobs_to_process) > 0 and driver.latest_block_number > 0:
# updates the latest read block number
blk_read = driver.latest_block_number + 1
env.config["block_continue"] = blk_read
if not driver.is_provider_received_job:
blk_read = env.config["block_continue"] = current_bn
except Exception as e:
log()
log(f"E: {e}")
if "Filter not found" in str(e) or "Read timed out" in str(e):
# HTTPSConnectionPool(host='core.bloxberg.org', port=443): Read timed out. (read timeout=10)
log("## sleeping for 60 seconds...", end="")
time.sleep(60)
log(ok())
else:
print_tb(e)
