def get(self, ipfs_hash, path, is_storage_paid):
if not is_ipfs_on():
raise IpfsNotConnected
output = run_with_output(
["ipfs", "get", ipfs_hash, f"--output={path}"])
logging.info(output)
if is_storage_paid:
# pin downloaded ipfs hash if storage is paid
output = check_output(["ipfs", "pin", "add",
ipfs_hash]).decode("utf-8").rstrip()
logging.info(output)
def diff_patch(path: Path, source_code_hash, index, target_path):
"""Apply diff patch.
"git diff HEAD" for detecting all the changes:
Shows all the changes between the working directory and HEAD (which includes changes in the index).
This shows all the changes since the last commit, whether or not they have been staged for commit
or not.
"""
is_file_empty = False
with cd(path):
log(f"==> Navigate to {path}")
"""TODO
if not is_initialized(path):
upload everything, changed files!
"""
repo = git.Repo(".", search_parent_directories=True)
try:
repo.git.config("core.fileMode",
"false") # git config core.fileMode false
# first ignore deleted files not to be added into git
run([env.BASH_SCRIPTS_PATH / "git_ignore_deleted.sh"])
head_commit_id = repo.rev_parse("HEAD")
sep = "~" # separator in between the string infos
patch_name = f"patch{sep}{head_commit_id}{sep}{source_code_hash}{sep}{index}.diff"
except:
return False
patch_upload_name = f"{patch_name}.gz" # file to be uploaded as zip
patch_file = f"{target_path}/{patch_upload_name}"
logging.info(f"patch_path={patch_upload_name}")
try:
repo.git.add(A=True)
diff_and_gzip(patch_file)
except:
return False
time.sleep(0.25)
if is_gzip_file_empty(patch_file):
log("==> Created patch file is empty, nothing to upload")
with suppress(Exception):
os.remove(patch_upload_name)
os.remove(patch_file)
is_file_empty = True
return patch_upload_name, patch_file, is_file_empty
def check_ipfs(self, ipfs_hash) -> None:
"""Append into self.ipfs_hashes."""
try:
ipfs_stat, cumulative_size = cfg.ipfs.is_hash_exists_online(
ipfs_hash)
if "CumulativeSize" not in ipfs_stat:
raise Exception(
"E: Markle not found! Timeout for the IPFS object stat retrieve"
)
except Exception as e:
print_tb(e)
raise e
self.ipfs_hashes.append(ipfs_hash)
self.cumulative_sizes[self.job_key] = cumulative_size
data_size_mb = byte_to_mb(cumulative_size)
logging.info(
f"data_transfer_out={data_size_mb} MB | Rounded={int(data_size_mb)} MB"
)
def gdrive_download_folder(self, name, key, source_code_hash, _id, cache_folder) -> bool:
log(f"{WHERE(1)}")
if self._is_cached(source_code_hash, _id):
return True
is_continue = False
with suppress(Exception):
output = self.job_infos[0]["storage_duration"][_id]
is_continue = True
if is_continue and not self.job_infos[0]["is_cached"][source_code_hash] and not output:
log("## Downloaded as temporary data file", "bold yellow")
self.folder_path_to_download[source_code_hash] = self.results_folder_prev
else:
self.folder_path_to_download[source_code_hash] = cache_folder
# self.assign_folder_path_to_download(_id, source_code_hash, cache_folder)
log(f"## downloading => {key}\nPath to download => {self.folder_path_to_download[source_code_hash]}")
if self.folder_type_dict[source_code_hash] == "folder":
try:
folder = self.folder_path_to_download[source_code_hash]
subprocess_call(
["gdrive", "download", "--recursive", key, "--force", "--path", folder],
10,
)
except Exception as e:
raise e
downloaded_folder_path = f"{self.folder_path_to_download[source_code_hash]}/{name}"
if not os.path.isdir(downloaded_folder_path):
# check before move operation
raise Exception(f"E: Folder ({downloaded_folder_path}) is not downloaded successfully")
self.data_transfer_in_requested = calculate_size(downloaded_folder_path)
logging.info(
f"data_transfer_in_requested={self.data_transfer_in_requested} MB | "
f"Rounded={int(self.data_transfer_in_requested)} MB"
)
else:
try:
folder = self.folder_path_to_download[source_code_hash]
cmd = ["gdrive", "download", key, "--force", "--path", folder]
subprocess_call(cmd, 10)
except Exception as e:
raise e
file_path = f"{self.folder_path_to_download[source_code_hash]}/{name}"
if not os.path.isfile(file_path):
raise Exception(f"{WHERE(1)} E: File {file_path} is not downloaded successfully")
filename = f"{self.folder_path_to_download[source_code_hash]}/{name}"
p1 = subprocess.Popen(
[
"ls",
"-ln",
filename,
],
stdout=subprocess.PIPE,
)
p2 = subprocess.Popen(["awk", "{print $5}"], stdin=p1.stdout, stdout=subprocess.PIPE)
p1.stdout.close() # type: ignore
# returns downloaded files size in bytes
self.data_transfer_in_requested = byte_to_mb(p2.communicate()[0].decode("utf-8").strip())
logging.info(
f"data_transfer_in_requested={self.data_transfer_in_requested} MB |"
f" Rounded={int(self.data_transfer_in_requested)} MB"
)
cores = sys.argv[6] # type: List[str] # noqa
elapsed_time = sys.argv[7] # type: List[str] # noqa
else:
provider = Ebb.w3.toChecksumAddress(env.PROVIDER_ID)
_from = Ebb.w3.toChecksumAddress(env.PROVIDER_ID)
job_key = "QmXFVGtxUBLfR2cYPNQtUjRxMv93yzUdej6kYwV1fqUD3U"
index = 0
job_id = 0
cores = ["1"]
elapsed_time = ["5"]
try:
tx_hash = Ebb.refund(provider, _from, job_key, index, job_id, cores,
elapsed_time)
receipt = get_tx_status(tx_hash)
if receipt["status"] == 1:
processed_logs = Ebb._eBlocBroker.events.LogRefundRequest(
).processReceipt(receipt, errors=DISCARD)
log(vars(processed_logs[0].args))
try:
logging.info(
f"refunded_wei={processed_logs[0].args['refundedWei']}")
log("SUCCESS", "green")
except Exception as e:
logging.error(f"E: Transaction is reverted. {e}")
except QuietExit:
pass
except Exception as e:
print_tb(e)
sys.exit(1)
def size(key, mime_type, folder_name, gdrive_info, results_folder_prev,
code_hashes, is_cached):
source_code_key = None
size_to_download = 0
if "folder" not in mime_type:
raise Exception
try:
output = get_file_id(key)
log(f"==> data_id=[magenta]{key}")
log(output, "bold green")
data_files_id = fetch_grive_output(output, "meta_data.json")
# key for the source_code elimination output*.tar.gz files
source_code_key = fetch_grive_output(output, f"{folder_name}.tar.gz")
if not data_files_id:
raise Exception
cmd = [
"gdrive",
"download",
"--recursive",
data_files_id, # first id is meta_data
"--force",
"--path",
results_folder_prev,
]
output = subprocess_call(cmd, 10)
print(output)
cmd = [
"gdrive",
"info",
"--bytes",
source_code_key,
"-c",
env.GDRIVE_METADATA,
]
gdrive_info = subprocess_call(cmd, 10)
except Exception as e:
print_tb(e)
# TODO: gdrive list --query "sharedWithMe"
raise e
md5sum = get_file_info(gdrive_info, "Md5sum")
_source_code_hash = code_hashes[0].decode("utf-8")
if md5sum != _source_code_hash:
# checks md5sum obtained from gdrive and given by the user
raise Exception(
f"E: md5sum does not match with the provided data {source_code_key}"
)
log(f"SUCCESS on folder={md5sum}", "bold green")
byte_size = int(get_file_info(gdrive_info, "Size"))
log(f"## code_hashes[0] == {_source_code_hash} | size={byte_size} bytes")
if not is_cached[code_hashes[0].decode("utf-8")]:
size_to_download += byte_size
try:
meta_data = get_data_key_ids(results_folder_prev)
except Exception as e:
raise e
data_key_dict = {}
if len(meta_data.items()) > 1:
idx = 0
for (k, v) in meta_data.items():
if idx == 0: # first item is for the source-code itself
_key = str(v)
output = get_file_id(_key)
data_key = fetch_grive_output(output, f"{k}.tar.gz")
cmd = [
"gdrive", "info", "--bytes", data_key, "-c",
env.GDRIVE_METADATA
]
gdrive_info = subprocess_call(cmd, 10)
log(f" * gdrive_info for [green]{k}[/green]:")
log(gdrive_info, "bold yellow")
idx += 1
else: # should start from the first index
_key = str(v)
try:
output = get_file_id(_key)
data_key = fetch_grive_output(output, f"{k}.tar.gz")
cmd = [
"gdrive", "info", "--bytes", data_key, "-c",
env.GDRIVE_METADATA
]
gdrive_info = subprocess_call(cmd, 10)
except Exception as e:
raise e
md5sum = get_file_info(gdrive_info, _type="Md5sum")
log(f" * gdrive_info for [green]{k}[/green]:")
log(gdrive_info, "bold yellow")
given_code_hash = code_hashes[idx].decode("utf-8")
log(f"==> given_code_hash={given_code_hash} idx={idx}")
if md5sum != given_code_hash:
# checks md5sum obtained from gdrive and given by the user
raise Exception(
f"E: md5sum does not match with the provided data{br(idx)}\n"
f"md5sum={md5sum} | given={given_code_hash}")
data_key_dict[md5sum] = data_key
_size = int(get_file_info(gdrive_info, "Size"))
log(f"==> code_hashes{br(idx)} == {code_hashes[idx].decode('utf-8')} size={_size} bytes"
)
byte_size += _size
if not is_cached[code_hashes[idx].decode("utf-8")]:
size_to_download += _size
if bool(data_key_dict):
data_link_file = f"{results_folder_prev}/meta_data_link.json"
with open(data_link_file, "w") as f:
json.dump(data_key_dict, f)
else:
raise Exception("E: Something is wrong. data_key_dict is empty")
output = byte_to_mb(size_to_download)
logging.info(
f"Total_size={byte_size} bytes | size to download={size_to_download} bytes => {output} MB"
)
return output, data_key_dict, source_code_key
def eudat_get_share_token(self, f_id):
"""Check key is already shared or not."""
folder_token_flag = {}
if not os.path.isdir(self.private_dir):
raise Exception(f"{self.private_dir} does not exist")
share_id_file = f"{self.private_dir}/{self.job_key}_share_id.json"
for idx, source_code_hash_text in enumerate(
self.code_hashes_to_process):
if self.cloudStorageID[idx] != StorageID.NONE:
folder_name = source_code_hash_text
self.folder_type_dict[folder_name] = None
source_fn = f"{folder_name}/{folder_name}.tar.gz"
if os.path.isdir(env.OWNCLOUD_PATH / f"{folder_name}"):
log(f"## eudat shared folder({folder_name}) is already accepted and "
"exists on the eudat's mounted folder")
if os.path.isfile(f"{env.OWNCLOUD_PATH}/{source_fn}"):
self.folder_type_dict[folder_name] = "tar.gz"
else:
self.folder_type_dict[folder_name] = "folder"
try:
info = config.oc.file_info(f"/{source_fn}")
logging.info("shared folder is already accepted")
size = info.attributes["{DAV:}getcontentlength"]
folder_token_flag[folder_name] = True
log(f"==> index={br(idx)}: /{source_fn} => {size} bytes")
except:
log(f"warning: shared_folder{br(source_code_hash_text, 'green')} is not accepted yet"
)
folder_token_flag[folder_name] = False
try: # TODO: add pass on template
data = read_json(share_id_file)
if isinstance(data, dict) and bool(data):
self.share_id = data
except:
pass
if self.share_id:
log("==> share_id:")
log(self.share_id, "bold")
for share_key, value in self.share_id.items(
): # there is only single item
try:
# TODO: if added before or some do nothing
if Ebb.mongo_broker.add_item_share_id(share_key,
value["share_id"],
value["share_token"]):
# adding into mongoDB for future usage
log(f"#> [green]{share_key}[/green] is added into mongoDB {ok()}"
)
except Exception as e:
print_tb(e)
log(f"E: {e}")
for attempt in range(config.RECONNECT_ATTEMPTS):
try:
share_list = config.oc.list_open_remote_share()
break
except Exception as e:
log(f"E: Failed to list_open_remote_share eudat [attempt={attempt}]"
)
print_tb(e)
time.sleep(1)
else:
break
else:
return False
self.accept_flag = 0
for idx, source_code_hash_text in enumerate(
self.code_hashes_to_process):
if self.cloudStorageID[idx] == StorageID.NONE:
self.accept_flag += 1
else:
# folder should not be registered data on the provider
#: search_token is priority
if not self.search_token(f_id, share_list,
source_code_hash_text):
try:
share_key = f"{source_code_hash_text}_{self.requester_id[:16]}"
shared_id = Ebb.mongo_broker.find_shareid_item(
key=share_key)
self.share_id[share_key] = {
"share_id": shared_id["share_id"],
"share_token": shared_id["share_token"],
}
self.accept_flag += 1
except Exception as e:
if "warning: " not in str(e):
log(f"E: {e}")
else:
log(str(e))
if folder_token_flag[folder_name] and bool(
self.share_id):
self.accept_flag += 1
else:
self.search_token(f_id, share_list, folder_name)
if self.accept_flag is len(self.code_hashes):
break
else:
if self.accept_flag is len(self.code_hashes):
logging.info("shared token already exists on mongoDB")
# else:
# raise Exception(f"E: could not find a shared file. Found ones are:\n{self.share_id}")
if bool(self.share_id):
with open(share_id_file, "w") as f:
json.dump(self.share_id, f)
else:
raise Exception("E: share_id is empty")