def get(self, ipfs_hash, path, is_storage_paid): if not is_ipfs_on(): raise IpfsNotConnected output = run_with_output( ["ipfs", "get", ipfs_hash, f"--output={path}"]) logging.info(output) if is_storage_paid: # pin downloaded ipfs hash if storage is paid output = check_output(["ipfs", "pin", "add", ipfs_hash]).decode("utf-8").rstrip() logging.info(output)
def diff_patch(path: Path, source_code_hash, index, target_path): """Apply diff patch. "git diff HEAD" for detecting all the changes: Shows all the changes between the working directory and HEAD (which includes changes in the index). This shows all the changes since the last commit, whether or not they have been staged for commit or not. """ is_file_empty = False with cd(path): log(f"==> Navigate to {path}") """TODO if not is_initialized(path): upload everything, changed files! """ repo = git.Repo(".", search_parent_directories=True) try: repo.git.config("core.fileMode", "false") # git config core.fileMode false # first ignore deleted files not to be added into git run([env.BASH_SCRIPTS_PATH / "git_ignore_deleted.sh"]) head_commit_id = repo.rev_parse("HEAD") sep = "~" # separator in between the string infos patch_name = f"patch{sep}{head_commit_id}{sep}{source_code_hash}{sep}{index}.diff" except: return False patch_upload_name = f"{patch_name}.gz" # file to be uploaded as zip patch_file = f"{target_path}/{patch_upload_name}" logging.info(f"patch_path={patch_upload_name}") try: repo.git.add(A=True) diff_and_gzip(patch_file) except: return False time.sleep(0.25) if is_gzip_file_empty(patch_file): log("==> Created patch file is empty, nothing to upload") with suppress(Exception): os.remove(patch_upload_name) os.remove(patch_file) is_file_empty = True return patch_upload_name, patch_file, is_file_empty
def check_ipfs(self, ipfs_hash) -> None: """Append into self.ipfs_hashes.""" try: ipfs_stat, cumulative_size = cfg.ipfs.is_hash_exists_online( ipfs_hash) if "CumulativeSize" not in ipfs_stat: raise Exception( "E: Markle not found! Timeout for the IPFS object stat retrieve" ) except Exception as e: print_tb(e) raise e self.ipfs_hashes.append(ipfs_hash) self.cumulative_sizes[self.job_key] = cumulative_size data_size_mb = byte_to_mb(cumulative_size) logging.info( f"data_transfer_out={data_size_mb} MB | Rounded={int(data_size_mb)} MB" )
def gdrive_download_folder(self, name, key, source_code_hash, _id, cache_folder) -> bool: log(f"{WHERE(1)}") if self._is_cached(source_code_hash, _id): return True is_continue = False with suppress(Exception): output = self.job_infos[0]["storage_duration"][_id] is_continue = True if is_continue and not self.job_infos[0]["is_cached"][source_code_hash] and not output: log("## Downloaded as temporary data file", "bold yellow") self.folder_path_to_download[source_code_hash] = self.results_folder_prev else: self.folder_path_to_download[source_code_hash] = cache_folder # self.assign_folder_path_to_download(_id, source_code_hash, cache_folder) log(f"## downloading => {key}\nPath to download => {self.folder_path_to_download[source_code_hash]}") if self.folder_type_dict[source_code_hash] == "folder": try: folder = self.folder_path_to_download[source_code_hash] subprocess_call( ["gdrive", "download", "--recursive", key, "--force", "--path", folder], 10, ) except Exception as e: raise e downloaded_folder_path = f"{self.folder_path_to_download[source_code_hash]}/{name}" if not os.path.isdir(downloaded_folder_path): # check before move operation raise Exception(f"E: Folder ({downloaded_folder_path}) is not downloaded successfully") self.data_transfer_in_requested = calculate_size(downloaded_folder_path) logging.info( f"data_transfer_in_requested={self.data_transfer_in_requested} MB | " f"Rounded={int(self.data_transfer_in_requested)} MB" ) else: try: folder = self.folder_path_to_download[source_code_hash] cmd = ["gdrive", "download", key, "--force", "--path", folder] subprocess_call(cmd, 10) except Exception as e: raise e file_path = f"{self.folder_path_to_download[source_code_hash]}/{name}" if not os.path.isfile(file_path): raise Exception(f"{WHERE(1)} E: File {file_path} is not downloaded successfully") filename = f"{self.folder_path_to_download[source_code_hash]}/{name}" p1 = subprocess.Popen( [ "ls", "-ln", filename, ], stdout=subprocess.PIPE, ) p2 = subprocess.Popen(["awk", "{print $5}"], stdin=p1.stdout, stdout=subprocess.PIPE) p1.stdout.close() # type: ignore # returns downloaded files size in bytes self.data_transfer_in_requested = byte_to_mb(p2.communicate()[0].decode("utf-8").strip()) logging.info( f"data_transfer_in_requested={self.data_transfer_in_requested} MB |" f" Rounded={int(self.data_transfer_in_requested)} MB" )
cores = sys.argv[6] # type: List[str] # noqa elapsed_time = sys.argv[7] # type: List[str] # noqa else: provider = Ebb.w3.toChecksumAddress(env.PROVIDER_ID) _from = Ebb.w3.toChecksumAddress(env.PROVIDER_ID) job_key = "QmXFVGtxUBLfR2cYPNQtUjRxMv93yzUdej6kYwV1fqUD3U" index = 0 job_id = 0 cores = ["1"] elapsed_time = ["5"] try: tx_hash = Ebb.refund(provider, _from, job_key, index, job_id, cores, elapsed_time) receipt = get_tx_status(tx_hash) if receipt["status"] == 1: processed_logs = Ebb._eBlocBroker.events.LogRefundRequest( ).processReceipt(receipt, errors=DISCARD) log(vars(processed_logs[0].args)) try: logging.info( f"refunded_wei={processed_logs[0].args['refundedWei']}") log("SUCCESS", "green") except Exception as e: logging.error(f"E: Transaction is reverted. {e}") except QuietExit: pass except Exception as e: print_tb(e) sys.exit(1)
def size(key, mime_type, folder_name, gdrive_info, results_folder_prev, code_hashes, is_cached): source_code_key = None size_to_download = 0 if "folder" not in mime_type: raise Exception try: output = get_file_id(key) log(f"==> data_id=[magenta]{key}") log(output, "bold green") data_files_id = fetch_grive_output(output, "meta_data.json") # key for the source_code elimination output*.tar.gz files source_code_key = fetch_grive_output(output, f"{folder_name}.tar.gz") if not data_files_id: raise Exception cmd = [ "gdrive", "download", "--recursive", data_files_id, # first id is meta_data "--force", "--path", results_folder_prev, ] output = subprocess_call(cmd, 10) print(output) cmd = [ "gdrive", "info", "--bytes", source_code_key, "-c", env.GDRIVE_METADATA, ] gdrive_info = subprocess_call(cmd, 10) except Exception as e: print_tb(e) # TODO: gdrive list --query "sharedWithMe" raise e md5sum = get_file_info(gdrive_info, "Md5sum") _source_code_hash = code_hashes[0].decode("utf-8") if md5sum != _source_code_hash: # checks md5sum obtained from gdrive and given by the user raise Exception( f"E: md5sum does not match with the provided data {source_code_key}" ) log(f"SUCCESS on folder={md5sum}", "bold green") byte_size = int(get_file_info(gdrive_info, "Size")) log(f"## code_hashes[0] == {_source_code_hash} | size={byte_size} bytes") if not is_cached[code_hashes[0].decode("utf-8")]: size_to_download += byte_size try: meta_data = get_data_key_ids(results_folder_prev) except Exception as e: raise e data_key_dict = {} if len(meta_data.items()) > 1: idx = 0 for (k, v) in meta_data.items(): if idx == 0: # first item is for the source-code itself _key = str(v) output = get_file_id(_key) data_key = fetch_grive_output(output, f"{k}.tar.gz") cmd = [ "gdrive", "info", "--bytes", data_key, "-c", env.GDRIVE_METADATA ] gdrive_info = subprocess_call(cmd, 10) log(f" * gdrive_info for [green]{k}[/green]:") log(gdrive_info, "bold yellow") idx += 1 else: # should start from the first index _key = str(v) try: output = get_file_id(_key) data_key = fetch_grive_output(output, f"{k}.tar.gz") cmd = [ "gdrive", "info", "--bytes", data_key, "-c", env.GDRIVE_METADATA ] gdrive_info = subprocess_call(cmd, 10) except Exception as e: raise e md5sum = get_file_info(gdrive_info, _type="Md5sum") log(f" * gdrive_info for [green]{k}[/green]:") log(gdrive_info, "bold yellow") given_code_hash = code_hashes[idx].decode("utf-8") log(f"==> given_code_hash={given_code_hash} idx={idx}") if md5sum != given_code_hash: # checks md5sum obtained from gdrive and given by the user raise Exception( f"E: md5sum does not match with the provided data{br(idx)}\n" f"md5sum={md5sum} | given={given_code_hash}") data_key_dict[md5sum] = data_key _size = int(get_file_info(gdrive_info, "Size")) log(f"==> code_hashes{br(idx)} == {code_hashes[idx].decode('utf-8')} size={_size} bytes" ) byte_size += _size if not is_cached[code_hashes[idx].decode("utf-8")]: size_to_download += _size if bool(data_key_dict): data_link_file = f"{results_folder_prev}/meta_data_link.json" with open(data_link_file, "w") as f: json.dump(data_key_dict, f) else: raise Exception("E: Something is wrong. data_key_dict is empty") output = byte_to_mb(size_to_download) logging.info( f"Total_size={byte_size} bytes | size to download={size_to_download} bytes => {output} MB" ) return output, data_key_dict, source_code_key
def eudat_get_share_token(self, f_id): """Check key is already shared or not.""" folder_token_flag = {} if not os.path.isdir(self.private_dir): raise Exception(f"{self.private_dir} does not exist") share_id_file = f"{self.private_dir}/{self.job_key}_share_id.json" for idx, source_code_hash_text in enumerate( self.code_hashes_to_process): if self.cloudStorageID[idx] != StorageID.NONE: folder_name = source_code_hash_text self.folder_type_dict[folder_name] = None source_fn = f"{folder_name}/{folder_name}.tar.gz" if os.path.isdir(env.OWNCLOUD_PATH / f"{folder_name}"): log(f"## eudat shared folder({folder_name}) is already accepted and " "exists on the eudat's mounted folder") if os.path.isfile(f"{env.OWNCLOUD_PATH}/{source_fn}"): self.folder_type_dict[folder_name] = "tar.gz" else: self.folder_type_dict[folder_name] = "folder" try: info = config.oc.file_info(f"/{source_fn}") logging.info("shared folder is already accepted") size = info.attributes["{DAV:}getcontentlength"] folder_token_flag[folder_name] = True log(f"==> index={br(idx)}: /{source_fn} => {size} bytes") except: log(f"warning: shared_folder{br(source_code_hash_text, 'green')} is not accepted yet" ) folder_token_flag[folder_name] = False try: # TODO: add pass on template data = read_json(share_id_file) if isinstance(data, dict) and bool(data): self.share_id = data except: pass if self.share_id: log("==> share_id:") log(self.share_id, "bold") for share_key, value in self.share_id.items( ): # there is only single item try: # TODO: if added before or some do nothing if Ebb.mongo_broker.add_item_share_id(share_key, value["share_id"], value["share_token"]): # adding into mongoDB for future usage log(f"#> [green]{share_key}[/green] is added into mongoDB {ok()}" ) except Exception as e: print_tb(e) log(f"E: {e}") for attempt in range(config.RECONNECT_ATTEMPTS): try: share_list = config.oc.list_open_remote_share() break except Exception as e: log(f"E: Failed to list_open_remote_share eudat [attempt={attempt}]" ) print_tb(e) time.sleep(1) else: break else: return False self.accept_flag = 0 for idx, source_code_hash_text in enumerate( self.code_hashes_to_process): if self.cloudStorageID[idx] == StorageID.NONE: self.accept_flag += 1 else: # folder should not be registered data on the provider #: search_token is priority if not self.search_token(f_id, share_list, source_code_hash_text): try: share_key = f"{source_code_hash_text}_{self.requester_id[:16]}" shared_id = Ebb.mongo_broker.find_shareid_item( key=share_key) self.share_id[share_key] = { "share_id": shared_id["share_id"], "share_token": shared_id["share_token"], } self.accept_flag += 1 except Exception as e: if "warning: " not in str(e): log(f"E: {e}") else: log(str(e)) if folder_token_flag[folder_name] and bool( self.share_id): self.accept_flag += 1 else: self.search_token(f_id, share_list, folder_name) if self.accept_flag is len(self.code_hashes): break else: if self.accept_flag is len(self.code_hashes): logging.info("shared token already exists on mongoDB") # else: # raise Exception(f"E: could not find a shared file. Found ones are:\n{self.share_id}") if bool(self.share_id): with open(share_id_file, "w") as f: json.dump(self.share_id, f) else: raise Exception("E: share_id is empty")