def commit_changes(path):
with cd(path):
repo = git.Repo(".", search_parent_directories=True)
try:
output = run(["ls", "-l", ".git/refs/heads"])
except Exception as e:
raise Exception("E: Problem on git.commit_changes()") from e
if output == "total 0":
logging.warning("There is no first commit")
else:
changed_files = [item.a_path for item in repo.index.diff(None)]
if len(changed_files) > 0:
log("==> adding changed files:")
for _file in changed_files:
log(_file, "bold")
repo.git.add(A=True)
if len(repo.index.diff("HEAD")) == 0:
log(f"==> {path}\n is committed with the given changes using git"
)
try:
add_all(repo)
except Exception as e:
log(f"E: {e}")
raise e
def submit_slurm_job(self, job_core_num, sbatch_file_path):
"""Slurm submits job.
* Real mode -n is used.
* For Emulator-mode -N use 'sbatch -c'
* cmd: sudo su - $requester_id -c "cd $results_folder && firejail --noprofile \
sbatch -c$job_core_num $results_folder/${job_key}*${index}.sh --mail-type=ALL
"""
for _attempt in range(10):
try:
cmd = f'sbatch -n {job_core_num} "{sbatch_file_path}" --mail-type=ALL'
with cd(self.results_folder):
try:
job_id = _run_as_sudo(env.SLURMUSER, cmd, shell=True)
return job_id
except Exception as e:
if "Invalid account" in str(e):
remove_user(env.SLURMUSER)
add_user_to_slurm(env.SLURMUSER)
job_id = _run_as_sudo(env.SLURMUSER,
cmd,
shell=True)
time.sleep(
1) # wait 1 second for slurm idle core to be updated
except Exception as e:
print_tb(e)
slurm.remove_user(self.requester_id)
slurm.add_user_to_slurm(self.requester_id)
raise Exception("E: sbatch could not submit the job")
def is_repo(folders):
for folder in folders:
if not isinstance(folder, bytes):
with cd(folder):
if not is_initialized(folder):
log(f"warning: .git does not exits in {folder}. Applying: git init ",
end="")
run(["git", "init", "--initial-branch=master"])
log(ok())
def initialize_check(path):
"""Validate if .git/ folder exist within the target folder."""
with cd(path):
if not is_initialized(path):
try:
log(f"## git_repo={path}")
log("Creating an empty Git repository ", end="")
run(["git", "init", "--initial-branch=master"])
log(ok())
add_all()
except Exception as e:
log(f"E: {e}")
raise e
def eudat_download_folder(self, results_folder_prev, folder_name):
"""Download corresponding folder from the EUDAT.
Always assumes job is sent as .tar.gz file
"""
# TODO: check hash of the downloaded file is correct or not
cached_tar_file = f"{results_folder_prev}/{folder_name}.tar.gz"
log("#> downloading [green]output.zip[/green] for:", end="")
log(f"{folder_name} => {cached_tar_file} ", "bold")
key = folder_name
share_key = f"{folder_name}_{self.requester_id[:16]}"
for attempt in range(1):
try:
log("## Trying [blue]wget[/blue] approach...")
token = self.share_id[share_key]["share_token"]
if token:
download_fn = f"{cached_tar_file.replace('.tar.gz', '')}_{self.requester_id}.download"
cmd = [
"wget",
"-O",
download_fn,
"-c",
f"https://b2drop.eudat.eu/s/{token}/download",
"-q",
"--show-progres",
"--progress=bar:force",
]
log(" ".join(cmd), is_code=True, color="yellow")
run(cmd)
with cd(results_folder_prev):
run(["unzip", "-o", "-j", download_fn])
_remove(download_fn)
self.tar_downloaded_path[folder_name] = cached_tar_file
log(f"## download file from eudat {ok()}")
return
except:
log("E: Failed to download eudat file via wget.\nTrying config.oc.get_file() approach..."
)
if config.oc.get_file(f"/{key}/{folder_name}.tar.gz",
cached_tar_file):
self.tar_downloaded_path[folder_name] = cached_tar_file
log(ok())
return
else:
logging.error(
f"E: Something is wrong, oc could not retrieve the file [attempt:{attempt}]"
)
raise Exception("Eudat download error")
def diff_patch(path: Path, source_code_hash, index, target_path):
"""Apply diff patch.
"git diff HEAD" for detecting all the changes:
Shows all the changes between the working directory and HEAD (which includes changes in the index).
This shows all the changes since the last commit, whether or not they have been staged for commit
or not.
"""
is_file_empty = False
with cd(path):
log(f"==> Navigate to {path}")
"""TODO
if not is_initialized(path):
upload everything, changed files!
"""
repo = git.Repo(".", search_parent_directories=True)
try:
repo.git.config("core.fileMode",
"false") # git config core.fileMode false
# first ignore deleted files not to be added into git
run([env.BASH_SCRIPTS_PATH / "git_ignore_deleted.sh"])
head_commit_id = repo.rev_parse("HEAD")
sep = "~" # separator in between the string infos
patch_name = f"patch{sep}{head_commit_id}{sep}{source_code_hash}{sep}{index}.diff"
except:
return False
patch_upload_name = f"{patch_name}.gz" # file to be uploaded as zip
patch_file = f"{target_path}/{patch_upload_name}"
logging.info(f"patch_path={patch_upload_name}")
try:
repo.git.add(A=True)
diff_and_gzip(patch_file)
except:
return False
time.sleep(0.25)
if is_gzip_file_empty(patch_file):
log("==> Created patch file is empty, nothing to upload")
with suppress(Exception):
os.remove(patch_upload_name)
os.remove(patch_file)
is_file_empty = True
return patch_upload_name, patch_file, is_file_empty
def upload_results(encoded_share_token, output_file_name, path, max_retries=1):
"""Implement wrapper for the _upload_results function."""
with cd(path):
for _ in range(max_retries):
p, output, error = _upload_results(encoded_share_token, output_file_name)
if error:
log(error)
if "warning: Couldn't read data from file" in error:
raise Exception("E: EUDAT repository did not successfully uploaded")
if p.returncode != 0 or "<d:error" in output:
log("E: EUDAT repository did not successfully uploaded")
log(f" curl is failed. {p.returncode} => {br(error)} {output}")
time.sleep(1) # wait 1 second for next step retry to upload
else: # success on upload
return True
raise Exception(f"Upload results into cloud failed after {max_retries} tries")
def apply_patch(git_folder, patch_file, is_gpg=False):
"""Apply git patch.
output = repo.git.apply("--reject", "--whitespace=fix",
"--ignore-space-change", "--ignore-whitespace", "--verbose", patch_file)
__ https://stackoverflow.com/a/15375869/2402577
"""
if is_gpg:
cfg.ipfs.decrypt_using_gpg(patch_file)
with cd(git_folder):
base_name = path_leaf(patch_file)
log(f"==> [magenta]{base_name}")
# folder_name = base_name_split[2]
#
# base_name_split = base_name.split("_")
# git_hash = base_name_split[1]
# run(["git", "checkout", git_hash])
# run(["git", "reset", "--hard"])
# run(["git", "clean", "-f"])
# echo "\n" >> patch_file.txt seems like fixing it
#
# with open(patch_file, "a") as myfile:
# myfile.write("\n")
cmd = [
"git",
"apply",
"--reject",
"--whitespace=fix",
"--ignore-space-change",
"--ignore-whitespace",
"--verbose",
patch_file,
] # ,is_quiet=True,
cmd_summary = cmd.copy()
cmd_summary.insert(3, "--summary")
output = run(cmd_summary)
log(output)
output = run(cmd)
log(output)
def is_initialized(path) -> bool:
"""Check whether given the path is initialized with git.
__ https://stackoverflow.com/a/16925062/2402577
"""
with cd(path):
try:
*_, output, err = popen_communicate(
["git", "rev-parse", "--is-inside-work-tree"]) # noqa
if output == "true":
#: checks is the give path top git folder
git.Repo(".", search_parent_directories=False)
return True
except InvalidGitRepositoryError as e:
log(f"warning: InvalidGitRepositoryError at path {e}")
return False
except Exception as e:
log(f"warning: {e}")
return False
return output == "true"