def connect(remote, debug=False, progress_bar=None):
attempts = 0
pass_required = False
prompt_str = ""
passwd = None
if remote.auth_type == "password":
prompt_str = "Password ({}):".format(remote.name)
pass_required = True
elif remote.passphrase_required():
prompt_str = "Key passphrase ({}):".format(remote.name)
pass_required = True
while attempts < 3 or attempts == 0:
try:
if pass_required:
_printer.print_msg(prompt_str, "input", end='')
passwd = getpass.getpass("")
if progress_bar is not None:
remote.connect(passwd, progress_callback=progress_bar.callback)
else:
remote.connect(passwd)
except paramiko.ssh_exception.AuthenticationException as error:
attempts += 1
if pass_required:
if attempts < 3:
_printer.print_msg("Authentication failed", "warning")
continue
else:
raise Exception(str(error).rstrip('.') + " (3 attempts)")
raise
break
def remote_init_action(args):
remote_name = args.remote
study_path = os.path.abspath('.')
study_name = os.path.basename(study_path)
with action_error_handler(args.debug):
r = get_remote(study_path, remote_name)
connect(r, debug=args.debug)
r.init_remote()
_printer.print_msg("Done.", "info")
def print_tree_action(args):
study_path = os.path.abspath('.')
study_name = os.path.basename(study_path)
with action_error_handler(args.debug):
study = Study(study_name, study_path)
_printer.print_msg("Printing param tree...", "info")
_printer.indent_level = 1
study.param_file.print_tree()
_printer.indent_level = 0
_printer.print_msg("Done.", "info")
def action_error_handler(debug):
try:
yield
except Exception as error:
if debug:
raise
else:
_printer.indent_level = 0
_printer.print_msg(str(error), "error")
_printer.print_msg("Aborting...", "error")
sys.exit(1)
def generate_action(args):
study_path = os.path.abspath('.')
study_name = os.path.basename(study_path)
with action_error_handler(args.debug):
study = Study(study_name, study_path)
sb = StudyGenerator(study,
short_name=args.shortname,
build_once=args.build_once,
keep_onerror=args.keep_on_error,
abort_undefined=args.abort_undefined)
r = None
if args.local_remote is not None:
r = get_remote(study_path, args.local_remote)
sb.generate_cases(r)
_printer.print_msg("Done.", "info")
def postproc_action(args):
study_path = os.path.abspath('.')
study_name = os.path.basename(study_path)
with action_error_handler(args.debug):
study = Study(study_name, study_path)
study.load()
try:
# Insert study path to load postproc functions
sys.path.insert(0, study_path)
import postproc
except Exception as err:
raise
raise Exception("File 'postproc.py' not found in study directory.")
create_results_table(postproc.POSTPROC_TABLE_FIELDS, study)
_printer.indent_level = 0
_printer.print_msg("Done.", "info")
def load(self):
cases = []
try:
with open(self.file_path, 'r') as rfile:
json_data = rfile.read()
json_data = json.loads(json_data)
except IOError as e:
raise Exception("Problem opening 'cases.info' file - %s." %
e.strerror)
for case_dict in json_data["cases"]:
c = Case()
c.init_from_dict(case_dict)
cases.append(c)
self.loaded = True
try:
params = json_data["params"]
except KeyError:
_printer.print_msg(
"Missing 'params' section in 'cases.info' file. ", "warning")
params = []
return {"cases": cases, "params": params}
def delete_action(args):
study_path = os.path.abspath('.')
study_name = os.path.basename(study_path)
with action_error_handler(args.debug):
study = Study(study_name, study_path, load_param_file=False)
study.load()
_printer.print_msg("Selected %d cases to delete..." % study.nof_cases,
"info")
if args.yes:
opt = 'y'
else:
_printer.print_msg("Are you sure to delete?[Y,y]: ",
"input",
end="")
opt = raw_input("")
if opt in ['y', 'Y']:
_printer.print_msg("Deleting study files...", "info")
study.delete()
else:
_printer.print_msg("Delete aborted.", "info")
_printer.print_msg("Done.", "info")
def get_remote(study_path, remote_name_in):
# _printer.print_msg("Testing connection to remote '%s'..." % r.name, end='')
# r.available()
# print "OK"
remotes = RemotesFile(study_path)
remotes.load()
r = remote.Remote()
# Set to "default" if args.remote is None
if remote_name_in == None:
remote_name = remotes.default_remote
_printer.print_msg("Using default remote '{}'...".format(remote_name))
else:
remote_name = remote_name_in
try:
remote_yaml = remotes[remote_name]
except KeyError:
raise Exception(
"Remote '{}' not found in 'remotes.yaml'.".format(remote_name))
r.configure(remote_name, remote_yaml)
return r
def _create_instance(self, instance_name, instance, local_remote=None):
_printer.print_msg("Creating case '%s'..." % instance_name,
verbose=True,
end="")
casedir = os.path.join(self.study.path, instance_name)
studydir = os.path.dirname(casedir)
# try:
# shutil.copytree(self.template_path, casedir)
# except Exception:
# return instance_name
shutil.copytree(self.template_path, casedir)
if local_remote is not None:
local_remote_path = os.path.join(
self.study.path, "submit.{}.sh".format(local_remote.name))
shutil.copy(local_remote_path, os.path.join(casedir, "submit.sh"))
try:
# self._create_instance_infofile(instance)
# Create paths for files listed for replace params on them
file_paths = []
for path in self.study.param_file["FILES"]:
for f in path["files"]:
p = os.path.join(
os.path.join(self.study.path, instance_name),
path["path"])
p = os.path.join(p, f)
file_paths.append(p)
# Add paramate specific params
params = {
"PARAMATE-CN": instance_name,
"PARAMATE-SN": self.study.param_file["STUDY"]["name"],
"PARAMATE-CD": casedir,
"PARAMATE-SD": studydir
}
if local_remote is not None:
file_paths.append(os.path.join(casedir, "submit.sh"))
params.update({"PARAMATE-RWD": local_remote.workdir})
params.update({"PARAMATE-REMOTE": local_remote.name})
params.update(instance)
replace_placeholders(file_paths, params, self.abort_undefined)
if not self.build_once:
# Force execution permissions to 'build.sh'
_printer.print_msg("Building...",
msg_type="unformated",
verbose=True,
end="")
build_script_path = os.path.join(casedir, "build.sh")
os.chmod(
build_script_path, stat.S_IXUSR
| stat.S_IMODE(os.lstat(build_script_path).st_mode))
self.execute_build_script(build_script_path)
_printer.print_msg("Done.",
verbose=True,
msg_type="unformated")
except Exception:
if not self.keep_onerror:
shutil.rmtree(casedir)
raise
return instance_name
def print_tree(self):
root = self.sections["PARAMS-MULTIVAL"].tree
# print ""
_printer.print_msg("", "blank")
for l in str(RenderTree(root,
AsciiStyle()).by_attr("label")).split('\n'):
_printer.print_msg(l)
_printer.print_msg("", "blank")
def _upload(self,
remote,
name,
base_path,
upload_cases,
keep_targz=False,
force=False):
if not remote.remote_dir_exists(remote.workdir):
raise Exception(
"Remote work directory '%s' do not exists. Use 'remote-init' command to create it."
% remote.workdir)
remotedir = os.path.join(remote.workdir, name)
_printer.print_msg("Checking remote state...")
for case in upload_cases:
remote_casedir = os.path.join(remotedir, case)
if remote.remote_dir_exists(remote_casedir) and not force:
raise RemoteDirExists("Study '%s' - Case directory '%s' already exists in remote '%s'."\
% (self.study.name, case, remote.name))
upload_files = upload_cases + self.DEFAULT_UPLOAD_FILES
_printer.print_msg("Compressing study...")
tar_name = self._compress(name, base_path, upload_files)
upload_src = os.path.join(self.tmpdir, tar_name)
upload_dest = remote.workdir
remote.upload(upload_src, upload_dest)
extract_src = os.path.join(upload_dest, tar_name)
extract_dest = upload_dest
_printer.print_msg("Extracting study in remote...")
try:
out = remote.command(
"tar -xzf %s --directory %s --warning=no-timestamp" %
(extract_src, extract_dest))
# For older versions of tar. Not sure how they will handle the timestamp issue though.
except Exception as error:
try:
out = remote.command("tar -xzf %s --directory %s" %
(extract_src, extract_dest))
except Exception:
raise Exception(
"Unable to decompress '%s.tar.gz' in remote. Check version of 'tar' command in the remote."
% tar_name)
_printer.print_msg("Cleaning...")
os.remove(upload_src)
if not keep_targz:
out = remote.command("rm -f %s" % extract_src)
def generate_cases(self, local_remote=None):
self._generate_instances()
# Check if build.sh has to be run before generating the instances
nof_instances = len(self.instances)
_printer.print_msg("Generating {} cases...".format(nof_instances))
if not os.path.exists(self.template_path):
raise Exception("Cannot find 'template' directory!")
if os.path.exists(self.build_script_path):
if self.build_once:
_printer.print_msg("Building once from 'build.sh'...")
# Force execution permissions to 'build.sh'
os.chmod(
self.build_script_path, stat.S_IXUSR
| stat.S_IMODE(os.lstat(self.build_script_path).st_mode))
self.execute_build_script(self.build_script_path)
else:
if self.build_once:
raise Exception(
"No 'build.sh' script found but '--build-once' option was specified."
)
for instance_id, instance in enumerate(self.instances):
# Resolve generators
instance.resolve_params()
multival_params = self._get_multival_params(instance)
singleval_params = self._get_singleval_params(instance)
instance_name = self._instance_directory_string(
instance_id, multival_params, nof_instances, self.short_name)
self._create_instance(instance_name,
instance,
local_remote=local_remote)
self.study.add_case(instance_name,
multival_params,
singleval_params,
short_name=self.short_name,
local_remote=local_remote)
self.study.save()
_printer.print_msg("Success: Created %d cases." % nof_instances)
def state_action(args,
action,
allowed_states,
action_func,
output_handler,
action_progress_bar=None):
study_path = os.path.abspath('.')
study_name = os.path.basename(study_path)
study = Study(study_name, study_path)
study.load()
if args.selector is None:
case_selector = "*"
else:
case_selector = args.selector
with action_error_handler(args.debug):
cases_idx = decode_case_selector(case_selector, study.nof_cases)
study.set_selection(cases_idx)
remote_cases, no_remote_cases = get_cases_byremote(cases_idx,
study,
allowed_states,
remote=args.remote)
nof_remotes = len(remote_cases.keys())
nof_selected_cases = len(cases_idx)
nof_no_remote_cases = len(no_remote_cases)
nof_remote_cases = nof_selected_cases - nof_no_remote_cases
# Print info about the state of selected cases
_printer.print_msg(
"Selected {} cases for action '{}':".format(nof_selected_cases,
action), "info")
_printer.indent_level = 1
if nof_no_remote_cases:
_printer.print_msg(
"- {} cases with no associated remote...".format(
nof_no_remote_cases), "info")
if nof_remote_cases > 0:
_printer.print_msg(
"- {} cases in ({} remotes)':".format(
len(cases_idx) - len(no_remote_cases), nof_remotes), "info")
for counter, (remote_name,
remote_info) in enumerate(remote_cases.items()):
_printer.indent_level = 2
_printer.print_msg(
"[{}] '{}': {} cases selected".format(
counter + 1, remote_name, remote_info["nof_selected"]),
"info")
for status, case_info in remote_info["cases"].items():
if case_info["nof"] > 0:
_printer.indent_level = 3
_printer.print_msg("{}: {}".format(status,
case_info["nof"]))
nof_excluded_cases = remote_info["nof_selected"] - remote_info[
"nof_valid"]
if nof_excluded_cases > 0:
_printer.print_msg(
"Found {} cases with a state not in {}. They will be ignored."
.format(nof_excluded_cases, allowed_states))
# The cases which has no remote are the ones to upload
if action == "upload":
with action_error_handler(args.debug):
_printer.indent_level = 0
r = get_remote(study_path, args.remote)
remote_cases = {}
remote_cases[r.name] = {
"nof_valid": len(no_remote_cases),
"valid_cases": no_remote_cases
}
# Iterate over remotes
for remote_name, remote_info in remote_cases.items():
# If not valid cases to perform action go to next remote
if remote_info["nof_valid"] == 0:
continue
# Continue to the next remote if there are not cases to download
study.set_selection(remote_info["valid_cases"])
_printer.print_msg("", "blank")
remote_header = "[{}: '{}']".format(action.capitalize(), remote_name)
_printer.indent_level = 0
_printer.print_msg(remote_header, "info")
_printer.indent_level = 1
# Ask for confirmation
if args.yes:
opt = 'y'
else:
_printer.print_msg("Perform action '{}' on '{}'?[Y,y]: ".format(
action, remote_name),
"input",
end="")
opt = raw_input("")
if not opt in ['y', 'Y']:
_printer.print_msg("Skipping...")
_printer.print_msg("", "blank")
continue
with action_error_handler(args.debug):
r = get_remote(study_path, remote_name)
connect(r, debug=args.debug, progress_bar=action_progress_bar)
sm = remote.StudyManager(study)
# The update affect all cases not only the selection
sm.update_status(r)
# Upload is not affected by update_status()
if action == "upload":
valid_cases = remote_cases[remote_name]["valid_cases"]
else:
remote_cases_updated, no_remote_cases = get_cases_byremote(
cases_idx, study, allowed_states, remote=remote_name)
valid_cases = remote_cases_updated[remote_name]["valid_cases"]
output = ""
if valid_cases:
_printer.print_msg(
"Performing action '{}' on {} cases...".format(
action, len(valid_cases)), "info")
study.set_selection(valid_cases)
output = action_func(sm, r)
output_handler(output)
else:
_printer.print_msg("No jobs running found. Skipping...")
r.close()
_printer.print_msg("", "blank")
_printer.indent_level = 0
if sum([r["nof_valid"] for r in remote_cases.values()]) == 0:
_printer.print_msg("Nothing to do for action: '{}'".format(action),
"info")
_printer.print_msg("Done.", "info")
def clean_action(args):
study_path = os.path.abspath('.')
with action_error_handler(args.debug):
r = get_remote(study_path, args.remote)
connect(r, debug=args.debug)
_printer.print_msg("Done.", "info")
def init_remote(self):
if not self.remote_dir_exists(self.workdir):
time.sleep(1)
self.command("mkdir -p %s" % self.workdir)
_printer.print_msg("Remote workdir created.")
else:
raise Exception("Directory %s already exists in remote '%s'." %
(self.workdir, self.name))
cmd_not_available = False
_printer.print_msg("Checking remote dependencies...")
if not self.cmd_avail("qsub"):
_printer.print_msg(
"Warning: Command 'qsub' not available in '%s'." % self.name,
ignore_quiet=True)
cmd_not_available = True
if not self.cmd_avail("qstat"):
_printer.print_msg(
"Warning: Command 'qstat' not available in '%s'." % self.name,
ignore_quiet=True)
cmd_not_available = True
if not self.cmd_avail("qdel"):
_printer.print_msg(
"Warning: Command 'qdel' not available in '%s'." % self.name,
ignore_quiet=True)
cmd_not_available = True
if cmd_not_available:
_printer.print_msg("Info: Sometimes it is necessary to add the path where the\n" +\
" binaries qsub/qstat/qdel are located in the remote to the\n" +\
" ~/.bashrc or ~/.cshrc files.", ignore_quiet=True)
def output_handler_job_status(output):
_printer.indent_level = 2
_printer.print_msg("")
for l in output:
_printer.print_msg(l, end='')
def download(self, remote, force=False, compress_only=False):
remote_studydir = os.path.join(remote.workdir, self.study.name)
if not remote.remote_dir_exists(remote_studydir):
raise Exception("Study '%s' does not exists in remote '%s'." %
(self.study.name, remote.name))
compress_dirs = ""
cases_regexp = self._cases_regexp()
for path in self.study.param_file["DOWNLOAD"]:
include_list = []
exclude_list = []
path_name = path["path"]
# TODO: Move checks of params.yaml to the Sections checkers in PARAMATE.py
include_exists = "include" in path
exclude_exists = "exclude" in path
path_wildcard = os.path.join(cases_regexp, path["path"])
if exclude_exists and include_exists:
raise Exception("Both 'exclude' and 'include' defined for download path '%s'."\
% path["path"])
else:
if include_exists:
include_list = [
os.path.join(path_wildcard, f) for f in path["include"]
]
compress_dirs += " " + " ".join(include_list)
elif exclude_exists:
exclude_list = path["exclude"]
for f in exclude_list:
compress_dirs += " --exclude=%s" % f
compress_dirs += " " + path_wildcard
else:
compress_dirs += " " + path_wildcard
compress_src = os.path.join(remote_studydir,
self.study.name + ".tar.gz")
tar_cmd = "tar -czf %s %s" % (compress_src, compress_dirs)
#TODO: REMOVE THIS
force = True
_printer.print_msg("Compressing study...")
try:
if force:
tar_cmd += " --ignore-failed-read"
remote.command("cd %s && %s" % (remote_studydir, tar_cmd) ,\
fail_on_error=False, timeout=60)
except Exception as error:
if remote.command_status != 0:
remote.command("cd %s && rm -f %s" %
(remote_studydir, compress_src),
timeout=60)
raise Exception(error)
if not compress_only:
remote.download(compress_src, self.study.path)
_printer.print_msg("Decompressing study...")
tar_path = os.path.join(self.study.path,
self.study.name) + ".tar.gz"
self._decompress(tar_path, self.study.path)
for case in self.study.case_selection:
case.status = "DOWNLOADED"
self.study.save()
_printer.print_msg("Cleaning...")
remote.command("cd %s && rm -f %s" %
(remote_studydir, compress_src),
timeout=60)
def output_handler_job_delete(output):
_printer.print_msg(
"Marked for deletion {} cases. Waiting...".format(output))