def terminate_task(self, identifier) -> TaskStatus:
from time import sleep
try:
url = self.url_abort(identifier)
data = parse.urlencode({}).encode()
req = request.Request(
url, data=data) # this will make the method "POST"
r = request.urlopen(req)
data = r.read()
Logger.debug("Janis has issued abort request to Cromwell: " +
str(data))
taskstatus = self.poll_task(identifier)
while taskstatus not in TaskStatus.final_states():
Logger.debug(
f"Task status ({taskstatus}) has not moved to final state after aborting..."
)
sleep(1)
taskstatus = self.poll_task(identifier)
Logger.info(
f"Workflow with Cromwell identifier ({identifier} has been terminated ({taskstatus})."
)
self.progress_callbacks.pop(identifier)
except Exception as e:
raise Exception(
f"Failed to abort workflow with id = {identifier} :: {e}")
return TaskStatus.ABORTED
def stop_engine(self):
if not self.is_started:
return Logger.debug(
"Cromwell has already shut down, skipping shut down request")
if self._logger:
self._logger.terminate()
self.should_stop = True
if self._timer_thread:
self._timer_thread.set()
if not self.process_id:
self.is_started = False
Logger.info("Janis isn't managing Cromwell, skipping the shutdown")
return
Logger.info("Stopping cromwell")
if self.process_id:
try:
process = os.getpgid(int(self.process_id))
os.killpg(process, signal.SIGTERM)
Logger.info("Stopped cromwell")
except Exception as e:
# can't do
Logger.warn("Couldn't stop Cromwell process: " + str(e))
else:
Logger.warn(
"Couldn't stop Cromwell process as Janis wasn't managing it")
Logger.debug("Setting 'cromwell.is_started' to False")
self.is_started = False
def run_delete_database_script(self, execution_dir: str):
try:
import subprocess, os
from janis_assistant.management.envvariables import EnvVariables
file_path = os.getenv(EnvVariables.db_script_generator_cleanup)
if file_path is None:
raise Exception(
f"Couldn't delete generated database credentials as couldn't find value in env var '{EnvVariables.db_script_generator_cleanup}'"
)
Logger.debug(
f"Found path '{EnvVariables.db_script_generator_cleanup}' to delete database credentials"
)
# if not os.path.exists(file_path):
# raise Exception(f"Couldn't locate script '{file_path}' to execute")
val = collect_output_from_command(f"{file_path} {execution_dir}",
stderr=Logger.guess_log,
shell=True)
if val is not None and len(val) > 0:
Logger.info(
f"Successfully deleted DB credentials and received message: {val}"
)
else:
Logger.info("Deleted credentials with rc=0")
except Exception as e:
Logger.warn(
f"Failed to delete database configuration details for execution directory '{execution_dir}': "
+ repr(e))
def get(self, type_name, tag: Optional[str]) -> Optional[T]:
if type_name not in self.registry:
return None
tagged_objs = self.registry[type_name]
versions_without_default = set(tagged_objs.keys())
if self.default_tag in versions_without_default:
versions_without_default.remove(self.default_tag)
Logger.debug(
f"'{type_name}' has {len(versions_without_default)} versions ({', '.join(versions_without_default)})"
)
if tag is None or tag == self.default_tag:
if self.default_tag in tagged_objs:
Logger.info(
f"Using the default tag for '{type_name}' from {len(versions_without_default)} version(s): {', '.join(versions_without_default)}"
)
return tagged_objs.get(self.default_tag)[0]
return None
if tag not in tagged_objs:
Logger.log(
"Found collection '{tool}' in registry, but couldn't find tag '{tag}'"
.format(tool=type_name, tag=tag))
return None
return tagged_objs[tag]
def get_workflow_metadatadb(execpath, wid, readonly=False):
connection = None
sqlpath = WorkflowDbManager.get_sql_path_base(execpath)
if not wid:
Logger.debug("Opening database connection to get wid from: " +
sqlpath)
try:
connection = sqlite3.connect(f"file:{sqlpath}?mode=ro",
uri=True)
except:
Logger.critical("Error when opening DB connection to: " +
sqlpath)
raise
wid = RunDbProvider(db=connection).get_latest()
if not wid:
raise Exception("Couldn't get WID in task directory")
retval = WorkflowMetadataDbProvider(sqlpath, wid, readonly=readonly)
if connection:
connection.close()
return retval
def cwl_type(self, has_default=False):
inner_types = [a.cwl_type(has_default=has_default) for a in self.subtypes]
try:
inner_types = list(set(inner_types))
except Exception as e:
Logger.debug(f"Error creating set from ({inner_types}): {e}")
if len(inner_types) == 1:
return inner_types[0]
return inner_types
def link_copy_or_fail(source: str, dest: str, force=False):
"""
Eventually move this to some generic util class
:param source: Source to link from
:param dest: Place to link to
:param force: Overwrite destination if it exists
:return:
"""
try:
to_copy = [(
LocalFileScheme.prepare_path(source),
LocalFileScheme.prepare_path(dest),
)]
while len(to_copy) > 0:
s, d = to_copy.pop(0)
# Check if path is Null/None
if not s:
continue
if not d:
continue
if os.path.exists(d) and force:
Logger.debug(f"Destination exists, overwriting '{d}'")
if os.path.isdir(d):
rmtree(d)
else:
os.remove(d)
Logger.log(f"Hard linking {s} → {d}")
if os.path.isdir(s):
os.makedirs(d, exist_ok=True)
for f in os.listdir(s):
to_copy.append((os.path.join(s, f), os.path.join(d,
f)))
continue
try:
os.link(s, d)
except FileExistsError:
Logger.critical(
"The file 'd' already exists. The force flag is required to overwrite."
)
except Exception as e:
Logger.warn("Couldn't link file: " + str(e))
# if this fails, it should error
Logger.log(f"Copying file {s} → {d}")
copyfile(s, d)
except Exception as e:
Logger.critical(
f"An unexpected error occurred when link/copying {source} -> {dest}: {e}"
)
def db_connection(self):
path = self.get_sql_path()
try:
if self.readonly:
Logger.debug(
"Opening database connection to in READONLY mode: " + path)
return sqlite3.connect(f"file:{path}?mode=ro", uri=True)
Logger.debug("Opening database connection: " + path)
return sqlite3.connect(path)
except:
Logger.critical("Error when opening DB connection to: " + path)
raise
def poll_task(self, identifier) -> Optional[TaskStatus]:
if self.error_message:
return TaskStatus.FAILED
url = self.url_poll(identifier=identifier)
try:
r = request.urlopen(url)
data = r.read()
res = json.loads(data.decode(
r.info().get_content_charset("utf-8")))
return cromwell_status_to_status(res["status"])
except Exception as e:
Logger.debug("Error polling Cromwell task:" + str(e))
return None
def get_config_from_script(self, execution_dir: str):
try:
import subprocess, os, json
from janis_assistant.management.envvariables import EnvVariables
from janis_assistant.engines.cromwell.cromwellconfiguration import (
CromwellConfiguration, )
file_path = os.getenv(EnvVariables.db_script_generator)
Logger.debug(
f"Found path '{EnvVariables.db_script_generator}' to generate database credentials"
)
if file_path is None:
raise Exception(
f"Couldn't get database credentials as couldn't find value in env var '{EnvVariables.db_script_generator}'"
)
# if not os.path.exists(file_path):
# raise Exception(f"Couldn't locate script '{file_path}' to execute")
try:
val = collect_output_from_command(
f"{file_path} {execution_dir}",
stderr=Logger.guess_log,
shell=True)
except Exception as e:
Logger.critical(
f"Failed to generate database credentials ({repr(e)})")
raise
d = json.loads(val)
Logger.debug("Received keys from database credentials script: " +
", ".join(d.keys()))
keys = {"username", "password", "database", "host"}
missing_keys = {k for k in keys if k not in d}
if len(missing_keys) > 0:
raise Exception(
"The script to generate database credentials was missing the keys: "
+ ", ".join(missing_keys))
return CromwellConfiguration.Database.mysql(
username=d["username"],
password=d["password"],
database=d["database"],
url=d["host"],
)
except Exception as e:
Logger.critical(
"Failed to get database configuration details from script: " +
repr(e))
raise
def start_from_paths(self, wid, source_path: str, input_path: str,
deps_path: str):
from janis_assistant.data.models.preparedjob import PreparedJob
jobfile = PreparedJob.instance()
self.taskmeta = {
"start": DateUtil.now(),
"status": TaskStatus.PROCESSING,
"jobs": {},
}
config: CWLToolConfiguration = self.config
if Logger.CONSOLE_LEVEL == LogLevel.VERBOSE:
config.debug = True
config.disable_color = True
# more options
if not config.tmpdir_prefix:
config.outdir = self.execution_dir + "/"
config.tmpdir_prefix = self.execution_dir + "/"
config.leave_tmpdir = True
if jobfile.call_caching_enabled:
config.cachedir = os.path.join(self.execution_dir, "cached/")
cmd = config.build_command_line(source_path, input_path)
Logger.debug("Running command: '" + " ".join(cmd) + "'")
process = subprocess.Popen(cmd,
stdout=subprocess.PIPE,
preexec_fn=os.setsid,
stderr=subprocess.PIPE)
self.taskmeta["status"] = TaskStatus.RUNNING
Logger.info("CWLTool has started with pid=" + str(process.pid))
self.process_id = process.pid
self._logger = CWLToolLogger(
wid,
process,
logfp=open(self.logfile, "a+"),
metadata_callback=self.task_did_update,
exit_function=self.task_did_exit,
)
return wid
def stringify_translated_workflow(wf):
try:
import black
try:
return black.format_str(wf,
mode=black.FileMode(line_length=82))
except black.InvalidInput:
Logger.warn(
"Check the generated Janis code carefully, as there might be a syntax error. You should report this error along with the workflow you're trying to generate from"
)
except ImportError:
Logger.debug(
"Janis can automatically format generated Janis code if you install black: https://github.com/psf/black"
)
return wf
def task_did_exit(self, logger: CWLToolLogger, status: TaskStatus):
Logger.debug("CWLTool fired 'did exit'")
self.taskmeta["status"] = status
self.taskmeta["finish"] = DateUtil.now()
self.taskmeta["outputs"] = logger.outputs
if status != TaskStatus.COMPLETED:
js: Dict[str, RunJobModel] = self.taskmeta.get("jobs")
for j in js.values():
if j.status != TaskStatus.COMPLETED:
j.status = status
if logger.error:
self.taskmeta["error"] = logger.error
for callback in self.progress_callbacks.get(logger.sid, []):
callback(self.metadata(logger.sid))
def start_from_paths(self, wid, source_path: str, input_path: str, deps_path: str):
print("TMP: " + os.getenv("TMPDIR"))
scale = ["--scale", str(self.scale)] if self.scale else []
loglevel = ["--logLevel=" + self.loglevel] if self.loglevel else []
cmd = ["toil-cwl-runner", "--stats", *loglevel, *scale, source_path, input_path]
Logger.debug("Running command: '" + " ".join(cmd) + "'")
process = subprocess.Popen(
cmd, stdout=subprocess.PIPE, preexec_fn=os.setsid, stderr=subprocess.PIPE
)
Logger.info("CWLTool has started with pid=" + str(process.pid))
for line in read_stdout(process):
if "Path to job store directory is" in line:
idx = line.index("Path to job store directory is")
Logger.critical("JOBSTORE DIR: " + line[idx + 1 :])
Logger.debug("toil: " + line)
print("finished")
def cp_from(
self,
source,
dest,
force=False,
report_progress: Optional[Callable[[float], None]] = None,
):
"""
Downloads a public blob from the bucket.
Source: https://cloud.google.com/storage/docs/access-public-data#code-samples
"""
self.check_if_has_gcp()
blob = self.get_blob_from_link(source)
size_mb = blob.size // (1024 * 1024)
Logger.debug(f"Downloading {source} -> {dest} ({size_mb} MB)")
blob.download_to_filename(dest)
print("Downloaded public blob {} from bucket {} to {}.".format(
blob.name, blob.bucket.name, dest))
def __init__(self, d: dict = None):
default = self.base()
d = d if d else {}
extra = "" if d is None else " from loaded config"
Logger.debug("Instantiating JanisConfiguration" + extra)
self.configdir = self.get_value_for_key(
d, JanisConfiguration.Keys.ConfigDir, default)
self.dbpath = os.path.join(self.configdir, "janis.db")
self.outputdir = self.get_value_for_key(
d, JanisConfiguration.Keys.OutputDir, default)
self.executiondir = self.get_value_for_key(
d, JanisConfiguration.Keys.ExecutionDir, default)
self.call_caching_enabled = JanisConfiguration.get_value_for_key(
d, self.Keys.CallCachingEnabled, default)
self.engine = self.get_value_for_key(d, JanisConfiguration.Keys.Engine,
default)
self.cromwell = JanisConfiguration.JanisConfigurationCromwell(
d.get(JanisConfiguration.Keys.Cromwell),
default.get(JanisConfiguration.Keys.Cromwell),
)
self.template = JanisConfiguration.JanisConfigurationTemplate(
d.get(JanisConfiguration.Keys.Template),
default.get(JanisConfiguration.Keys.Template),
)
self.recipes = JanisConfiguration.JanisConfigurationRecipes(
d.get(JanisConfiguration.Keys.Recipes),
default.get(JanisConfiguration.Keys.Recipes),
)
self.notifications = JanisConfiguration.JanisConfigurationNotifications(
d.get(JanisConfiguration.Keys.Notifications),
default.get(JanisConfiguration.Keys.Notifications),
)
self.environment = JanisConfiguration.JanisConfigurationEnvironment(
d.get(JanisConfiguration.Keys.Environment),
default.get(JanisConfiguration.Keys.Environment),
)
self.run_in_background = self.get_value_for_key(
d, JanisConfiguration.Keys.RunInBackground, default)
sp = self.get_value_for_key(d, JanisConfiguration.Keys.SearchPaths,
default)
self.searchpaths = sp if isinstance(sp, list) else [sp]
env_sp = EnvVariables.search_path.resolve(False)
if env_sp and env_sp not in self.searchpaths:
self.searchpaths.append(env_sp)
# Get's set by the template for now, but eventually we should be able to look it up
self.container = None
JanisConfiguration._managed = self
if self.template.template:
self.template.template.post_configuration_hook(self)
def translate(
self,
tool,
to_console=True,
tool_to_console=False,
with_resource_overrides=False,
to_disk=False,
write_inputs_file=True,
export_path=ExportPathKeywords.default,
should_validate=False,
should_zip=True,
merge_resources=False,
hints=None,
allow_null_if_not_optional=True,
additional_inputs: Dict = None,
max_cores=None,
max_mem=None,
max_duration=None,
with_container=True,
allow_empty_container=False,
container_override=None,
):
str_tool, tr_tools = None, []
if tool.type() == ToolType.Workflow:
tr_tool, tr_tools = self.translate_workflow(
tool,
with_container=with_container,
with_resource_overrides=with_resource_overrides,
allow_empty_container=allow_empty_container,
container_override=lowercase_dictkeys(container_override),
)
str_tool = self.stringify_translated_workflow(tr_tool)
elif isinstance(tool, CodeTool):
tr_tool = self.translate_code_tool_internal(
tool,
allow_empty_container=allow_empty_container,
container_override=lowercase_dictkeys(container_override),
)
str_tool = self.stringify_translated_tool(tr_tool)
else:
tr_tool = self.translate_tool_internal(
tool,
with_container=with_container,
with_resource_overrides=with_resource_overrides,
allow_empty_container=allow_empty_container,
container_override=lowercase_dictkeys(container_override),
)
str_tool = self.stringify_translated_tool(tr_tool)
tr_inp = self.build_inputs_file(
tool,
recursive=False,
merge_resources=merge_resources,
hints=hints,
additional_inputs=additional_inputs,
max_cores=max_cores,
max_mem=max_mem,
max_duration=max_duration,
)
tr_res = self.build_resources_input(tool, hints)
str_inp = self.stringify_translated_inputs(tr_inp)
str_tools = [(
"tools/" + self.tool_filename(t),
self.stringify_translated_workflow(tr_tools[t]),
) for t in tr_tools]
str_resources = self.stringify_translated_inputs(tr_res)
if to_console:
print("=== WORKFLOW ===")
print(str_tool)
if tool_to_console:
print("\n=== TOOLS ===")
[print(f":: {t[0]} ::\n" + t[1]) for t in str_tools]
print("\n=== INPUTS ===")
print(str_inp)
if not merge_resources and with_resource_overrides:
print("\n=== RESOURCES ===")
print(str_resources)
d = ExportPathKeywords.resolve(export_path,
workflow_spec=self.name,
workflow_name=tool.versioned_id())
fn_workflow = self.workflow_filename(tool)
fn_inputs = self.inputs_filename(tool)
fn_resources = self.resources_filename(tool)
if to_disk and write_inputs_file:
if not os.path.isdir(d):
os.makedirs(d)
with open(os.path.join(d, fn_inputs), "w+") as f:
Logger.log(f"Writing {fn_inputs} to disk")
f.write(str_inp)
Logger.log(f"Written {fn_inputs} to disk")
else:
Logger.log("Skipping writing input (yaml) job file")
if to_disk:
toolsdir = os.path.join(d, "tools")
if not os.path.isdir(toolsdir):
os.makedirs(toolsdir)
Logger.info(f"Exporting tool files to '{d}'")
with open(os.path.join(d, fn_workflow), "w+") as wf:
Logger.log(f"Writing {fn_workflow} to disk")
wf.write(str_tool)
Logger.log(f"Wrote {fn_workflow} to disk")
for (fn_tool, disk_str_tool) in str_tools:
with open(os.path.join(d, fn_tool), "w+") as toolfp:
Logger.log(f"Writing {fn_tool} to disk")
toolfp.write(disk_str_tool)
Logger.log(f"Written {fn_tool} to disk")
if not merge_resources and with_resource_overrides:
print("\n=== RESOURCES ===")
with open(os.path.join(d, fn_resources), "w+") as wf:
Logger.log(f"Writing {fn_resources} to disk")
wf.write(str_inp)
Logger.log(f"Wrote {fn_resources} to disk")
print(str_resources)
import subprocess
if should_zip:
Logger.debug("Zipping tools")
with Path(d):
FNULL = open(os.devnull, "w")
zip_result = subprocess.run(
["zip", "-r", "tools.zip", "tools/"], stdout=FNULL)
if zip_result.returncode == 0:
Logger.debug("Zipped tools")
else:
Logger.critical(str(zip_result.stderr.decode()))
if should_validate:
with Path(d):
Logger.info(f"Validating outputted {self.name}")
enved_vcs = [
(os.getenv(x[1:]) if x.startswith("$") else x)
for x in self.validate_command_for(
fn_workflow, fn_inputs, "tools/", "tools.zip")
]
cwltool_result = subprocess.run(enved_vcs)
if cwltool_result.returncode == 0:
Logger.info("Exported tool was validated by: " +
" ".join(enved_vcs))
else:
Logger.critical(str(cwltool_result.stderr))
return str_tool, str_inp, str_tools
def start_engine(self, additional_cromwell_options: List[str] = None):
from ...data.models.preparedjob import PreparedJob
job = PreparedJob.instance()
self._start_time = DateUtil.now()
self.timeout = job.cromwell.timeout or 10
if self.test_connection():
Logger.info("Engine has already been started")
return self
if self.connect_to_instance:
self.is_started = True
Logger.info(
"Cromwell environment discovered, skipping local instance")
return self
if self._process:
self.is_started = True
Logger.info(
f"Discovered Cromwell instance (pid={self._process}), skipping start"
)
return self
if self.config:
with open(self.config_path, "w+") as f:
f.writelines(self.config.output())
Logger.log("Finding cromwell jar")
cromwell_loc = self.resolve_jar(self.cromwelljar, job.cromwell,
job.config_dir)
Logger.info(f"Starting cromwell ({cromwell_loc})...")
cmd = ["java", "-DLOG_MODE=standard"]
if job.cromwell and job.cromwell.memory_mb:
cmd.extend([
f"-Xmx{job.cromwell.memory_mb}M",
f"-Xms{max(job.cromwell.memory_mb//2, 1)}M",
])
# if Logger.CONSOLE_LEVEL == LogLevel.VERBOSE:
# cmd.append("-DLOG_LEVEL=DEBUG")
if additional_cromwell_options:
cmd.extend(additional_cromwell_options)
self.port = find_free_port()
self.host = f"127.0.0.1:{self.port}"
cmd.append(f"-Dwebservice.port={self.port}")
cmd.append(f"-Dwebservice.interface=127.0.0.1")
if self.config_path and os.path.exists(self.config_path):
Logger.debug("Using configuration file for Cromwell: " +
self.config_path)
cmd.append("-Dconfig.file=" + self.config_path)
cmd.extend(["-jar", cromwell_loc, "server"])
Logger.debug(f"Starting Cromwell with command: '{' '.join(cmd)}'")
self._process = subprocess.Popen(
cmd,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
# preexec_fn creates a process group https://stackoverflow.com/a/4791612/
preexec_fn=os.setsid,
)
Logger.info("Cromwell is starting with pid=" + str(self._process.pid))
Logger.debug(
"Cromwell will start the HTTP server, reading logs to determine when this occurs"
)
self._logfp = open(self.logfile, "a+")
Logger.info(f"Will log Cromwell output to the file: {self.logfile}"
if bool(self._logfp) else
"Janis is NOT logging Cromwell output to a file")
for c in iter(self._process.stdout.readline,
"b"): # replace '' with b'' for Python 3
line = None
if c:
line = c.decode("utf-8").rstrip()
if not line:
rc = self._process.poll()
if rc is not None:
critical_suffix = f"Last received message '{line}'. "
Logger.critical(
f"Cromwell has exited with rc={rc}. {critical_suffix}The last lines of the logfile ({self.logfile}):"
)
Logger.critical("".join(tail(self._logfp, 10)))
return False
continue
if self._logfp and not self._logfp.closed:
self._logfp.write(line + "\n")
self._logfp.flush()
os.fsync(self._logfp.fileno())
Logger.debug("Cromwell: " + line)
# self.stdout.append(str(c))
if "service started on" in line:
self.process_id = self._process.pid
Logger.info("Service successfully started with pid=" +
str(self._process.pid))
break
# elif ansi_escape.match():
# raise Exception(cd)
self.is_started = True
if self._process:
self._logger = ProcessLogger(
process=self._process,
prefix="Cromwell: ",
logfp=self._logfp,
# exit_function=self.something_has_happened_to_cromwell,
)
return self
def task_did_update(self, logger: CWLToolLogger, job: RunJobModel):
Logger.debug(f"Updated task {job.id_} with status={job.status}")
self.taskmeta["jobs"][job.id_] = job
for callback in self.progress_callbacks.get(logger.sid, []):
callback(self.metadata(logger.sid))
def run(self):
finalstatus = None
iserroring = False
try:
for c in iter(self.process.stderr.readline, "b"):
if self.should_terminate:
return
line = None
if c:
line = c.decode("utf-8").rstrip()
if not line:
if self.process.poll() is not None:
finalstatus = TaskStatus.ABORTED
Logger.warn(
f"CWLTool finished with rc={self.process.returncode} but janis "
f"was unable to capture the workflow status. Marking as aborted"
)
break
continue
if self.logfp and not self.logfp.closed:
self.logfp.write(line + "\n")
self.logfp.flush()
os.fsync(self.logfp.fileno())
lowline = line.lower().lstrip()
if lowline.startswith("error"):
Logger.critical("cwltool: " + line)
iserroring = True
elif lowline.startswith("warn"):
iserroring = False
Logger.warn("cwltool: " + line)
elif lowline.startswith("info"):
iserroring = False
Logger.info("cwltool: " + line)
self.process_metadataupdate_if_match(line)
else:
Logger.debug("cwltool: " + line)
if iserroring:
self.error = (self.error or "") + "\n" + line
if "final process status is" in lowline:
if "fail" in line.lower():
finalstatus = TaskStatus.FAILED
elif "success" in line.lower():
finalstatus = TaskStatus.COMPLETED
else:
finalstatus = TaskStatus.ABORTED
break
j = ""
Logger.info("Process has completed")
if finalstatus == TaskStatus.COMPLETED:
for c in iter(self.process.stdout.readline, "s"):
if not c:
continue
line = c.decode("utf-8").rstrip()
Logger.debug(line)
if self.logfp and not self.logfp.closed:
self.logfp.write(line + "\n")
self.logfp.flush()
os.fsync(self.logfp.fileno())
j += line
try:
self.outputs = json.loads(j)
break
except:
continue
if self.error:
Logger.critical("Janis detected a CWLTool error: " +
self.error)
Logger.info("CWLTool detected transition to terminal status: " +
str(finalstatus))
self.terminate()
if self.exit_function:
self.exit_function(self, finalstatus)
except KeyboardInterrupt:
self.should_terminate = True
print("Detected keyboard interrupt")
# raise
except Exception as e:
print("Detected another error")
raise e
