def data_collection_hook(self):
"""Spawns data emulation using gphl simcal"""
data_collect_parameters = self.current_dc_parameters
if not api.gphl_workflow:
raise ValueError("Emulator requires GPhL workflow installation")
gphl_connection = api.gphl_connection
if not gphl_connection:
raise ValueError("Emulator requires GPhL connection installation")
# Get program locations
simcal_executive = gphl_connection.get_executable("simcal")
# Get environmental variables
envs = {
"BDG_home": gphl_connection.software_paths["BDG_home"],
"GPHL_INSTALLATION": gphl_connection.software_paths[
"GPHL_INSTALLATION"
],
}
for tag, val in self["environment_variables"].getProperties().items():
envs[str(tag)] = str(val)
# get crystal data
sample_name = self.getProperty("default_sample_name")
sample = self.sample_changer_hwobj.getLoadedSample()
if sample:
ss0 = sample.getName()
if ss0 and ss0.startswith(self.TEST_SAMPLE_PREFIX):
sample_name = ss0[len(self.TEST_SAMPLE_PREFIX) :]
sample_dir = gphl_connection.software_paths.get("gphl_test_samples")
if not sample_dir:
raise ValueError("Emulator requires gphl_test_samples dir specified")
sample_dir = os.path.join(sample_dir, sample_name)
if not os.path.isdir(sample_dir):
raise ValueError("Sample data directory %s does not exist" % sample_dir)
crystal_file = os.path.join(sample_dir, "crystal.nml")
if not os.path.isfile(crystal_file):
raise ValueError(
"Emulator crystal data file %s does not exist" % crystal_file
)
# in spite of the simcal_crystal_list name this returns an OrderdDict
crystal_data = f90nml.read(crystal_file)["simcal_crystal_list"]
if isinstance(crystal_data, list):
crystal_data = crystal_data[0]
input_data = self._get_simcal_input(data_collect_parameters, crystal_data)
# NB outfile is the echo output of the input file;
# image files templates are set in the input file
file_info = data_collect_parameters["fileinfo"]
if not os.path.exists(file_info["directory"]):
os.makedirs(file_info["directory"])
if not os.path.exists(file_info["directory"]):
os.makedirs(file_info["directory"])
infile = os.path.join(
file_info["directory"], "simcal_in_%s.nml" % self._counter
)
f90nml.write(input_data, infile, force=True)
outfile = os.path.join(
file_info["directory"], "simcal_out_%s.nml" % self._counter
)
logfile = os.path.join(
file_info["directory"], "simcal_log_%s.txt" % self._counter
)
self._counter += 1
hklfile = os.path.join(sample_dir, "sample.hkli")
if not os.path.isfile(hklfile):
raise ValueError("Emulator hkli file %s does not exist" % hklfile)
command_list = [
simcal_executive,
"--input",
infile,
"--output",
outfile,
"--hkl",
hklfile,
]
for tag, val in self["simcal_options"].getProperties().items():
command_list.extend(ConvertUtils.command_option(tag, val, prefix="--"))
logging.getLogger("HWR").info("Executing command: %s", command_list)
logging.getLogger("HWR").info(
"Executing environment: %s" % sorted(envs.items())
)
fp1 = open(logfile, "w")
fp2 = subprocess.STDOUT
# resource.setrlimit(resource.RLIMIT_STACK, (-1,-1))
try:
running_process = subprocess.Popen(
command_list, stdout=fp1, stderr=fp2, env=envs
)
gphl_connection.collect_emulator_process = running_process
except BaseException:
logging.getLogger("HWR").error("Error in spawning workflow application")
raise
finally:
fp1.close()
# This does waiting, so we want to collect the result afterwards
super(CollectEmulator, self).data_collection_hook()
logging.getLogger("HWR").info("Waiting for simcal collection emulation.")
# NBNB TODO put in time-out, somehow
return_code = running_process.wait()
process = gphl_connection.collect_emulator_process
gphl_connection.collect_emulator_process = None
if process == 'ABORTED':
logging.getLogger("HWR").info("Simcal collection emulation aborted")
elif return_code:
raise RuntimeError(
"simcal process terminated with return code %s" % return_code
)
else:
logging.getLogger("HWR").info("Simcal collection emulation successful")
return
def start_workflow(self, workflow_queue, workflow_model_obj):
# NBNB All command line option values are put in quotes (repr) when
# the workflow is invoked remotely through ssh.
self.workflow_queue = workflow_queue
if self.get_state() != States.OFF:
# NB, for now workflow is started as the connection is made,
# so we are never in state 'ON'/STANDBY
raise RuntimeError(
"Workflow is already running, cannot be started")
self._workflow_name = workflow_model_obj.get_type()
params = workflow_model_obj.get_workflow_parameters()
in_shell = self.hasObject("ssh_options")
if in_shell:
dd0 = self["ssh_options"].getProperties().copy()
#
host = dd0.pop("Host")
command_list = ["ssh"]
if "ConfigFile" in dd0:
command_list.extend(("-F", dd0.pop("ConfigFile")))
for tag, val in sorted(dd0.items()):
command_list.extend(("-o", "%s=%s" % (tag, val)))
# command_list.extend(('-o', tag, val))
command_list.append(host)
else:
command_list = []
command_list.append(self.software_paths["java_binary"])
# # HACK - debug options REMOVE!
# import socket
# sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
# sock.connect(("8.8.8.8", 80))
# ss0 = "-agentlib:jdwp=transport=dt_socket,address=%s:8050,server=y,suspend=y"
# command_list.append(ss0 % sock.getsockname()[0])
for tag, val in sorted(
params.get("invocation_properties", {}).items()):
command_list.extend(
ConvertUtils.java_property(tag, val, quote_value=in_shell))
# We must get hold of the options here, as we need wdir for a property
workflow_options = dict(params.get("options", {}))
calibration_name = workflow_options.get("calibration")
if calibration_name:
# Expand calibration base name - to simplify identification.
workflow_options["calibration"] = "%s_%s" % (
calibration_name,
workflow_model_obj.get_name(),
)
elif not workflow_options.get("strategy"):
workflow_options[
"strategy"] = workflow_model_obj.get_characterisation_strategy(
)
path_template = workflow_model_obj.get_path_template()
if "prefix" in workflow_options:
workflow_options["prefix"] = path_template.base_prefix
workflow_options["wdir"] = os.path.join(
path_template.process_directory, self.getProperty("gphl_subdir"))
# Hardcoded - location for log output
command_list.extend(
ConvertUtils.java_property("co.gphl.wf.wdir",
workflow_options["wdir"],
quote_value=in_shell))
ll0 = ConvertUtils.command_option(
"cp",
self.software_paths["gphl_java_classpath"],
quote_value=in_shell)
command_list.extend(ll0)
command_list.append(params["application"])
for keyword, value in params.get("properties", {}).items():
command_list.extend(
ConvertUtils.java_property(keyword,
value,
quote_value=in_shell))
for keyword, value in self.java_properties.items():
command_list.extend(
ConvertUtils.java_property(keyword,
value,
quote_value=in_shell))
for keyword, value in workflow_options.items():
command_list.extend(
ConvertUtils.command_option(keyword,
value,
quote_value=in_shell))
#
wdir = workflow_options.get("wdir")
# NB this creates the appdir as well (wdir is within appdir)
if not os.path.isdir(wdir):
try:
os.makedirs(wdir)
except Exception:
# No need to raise error - program will fail downstream
logging.getLogger("HWR").error(
"Could not create GPhL working directory: %s", wdir)
for ss0 in command_list:
ss0 = ss0.split("=")[-1]
if ss0.startswith(
"/") and "*" not in ss0 and not os.path.exists(ss0):
logging.getLogger("HWR").warning("File does not exist : %s" %
ss0)
logging.getLogger("HWR").info("GPhL execute :\n%s",
" ".join(command_list))
# Get environmental variables
envs = os.environ.copy()
# # Trick to allow unauthorised account (e.g. ESRF: opid30) to run GPhL programs
# # Any value is OK, just setting it is enough.
# envs["AutoPROCWorkFlowUser"] = "******"
# Hack to pass alternative installation dir for processing
val = self.software_paths.get("gphl_wf_processing_installation")
if val:
envs["GPHL_PROC_INSTALLATION"] = val
# These env variables are needed in some cases for wrapper scripts
# Specifically for the stratcal wrapper.
envs["GPHL_INSTALLATION"] = self.software_paths["GPHL_INSTALLATION"]
envs["BDG_home"] = self.software_paths["BDG_home"]
logging.getLogger("HWR").info(
"Executing GPhL workflow, in environment %s", envs)
try:
self._running_process = subprocess.Popen(command_list, env=envs)
except Exception:
logging.getLogger().error("Error in spawning workflow application")
raise
logging.getLogger("py4j.clientserver").setLevel(logging.WARNING)
self.set_state(States.RUNNING)
logging.getLogger("HWR").debug(
"GPhL workflow pid, returncode : %s, %s" %
(self._running_process.pid, self._running_process.returncode))
def data_collection_hook(self):
"""Spawns data emulation using gphl simcal"""
data_collect_parameters = self.current_dc_parameters
if not HWR.beamline.gphl_workflow:
raise ValueError("Emulator requires GPhL workflow installation")
gphl_connection = HWR.beamline.gphl_connection
if not gphl_connection:
raise ValueError("Emulator requires GPhL connection installation")
# Get program locations
simcal_executive = gphl_connection.get_executable("simcal")
# Get environmental variables
envs = {
"BDG_home":
gphl_connection.software_paths["BDG_home"],
"GPHL_INSTALLATION":
gphl_connection.software_paths["GPHL_INSTALLATION"],
}
text_type = ConvertUtils.text_type
for tag, val in self["environment_variables"].getProperties().items():
envs[text_type(tag)] = text_type(val)
# get crystal data
sample_name = self.getProperty("default_sample_name")
sample = HWR.beamline.sample_changer.getLoadedSample()
if sample:
ss0 = sample.getName()
if ss0 and ss0.startswith(self.TEST_SAMPLE_PREFIX):
sample_name = ss0[len(self.TEST_SAMPLE_PREFIX):]
sample_dir = gphl_connection.software_paths.get("gphl_test_samples")
if not sample_dir:
raise ValueError(
"Emulator requires gphl_test_samples dir specified")
sample_dir = os.path.join(sample_dir, sample_name)
if not os.path.isdir(sample_dir):
raise ValueError("Sample data directory %s does not exist" %
sample_dir)
crystal_file = os.path.join(sample_dir, "crystal.nml")
if not os.path.isfile(crystal_file):
raise ValueError("Emulator crystal data file %s does not exist" %
crystal_file)
# in spite of the simcal_crystal_list name this returns an OrderdDict
crystal_data = f90nml.read(crystal_file)["simcal_crystal_list"]
if isinstance(crystal_data, list):
crystal_data = crystal_data[0]
input_data = self._get_simcal_input(data_collect_parameters,
crystal_data)
# NB outfile is the echo output of the input file;
# image files templates are set in the input file
file_info = data_collect_parameters["fileinfo"]
if not os.path.exists(file_info["directory"]):
os.makedirs(file_info["directory"])
if not os.path.exists(file_info["directory"]):
os.makedirs(file_info["directory"])
infile = os.path.join(file_info["directory"],
"simcal_in_%s.nml" % self._counter)
f90nml.write(input_data, infile, force=True)
outfile = os.path.join(file_info["directory"],
"simcal_out_%s.nml" % self._counter)
logfile = os.path.join(file_info["directory"],
"simcal_log_%s.txt" % self._counter)
self._counter += 1
hklfile = os.path.join(sample_dir, "sample.hkli")
if not os.path.isfile(hklfile):
raise ValueError("Emulator hkli file %s does not exist" % hklfile)
command_list = [
simcal_executive,
"--input",
infile,
"--output",
outfile,
"--hkl",
hklfile,
]
for tag, val in self["simcal_options"].getProperties().items():
command_list.extend(
ConvertUtils.command_option(tag, val, prefix="--"))
logging.getLogger("HWR").info("Executing command: %s", command_list)
logging.getLogger("HWR").info("Executing environment: %s" %
sorted(envs.items()))
fp1 = open(logfile, "w")
fp2 = subprocess.STDOUT
# resource.setrlimit(resource.RLIMIT_STACK, (-1,-1))
try:
running_process = subprocess.Popen(command_list,
stdout=fp1,
stderr=fp2,
env=envs)
gphl_connection.collect_emulator_process = running_process
except BaseException:
logging.getLogger("HWR").error(
"Error in spawning workflow application")
raise
finally:
fp1.close()
# This does waiting, so we want to collect the result afterwards
super(CollectEmulator, self).data_collection_hook()
logging.getLogger("HWR").info(
"Waiting for simcal collection emulation.")
# NBNB TODO put in time-out, somehow
return_code = running_process.wait()
process = gphl_connection.collect_emulator_process
gphl_connection.collect_emulator_process = None
if process == "ABORTED":
logging.getLogger("HWR").info(
"Simcal collection emulation aborted")
elif return_code:
raise RuntimeError(
"simcal process terminated with return code %s" % return_code)
else:
logging.getLogger("HWR").info(
"Simcal collection emulation successful")
return
def start_workflow(self, workflow_queue, workflow_model_obj):
# NBNB All command line option values are put in quotes (repr) when
# the workflow is invoked remotely through ssh.
self.workflow_queue = workflow_queue
if self.get_state() != States.OFF:
# NB, for now workflow is started as the connection is made,
# so we are never in state 'ON'/STANDBY
raise RuntimeError("Workflow is already running, cannot be started")
self._workflow_name = workflow_model_obj.get_type()
params = workflow_model_obj.get_workflow_parameters()
in_shell = self.hasObject("ssh_options")
if in_shell:
dd0 = self["ssh_options"].getProperties().copy()
#
host = dd0.pop("Host")
command_list = ["ssh"]
if "ConfigFile" in dd0:
command_list.extend(("-F", dd0.pop("ConfigFile")))
for tag, val in sorted(dd0.items()):
command_list.extend(("-o", "%s=%s" % (tag, val)))
# command_list.extend(('-o', tag, val))
command_list.append(host)
else:
command_list = []
command_list.append(self.software_paths["java_binary"])
# # HACK - debug options REMOVE!
# import socket
# sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
# sock.connect(("8.8.8.8", 80))
# ss0 = "-agentlib:jdwp=transport=dt_socket,address=%s:8050,server=y,suspend=y"
# command_list.append(ss0 % sock.getsockname()[0])
for tag, val in sorted(params.get("invocation_properties", {}).items()):
command_list.extend(
ConvertUtils.java_property(tag, val, quote_value=in_shell)
)
# We must get hold of the options here, as we need wdir for a property
workflow_options = dict(params.get("options", {}))
calibration_name = workflow_options.get("calibration")
if calibration_name:
# Expand calibration base name - to simplify identification.
workflow_options["calibration"] = "%s_%s" % (
calibration_name,
workflow_model_obj.get_name(),
)
path_template = workflow_model_obj.get_path_template()
if "prefix" in workflow_options:
workflow_options["prefix"] = path_template.base_prefix
workflow_options["wdir"] = os.path.join(
path_template.process_directory, self.getProperty("gphl_subdir")
)
# Hardcoded - location for log output
command_list.extend(
ConvertUtils.java_property(
"co.gphl.wf.wdir", workflow_options["wdir"], quote_value=in_shell
)
)
ll0 = ConvertUtils.command_option(
"cp", self.software_paths["gphl_java_classpath"], quote_value=in_shell
)
command_list.extend(ll0)
command_list.append(params["application"])
for keyword, value in params.get("properties", {}).items():
command_list.extend(
ConvertUtils.java_property(keyword, value, quote_value=in_shell)
)
for keyword, value in self.java_properties.items():
command_list.extend(
ConvertUtils.java_property(keyword, value, quote_value=in_shell)
)
for keyword, value in workflow_options.items():
command_list.extend(
ConvertUtils.command_option(keyword, value, quote_value=in_shell)
)
#
wdir = workflow_options.get("wdir")
# NB this creates the appdir as well (wdir is within appdir)
if not os.path.isdir(wdir):
try:
os.makedirs(wdir)
except BaseException:
# No need to raise error - program will fail downstream
logging.getLogger("HWR").error(
"Could not create GPhL working directory: %s", wdir
)
for ss0 in command_list:
ss0 = ss0.split("=")[-1]
if ss0.startswith("/") and "*" not in ss0 and not os.path.exists(ss0):
logging.getLogger("HWR").warning("File does not exist : %s" % ss0)
logging.getLogger("HWR").info("GPhL execute :\n%s", " ".join(command_list))
# Get environmental variables
envs = os.environ.copy()
# # Trick to allow unauthorised account (e.g. ESRF: opid30) to run GPhL programs
# # Any value is OK, just setting it is enough.
# envs["AutoPROCWorkFlowUser"] = "******"
# Hack to pass alternative installation dir for processing
val = self.software_paths.get("gphl_wf_processing_installation")
if val:
envs["GPHL_PROC_INSTALLATION"] = val
# These env variables are needed in some cases for wrapper scripts
# Specifically for the stratcal wrapper.
envs["GPHL_INSTALLATION"] = self.software_paths["GPHL_INSTALLATION"]
envs["BDG_home"] = self.software_paths["BDG_home"]
logging.getLogger("HWR").info(
"Executing GPhL workflow, in environment %s", envs
)
try:
self._running_process = subprocess.Popen(command_list, env=envs)
except BaseException:
logging.getLogger().error("Error in spawning workflow application")
raise
logging.getLogger("py4j.clientserver").setLevel(logging.WARNING)
self.set_state(States.RUNNING)
logging.getLogger("HWR").debug(
"GPhL workflow pid, returncode : %s, %s"
% (self._running_process.pid, self._running_process.returncode)
)