def init_detectors(self):
"""Set up the detectors"""
self.logger.debug("Setting up the detectors")
# Shorten variable names
site = self.site
import sites.detectors.necat_dectris_eiger16m
# A single detector
if site.DETECTOR:
#detector, suffix = site.DETECTOR
detector = site.DETECTOR
detector = detector.lower()
self.detectors[self.site_ids[0].upper()] = load_module(
seek_module=detector,
directories=("sites.detectors", "detectors"),
logger=self.logger)
# Multiple detectors
elif site.DETECTORS:
for site_id in self.site_ids:
detector, suffix = site.DETECTORS[site_id]
detector = detector.lower()
self.detectors[site_id.upper()] = load_module(
seek_module=detector,
directories=("sites.detectors", "detectors"))
def init_detectors(self):
"""Set up the detectors"""
self.logger.debug("Setting up the detectors")
# Shorten variable names
site = self.site
import sites.detectors.necat_dectris_eiger16m
# A single detector
if site.DETECTOR:
#detector, suffix = site.DETECTOR
detector = site.DETECTOR
detector = detector.lower()
self.detectors[self.site_ids[0].upper()] = load_module(
seek_module=detector,
directories=("sites.detectors", "detectors"),
logger=self.logger)
# Multiple detectors
elif site.DETECTORS:
for site_id in self.site_ids:
detector, suffix = site.DETECTORS[site_id]
detector = detector.lower()
self.detectors[site_id.upper()] = load_module(
seek_module=detector,
directories=("sites.detectors", "detectors"))
def run(self):
"""
Orchestrate the adapter's actions
"""
# Check if command is ECHO
if self.message['command'] == 'ECHO':
# Load the simple_echo module
echo = load_module(
seek_module='launch.launcher_adapters.echo_simple')
# send message to simple_echo
echo.LauncherAdapter(self.site, self.message, self.settings)
else:
# Adjust the message to this site
#self.fix_command()
self.message = launch_tools.fix_command(self.message)
# Put the command into a file
#command_file = launch_tools.write_command_file(self.settings["launch_dir"],
command_file = launch_tools.write_command_file(
os.path.join(self.settings["launch_dir"], 'command_files'),
self.message["command"], self.message)
# Set the site tag from input
site_tag = launch_tools.get_site_tag(self.message).split('_')[0]
# Call the launch process on the command file
self.logger.debug("rapd.launch -s %s %s", site_tag, command_file)
Popen(["rapd.launch", "-s", site_tag, command_file])
def run(self):
"""
Orchestrate the adapter's actions
"""
# Check if command is ECHO
if self.message['command'] == 'ECHO':
# Load the simple_echo module
echo = load_module(seek_module='launch.launcher_adapters.echo_simple')
# send message to simple_echo
echo.LauncherAdapter(self.site, self.message, self.settings)
else:
# Adjust the message to this site
#self.fix_command()
self.message = launch_tools.fix_command(self.message)
# Put the command into a file
command_file = launch_tools.write_command_file(self.settings["launch_dir"],
self.message["command"],
self.message)
# Set the site tag from input
site_tag = launch_tools.get_site_tag(self.message).split('_')[0]
# Call the launch process on the command file
self.logger.debug("rapd.launch -s %s %s", site_tag, command_file)
Popen(["rapd.launch", "-s",site_tag, command_file])
def process_labelit(self, inp=False):
"""
Initiate Labelit runs.
"""
if self.verbose:
self.logger.debug("LabelitPP::process_labelit")
try:
queue = Queue()
params = {}
params["test"] = self.test
params["cluster"] = False # self.cluster_use # TODO
params["verbose"] = self.verbose
args1 = {}
if inp:
args1["input"] = inp
else:
args1["input"] = self.images
args1["output"] = queue
args1["params"] = params
args1["logger"] = self.logger
# Import the RunLabelit class
agent = load_module(seek_module="rapd_agent_index+strategy",
directories=self.agent_directories,
logger=self.logger)
Process(target=agent.RunLabelit, kwargs=args1).start()
self.labelit_results = queue.get()
self.labelit_log = queue.get()
except:
self.logger.exception("**Error in LabelitPP.process_labelit**")
def load_agent(self, command):
"""Load the agent file for this command"""
# Agent we are looking for
seek_module = "rapd_agent_%s" % command.lower()
self.agent = load_module(seek_module=seek_module, directories=self.site.RAPD_AGENT_DIRECTORIES)
def load_adapter(self):
"""Find and load the adapter"""
# Import the database adapter as database module
self.adapter = load_module(
seek_module=self.launcher["adapter"],
directories=self.site.LAUNCHER_SETTINGS["RAPD_LAUNCHER_ADAPTER_DIRECTORIES"]).LauncherAdapter
if self.logger:
self.logger.debug(self.adapter)
def load_adapter(self):
"""Find and load the adapter"""
# Import the database adapter as database module
self.adapter = load_module(
seek_module=self.specifications["adapter"],
directories=self.site.LAUNCHER_SETTINGS[
"RAPD_LAUNCHER_ADAPTER_DIRECTORIES"]).LauncherAdapter
self.logger.debug(self.adapter)
def load_cluster(image_dir):
"""
Load the computer cluster module so the job can be submitted.
"""
# Determine the site and load cluster. May be different from where images are from...
# Check the images
imgs = sort_dir(image_dir)
site = Utils.get_site(imgs[0],False)
cluster_site_name = os.getenv('RAPD_SITE1', False)
# Determine the site by env variable first.
if cluster_site_name:
cluster_site_name = cluster_site_name.lower()
cluster_site = load_module(seek_module=cluster_site_name, directories='sites')
else:
# Otherwise use the image origin as site.
cluster_site = load_module(seek_module=site[0], directories='sites')
cluster = load_module(cluster_site.CLUSTER_ADAPTER)
out = (site[1], site[0], cluster_site_name, imgs)
return ((out, cluster))
def load_cluster(image_dir):
"""
Load the computer cluster module so the job can be submitted.
"""
# Determine the site and load cluster. May be different from where images are from...
# Check the images
imgs = sort_dir(image_dir)
site = Utils.get_site(imgs[0], False)
cluster_site_name = os.getenv('RAPD_SITE1', False)
# Determine the site by env variable first.
if cluster_site_name:
cluster_site_name = cluster_site_name.lower()
cluster_site = load_module(seek_module=cluster_site_name,
directories='sites')
else:
# Otherwise use the image origin as site.
cluster_site = load_module(seek_module=site[0], directories='sites')
cluster = load_module(cluster_site.CLUSTER_ADAPTER)
out = (site[1], site[0], cluster_site_name, imgs)
return ((out, cluster))
def load_plugin(self, command):
"""
Load the plugin file for this command
Keyword arguments
command -- the command to be run (index+strategy for example)
"""
# plugin directories we are looking for
directories = []
for directory in self.site.RAPD_PLUGIN_DIRECTORIES:
directories.append(directory+".%s" % self.command.get("command").lower())
# Load the plugin from directories defined in site file
self.plugin = load_module(seek_module="plugin",
directories=directories,
logger=self.new_logger)
def load_plugin(self, command):
"""
Load the plugin file for this command
Keyword arguments
command -- the command to be run (index+strategy for example)
"""
# plugin directories we are looking for
directories = []
for directory in self.site.RAPD_PLUGIN_DIRECTORIES:
directories.append(directory +
".%s" % self.command.get("command").lower())
# Load the plugin from directories defined in site file
self.plugin = load_module(seek_module="plugin",
directories=directories,
logger=self.new_logger)
def Launcher_Adapter(site, message, settings):
# Check if command is ECHO
if message['command'] == 'ECHO':
# Load the simple_echo module
echo = load_module(seek_module='launch.launcher_adapters.echo_simple')
# send message to simple_echo
echo.LauncherAdapter(site, message, settings)
else:
# Adjust the message to this site
#self.fix_command()
message = launch_tools.fix_command(message)
# Put the command into a file
command_file = launch_tools.write_command_file(settings["launch_dir"],
message["command"],
message)
# Set the site tag from input
site_tag = launch_tools.get_site_tag(message).split('_')[0]
# Call the launch process on the command file
#self.logger.debug("rapd.launch -s %s %s", site_tag, command_file)
Popen(["rapd.launch", "-s", site_tag, command_file], shell=True)
def main():
"""
The main process
Setup logging, gather information, and run the plugin
"""
# Get the commandline args
commandline_args = get_commandline()
# Output log file is always verbose
log_level = 10
# Set up logging
logger = utils.log.get_logger(logfile_dir="./",
logfile_id="rapd_integrate",
level=log_level,
console=commandline_args.test)
# Set up terminal printer
# Verbosity
if commandline_args.verbose:
terminal_log_level = 10
elif commandline_args.json:
terminal_log_level = 100
else:
terminal_log_level = 50
tprint = utils.log.get_terminal_printer(verbosity=terminal_log_level,
no_color=commandline_args.no_color)
print_welcome_message(tprint)
# Print out commandline arguments
logger.debug("Commandline arguments:")
tprint(arg="\nCommandline arguments:", level=10, color="blue")
for pair in commandline_args._get_kwargs():
logger.debug(" arg:%s val:%s", pair[0], pair[1])
tprint(arg=" arg:%-20s val:%s" % (pair[0], pair[1]),
level=10,
color="white")
# Get the environmental variables
environmental_vars = utils.site.get_environmental_variables()
logger.debug("\n" + text.info + "Environmental variables" + text.stop)
tprint("\nEnvironmental variables", level=10, color="blue")
for key, val in environmental_vars.iteritems():
logger.debug(" " + key + " : " + val)
tprint(arg=" arg:%-20s val:%s" % (key, val), level=10, color="white")
# List sites?
if commandline_args.listsites:
tprint(arg="\nAvailable sites", level=99, color="blue")
commandline_utils.print_sites(left_buffer=" ")
if not commandline_args.listdetectors:
sys.exit()
# List detectors?
if commandline_args.listdetectors:
print "\n" + text.info + "Available detectors:" + text.stop
commandline_utils.print_detectors(left_buffer=" ")
sys.exit()
# Look for data based on the input template
data_files = commandline_utils.analyze_data_sources(
commandline_args.template,
mode="integrate",
start_image=commandline_args.start_image,
end_image=commandline_args.end_image)
if "hdf5_files" in data_files:
logger.debug("HDF5 source file(s)")
tprint(arg="\nHDF5 source file(s)", level=99, color="blue")
logger.debug(data_files["hdf5_files"])
for data_file in data_files["hdf5_files"]:
tprint(arg=" " + data_file, level=99, color="white")
logger.debug("CBF file(s) from HDF5 file(s)")
tprint(arg="\nData files", level=99, color="blue")
else:
logger.debug("Data to be integrated")
tprint(arg="\nData to be integrated", level=99, color="blue")
tprint(arg=" From %s" % data_files["data_files"][0],
level=99,
color="white")
tprint(arg=" To %s" % data_files["data_files"][-1],
level=99,
color="white")
# Need data
if len(data_files) == 0 and commandline_args.test == False:
raise Exception("No files input for integration.")
# Get site - commandline wins over the environmental variable
site = False
site_module = False
detector = {}
detector_module = False
if commandline_args.site:
site = commandline_args.site
elif environmental_vars.has_key("RAPD_SITE"):
site = environmental_vars["RAPD_SITE"]
if commandline_args.detector:
detector = commandline_args.detector
detector_module = detector_utils.load_detector(detector)
# If no site or detector, try to figure out the detector
if not (site or detector):
detector = detector_utils.get_detector_file(
data_files["data_files"][0])
if isinstance(detector, dict):
if detector.has_key("site"):
site_target = detector.get("site")
site_file = utils.site.determine_site(site_arg=site_target)
# print site_file
site_module = importlib.import_module(site_file)
detector_target = site_module.DETECTOR.lower()
detector_module = detector_utils.load_detector(detector_target)
elif detector.has_key("detector"):
site_module = False
detector_target = detector.get("detector")
detector_module = detector_utils.load_detector(detector_target)
# Have a detector - read in file data
if not detector_module:
raise Exception("No detector identified")
# Get header information
image_0_data = get_image_data(data_file=data_files["data_files"][0],
detector_module=detector_module,
site_module=site_module)
image_n_data = get_image_data(data_file=data_files["data_files"][-1],
detector_module=detector_module,
site_module=site_module)
logger.debug("Image header: %s, %s", image_0_data, image_n_data)
tprint(arg="\nImage headers", level=10, color="blue")
count = 0
for header in (image_0_data, image_n_data):
keys = header.keys()
keys.sort()
if count > 0:
tprint(arg="", level=10, color="white")
tprint(arg=" %s" % header["fullname"], level=10, color="white")
for key in keys:
tprint(arg=" arg:%-22s val:%s" % (key, header[key]),
level=10,
color="white")
count += 1
# Get the run data
run_data = get_run_data(detector_module, image_0_data, image_n_data,
commandline_args)
logger.debug("Run data: %s", run_data)
tprint(arg="\nRun data", level=10, color="blue")
keys = run_data.keys()
keys.sort()
for key in keys:
tprint(arg=" arg:%-22s val:%s" % (key, run_data[key]),
level=10,
color="white")
# Construct the command for the plugin
command = construct_command(image_0_data, run_data, commandline_args,
detector_module)
# Load the plugin
plugin = load_module(seek_module="plugin",
directories=["plugins.integrate"],
logger=logger)
tprint(arg="\nPlugin information", level=10, color="blue")
tprint(arg=" Plugin type: %s" % plugin.PLUGIN_TYPE,
level=10,
color="white")
tprint(arg=" Plugin subtype: %s" % plugin.PLUGIN_SUBTYPE,
level=10,
color="white")
tprint(arg=" Plugin version: %s" % plugin.VERSION,
level=10,
color="white")
tprint(arg=" Plugin id: %s" % plugin.ID, level=10, color="white")
# Instantiate the plugin
plugin.RapdPlugin(site=None, command=command, tprint=tprint, logger=logger)
def run(self):
"""
Orchestrate the adapter's actions
"""
if self.message['command'] == 'ECHO':
echo = load_module(seek_module='launch.launcher_adapters.echo_simple')
# send message to simple_echo
echo.LauncherAdapter(self.site, self.message, self.settings)
else:
# Load the cluster adapter for the site.
cluster = load_module(seek_module=self.site.CLUSTER_ADAPTER)
# Adjust the message to this site
# Get the new working directory. Message only has second part of the path.
#self.message = cluster.fix_command(self.message)
self.message = launch_tools.fix_command(self.message)
# Get the new working directory
work_dir = self.message["directories"]["work"]
# Get the launcher directory - Add command_files to keep files isolated
qsub_dir = self.message["directories"]["launch_dir"]+"/command_files"
# Put the message into a rapd-readable file
command_file = launch_tools.write_command_file(qsub_dir, self.message["command"], self.message)
# Set the site tag from input
site_tag = launch_tools.get_site_tag(self.message).split('_')[0]
# The command to launch the job
command_line = "rapd.launch -vs %s %s" % (site_tag, command_file)
# Parse a label for qsub job from the command_file name
qsub_label = os.path.basename(command_file).replace(".rapd", "")
# Determine the number of precessors to request for job
if hasattr(cluster, 'determine_nproc'):
nproc = cluster.determine_nproc(self.message['command'])
else:
nproc = 1
# Determine which cluster queue to run
if hasattr(cluster, 'check_queue'):
queue = cluster.check_queue(self.message['command'])
else:
queue = False
Thread(target=cluster.process_cluster,
kwargs={'command':command_line,
'work_dir':work_dir,
'logfile':False,
'batch_queue':queue,
'nproc':nproc,
'logger':self.logger,
'name':qsub_label,
'mp_event':False,
'timeout':False,
'pid_queue':False,
}).start()
"""
def main():
"""
The main process
Setup logging and instantiate the model"""
# Get the commandline args
commandline_args = get_commandline()
# Output log file is always verbose
log_level = 10
# Set up logging
logger = utils.log.get_logger(
logfile_dir="./",
logfile_id="rapd_mr",
level=log_level,
#console=commandline_args.test
console=False)
# Set up terminal printer
# Verbosity
if commandline_args.json:
terminal_log_level = 100
elif commandline_args.verbose:
terminal_log_level = 10
else:
terminal_log_level = 50
tprint = utils.log.get_terminal_printer(verbosity=terminal_log_level,
no_color=commandline_args.no_color,
progress=commandline_args.progress)
print_welcome_message(tprint)
logger.debug("Commandline arguments:")
tprint(arg="\nCommandline arguments:", level=10, color="blue")
for pair in commandline_args._get_kwargs():
logger.debug(" arg:%s val:%s", pair[0], pair[1])
tprint(arg=" arg:%-20s val:%s" % (pair[0], pair[1]),
level=10,
color="white")
# Get the environmental variables
environmental_vars = utils.site.get_environmental_variables()
logger.debug("" + text.info + "Environmental variables" + text.stop)
tprint("\nEnvironmental variables", level=10, color="blue")
for key, val in environmental_vars.iteritems():
logger.debug(" " + key + " : " + val)
tprint(arg=" arg:%-20s val:%s" % (key, val), level=10, color="white")
# Should working directory go up or down?
if environmental_vars.get("RAPD_DIR_INCREMENT") in ("up", "UP"):
commandline_args.dir_up = True
else:
commandline_args.dir_up = False
# Get site - commandline wins over the environmental variable
site = False
site_module = False
if commandline_args.site:
site = commandline_args.site
elif environmental_vars.has_key("RAPD_SITE"):
site = environmental_vars["RAPD_SITE"]
# If someone specifies the site or found in env.
if site and not site_module:
site_file = utils.site.determine_site(site_arg=site)
site_module = importlib.import_module(site_file)
# Construct the command
command = construct_command(commandline_args=commandline_args)
# Load the plugin
plugin = modules.load_module(seek_module="plugin",
directories=["plugins.mr"],
logger=logger)
# Print out plugin info
tprint(arg="\nPlugin information", level=10, color="blue")
tprint(arg=" Plugin type: %s" % plugin.PLUGIN_TYPE,
level=10,
color="white")
tprint(arg=" Plugin subtype: %s" % plugin.PLUGIN_SUBTYPE,
level=10,
color="white")
tprint(arg=" Plugin version: %s" % plugin.VERSION,
level=10,
color="white")
tprint(arg=" Plugin id: %s" % plugin.ID, level=10, color="white")
# Run the plugin
# Instantiate the plugin
plugin_instance = plugin.RapdPlugin(command=command,
site=site_module,
tprint=tprint,
logger=logger)
plugin_instance.start()
def main():
""" The main process
Setup logging and instantiate the model"""
# Get the commandline args
commandline_args = get_commandline()
# Output log file is always verbose
log_level = 10
# Set up logging
if commandline_args.logging:
logger = utils.log.get_logger(logfile_dir="./",
logfile_id="rapd_index",
level=log_level,
console=commandline_args.test)
# Set up terminal printer
# Verbosity
if commandline_args.verbose:
terminal_log_level = 10
elif commandline_args.json:
terminal_log_level = 100
elif commandline_args.json_fd:
terminal_log_level = 100
else:
terminal_log_level = 30
tprint = utils.log.get_terminal_printer(verbosity=terminal_log_level,
no_color=commandline_args.no_color,
progress=commandline_args.progress,
progress_fd=commandline_args.progress_fd)
print_welcome_message(tprint)
logger.debug("Commandline arguments:")
tprint(arg="\nCommandline arguments:", level=10, color="blue")
for pair in commandline_args._get_kwargs():
logger.debug(" arg:%s val:%s", pair[0], pair[1])
tprint(arg=" arg:%-20s val:%s" % (pair[0], pair[1]), level=10, color="white")
# Get the environmental variables
environmental_vars = utils.site.get_environmental_variables()
logger.debug("\n" + text.info + "Environmental variables" + text.stop)
tprint("\nEnvironmental variables", level=10, color="blue")
for key, val in environmental_vars.iteritems():
logger.debug(" " + key + " : " + val)
tprint(arg=" arg:%-20s val:%s" % (key, val), level=10, color="white")
# Should working directory go up or down?
if environmental_vars.get("RAPD_DIR_INCREMENT") in ("up", "UP"):
commandline_args.dir_up = True
else:
commandline_args.dir_up = False
# List sites?
if commandline_args.listsites:
tprint(arg="\nAvailable sites", level=98, color="blue")
commandline_utils.print_sites(left_buffer=" ")
if not commandline_args.listdetectors:
sys.exit()
# List detectors?
if commandline_args.listdetectors:
tprint(arg="Available detectors", level=98, color="blue")
commandline_utils.print_detectors(left_buffer=" ")
sys.exit()
# Get the data files
data_files = commandline_utils.analyze_data_sources(sources=commandline_args.sources,
mode="index",
hdf5_image_range=commandline_args.hdf5_image_range,
hdf5_wedge_range=commandline_args.hdf5_wedge_range)
if "hdf5_files" in data_files:
logger.debug("HDF5 source file(s)")
tprint(arg="\nHDF5 source file(s)", level=98, color="blue")
logger.debug(data_files["hdf5_files"])
for data_file in data_files["hdf5_files"]:
tprint(arg=" " + data_file, level=98, color="white")
logger.debug("CBF file(s) from HDF5 file(s)")
tprint(arg="\nData files", level=98, color="blue")
else:
logger.debug("Data file(s)")
tprint(arg="\nData file(s)", level=98, color="blue")
if len(data_files) == 0:
tprint(arg=" None", level=98, color="white")
else:
logger.debug(data_files["files"])
for data_file in data_files["files"]:
tprint(arg=" " + data_file, level=98, color="white")
# Need data
if len(data_files) == 0 and commandline_args.test == False:
if logger:
logger.exception("No files input for indexing.")
raise Exception, "No files input for indexing."
# Too much data?
if len(data_files) > 2:
if logger:
logger.exception("Too many files for indexing. 1 or 2 images accepted")
raise Exception, "Too many files for indexing. 1 or 2 images accepted"
# Get site - commandline wins over the environmental variable
site = False
site_module = False
#detector = {}
detector = False
detector_module = False
if commandline_args.site:
site = commandline_args.site
elif environmental_vars.has_key("RAPD_SITE"):
site = environmental_vars["RAPD_SITE"]
# Detector is defined by the user
if commandline_args.detector:
detector = commandline_args.detector
detector_module = detector_utils.load_detector(detector)
# If no site or detector, try to figure out the detector
# if not (site or detector):
if not detector:
detector = detector_utils.get_detector_file(data_files["files"][0])
if isinstance(detector, dict):
if detector.has_key("site"):
site_target = detector.get("site")
site_file = utils.site.determine_site(site_arg=site_target)
site_module = importlib.import_module(site_file)
detector_target = site_module.DETECTOR.lower()
detector_module = detector_utils.load_detector(detector_target)
elif detector.has_key("detector"):
site_module = False
detector_target = detector.get("detector")
detector_module = detector_utils.load_detector(detector_target)
# If someone specifies the site or found in env.
if site and not site_module:
site_file = utils.site.determine_site(site_arg=site)
site_module = importlib.import_module(site_file)
# Have a detector - read in file data
if detector_module:
image_headers = {}
for index, data_file in enumerate(data_files["files"]):
if site_module:
image_headers[data_file] = detector_module.read_header(data_file,
#site_module.BEAM_SETTINGS)
site_module.BEAM_INFO.get(site.upper(), {}))
else:
image_headers[data_file] = detector_module.read_header(data_file)
# If this image is derived from an hdf5 master file, tag it
if "hdf5_files" in data_files:
if len(data_files["hdf5_files"]) == len(data_files["files"]):
image_headers[data_file]["hdf5_source"] = data_files["hdf5_files"][index]
else:
image_headers[data_file]["hdf5_source"] = data_files["hdf5_files"][0]
logger.debug("Image headers: %s", image_headers)
print_headers(tprint, image_headers)
command = construct_command(image_headers=image_headers,
commandline_args=commandline_args,
detector_module=detector_module)
else:
if logger:
logger.exception("No detector module found")
raise Exception("No detector module found")
# If no site, error
# if site == False:
# print text.error+"Could not determine a site. Exiting."+text.stop
# sys.exit(9)
# Determine the site_file
# site_file = utils.site.determine_site(site_arg=site)
# Error out if no site_file to import
# if site_file == False:
# print text.error+"Could not find a site file. Exiting."+text.stop
# sys.exit(9)
# Import the site settings
# print "Importing %s" % site_file
# site_module = importlib.import_module(site_file)
# Single process lock?
# utils.lock.file_lock(site_module.CONTROL_LOCK_FILE)
# Instantiate the plugin
# Load the plugin from directories defined in site file
# for d in sys.path:
# if d.endswith("src"):
# toplevel_dir = d+".plugins"
plugin = load_module(seek_module="plugin",
directories=["plugins.index"],
logger=logger)
tprint(arg="\nPlugin information", level=10, color="blue")
tprint(arg=" Plugin type: %s" % plugin.PLUGIN_TYPE, level=10, color="white")
tprint(arg=" Plugin subtype: %s" % plugin.PLUGIN_SUBTYPE, level=10, color="white")
tprint(arg=" Plugin version: %s" % plugin.VERSION, level=10, color="white")
tprint(arg=" Plugin id: %s" % plugin.ID, level=10, color="white")
#plugin_instance = plugin.RapdPlugin(None, command, tprint, logger)
plugin_instance = plugin.RapdPlugin(site_module, command, tprint, logger)
plugin_instance.start()
def run(self):
"""
Orchestrate the adapter's actions
"""
if self.message['command'] == 'ECHO':
echo = load_module(
seek_module='launch.launcher_adapters.echo_simple')
# send message to simple_echo
echo.LauncherAdapter(self.site, self.message, self.settings)
else:
# Load the cluster adapter for the site.
cluster = load_module(seek_module=self.site.CLUSTER_ADAPTER)
# Adjust the message to this site
# Get the new working directory. Message only has second part of the path.
#self.message = cluster.fix_command(self.message)
self.message = launch_tools.fix_command(self.message)
# Get the new working directory
work_dir = self.message["directories"]["work"]
# Get the launcher directory - Add command_files to keep files isolated
qsub_dir = self.message["directories"][
"launch_dir"] + "/command_files"
# Put the message into a rapd-readable file
command_file = launch_tools.write_command_file(
qsub_dir, self.message["command"], self.message)
# Set the site tag from input
site_tag = launch_tools.get_site_tag(self.message).split('_')[0]
# The command to launch the job
command_line = "rapd.launch -vs %s %s" % (site_tag, command_file)
# Parse a label for qsub job from the command_file name
qsub_label = os.path.basename(command_file).replace(".rapd", "")
# Determine the number of precessors to request for job
if hasattr(cluster, 'determine_nproc'):
nproc = cluster.determine_nproc(self.message['command'])
else:
nproc = 1
# Determine which cluster queue to run
if hasattr(cluster, 'check_queue'):
queue = cluster.check_queue(self.message['command'])
else:
queue = False
Thread(target=cluster.process_cluster,
kwargs={
'command': command_line,
'work_dir': work_dir,
'logfile': False,
'batch_queue': queue,
'nproc': nproc,
'logger': self.logger,
'name': qsub_label,
'mp_event': False,
'timeout': False,
'pid_queue': False,
}).start()
"""
def main():
"""
The main process
Setup logging and instantiate the model"""
# Get the commandline args
commandline_args = get_commandline()
# Output log file is always verbose
log_level = 10
# Set up logging
logger = utils.log.get_logger(logfile_dir="./",
logfile_id="rapd_echo",
level=log_level,
console=commandline_args.test)
# Set up terminal printer
# tprint prints to the terminal, taking the arguments
# arg - the string to be printed (if using level of "progress" this needs to be an int)
# level - value 0 to 99 or "progress". Do NOT use a value of 100 or above
# 50 - alert
# 40 - error
# 30 - warning
# 20 - info
# 10 - debug
# color - color to print ("red" and so forth. See list in utils/text)
# newline - put in False if you don't want a newline at the end of your print
#
# Verbosity
if commandline_args.verbose:
terminal_log_level = 10
elif commandline_args.json:
terminal_log_level = 100
else:
terminal_log_level = 50
tprint = utils.log.get_terminal_printer(verbosity=terminal_log_level,
no_color=commandline_args.no_color,
progress=commandline_args.progress)
print_welcome_message(tprint)
logger.debug("Commandline arguments:")
tprint(arg="\nCommandline arguments:", level=10, color="blue")
for pair in commandline_args._get_kwargs():
logger.debug(" arg:%s val:%s", pair[0], pair[1])
tprint(arg=" arg:%-20s val:%s" % (pair[0], pair[1]), level=10, color="white")
# Get the environmental variables
environmental_vars = utils.site.get_environmental_variables()
logger.debug("" + text.info + "Environmental variables" + text.stop)
tprint("\nEnvironmental variables", level=10, color="blue")
for key, val in environmental_vars.iteritems():
logger.debug(" " + key + " : " + val)
tprint(arg=" arg:%-20s val:%s" % (key, val), level=10, color="white")
# Should working directory go up or down?
if environmental_vars.get("RAPD_DIR_INCREMENT") in ("up", "UP"):
commandline_args.dir_up = True
else:
commandline_args.dir_up = False
# Construct the command
command = construct_command(commandline_args=commandline_args,
logger=logger)
# Load the plugin
plugin = modules.load_module(seek_module="plugin",
directories=["plugins.echo"],
logger=logger)
# Print plugin info
tprint(arg="\nPlugin information", level=10, color="blue")
tprint(arg=" Plugin type: %s" % plugin.PLUGIN_TYPE, level=10, color="white")
tprint(arg=" Plugin subtype: %s" % plugin.PLUGIN_SUBTYPE, level=10, color="white")
tprint(arg=" Plugin version: %s" % plugin.VERSION, level=10, color="white")
tprint(arg=" Plugin id: %s" % plugin.ID, level=10, color="white")
# Run the plugin
plugin.RapdPlugin(command, tprint, logger)
def __init__(self, inp, output, logger=None):
logger.info('FindBeamCenter.__init__')
self.st = time.time()
self.input, self.cpu, self.verbose, self.clean = inp
self.output = output
self.logger = logger
# Store passed-in variables
#self.site = site
#self.command = command
self.reply_address = self.input["return_address"]
# Setting up data input
self.setup = self.input["directories"]
self.header = self.input["header1"]
self.header2 = self.input.get("header2", False)
self.site_parameters = self.input.get("site_parameters")
self.preferences = self.input.get("preferences", {})
self.controller_address = self.reply_address
# Setup the weird named rapd_agent_index+strategy pipeline
self.Labelit = load_module(seek_module="rapd_agent_index+strategy",
directories=('agents'),
logger=self.logger)
# Setup the proper computer cluster for site.
if self.site_parameters.get('cluster_site', False):
self.Cluster = load_module(
seek_module=self.site_parameters.get('cluster_site'),
directories=('sites.cluster'),
logger=self.logger)
else:
self.Cluster = load_module(
seek_module=self.site_parameters.get('img_site'),
directories=('sites.cluster'),
logger=self.logger)
#Number of labelit runs to fine tune beam center and spacing from original
#Should be even number for symmetry around original beam center
self.iterations2 = ITERATIONS
self.iterations2_space = 0.3
#self.iterations2 = 15
#self.iterations2_space = 0.1
#Sets the limit of the std dev (mm) of either x or y beam position in the top 10% of the solutions.
#self.std_dev = 0.1
self.std_dev = 0.25
#Sets settings to check output on my machine and does not send results to database.
self.cluster_use = self.Cluster.check_cluster()
#self.test = True
if self.cluster_use:
self.test = False
else:
self.test = True
#this is where I have chosen to place my results
self.labelit_results = {}
self.labelit_summary = False
#Labelit settings
self.twotheta = False
self.multiproc = True
self.labelit_jobs = {}
self.results = {}
self.pool = {}
self.new_input = {}
#Settings for all programs
self.x_beam = self.header.get('beam_center_x')
self.y_beam = self.header.get('beam_center_y')
self.working_dir = self.setup.get('work')
self.sg = self.header.get('spacegroup', 'None')
self.distance = os.path.basename(self.working_dir)
self.vendortype = self.header.get('vendortype')
Process.__init__(self, name='FindBeamCenter')
self.start()
#logger.addHandler(handler)
# Set up logging
logger = utils.log.get_logger(
logfile_dir="/gpfs6/users/necat/Jon/RAPD_test/Output",
logfile_id="rapd_mr",
level=1,
#console=commandline_args.test
console=False)
# Setup cluster
import sites.necat as site
from utils.modules import load_module
cluster_launcher = load_module(site.CLUSTER_ADAPTER)
launcher = cluster_launcher.process_cluster
# Setup local_subprocess
#launcher = local_subprocess
# Setup redis
redis_database = importlib.import_module('database.redis_adapter')
#redis_database = redis_database.Database(settings=site.CONTROL_DATABASE_SETTINGS)
redis = redis_database.Database(settings=site.CONTROL_DATABASE_SETTINGS)
#redis = redis_database.connect_to_redis()
"""
cif = '/gpfs6/users/necat/rapd2/integrate/2018-06-06/JDO_PUCK2_A14_Run4_1/rapd_pdbquery_JDO_PUCK2_A14_Run4_1_free/Phaser_1Z7E/1z7e.cif'
l = ['2FGE_E', '2FGE']
for i in l:
print i.split('_')[0]
print len(i.split('_'))
#from multiprocessing import Process
#from multiprocessing import Queue as mp_Queue
#from Queue import Queue as t_Queue
from threading import Thread
import json
# RAPD imports
#import utils.launch_tools as launch_tools
from utils.modules import load_module
#from utils.text import json
#from bson.objectid import ObjectId
command_file = '/gpfs6/users/necat/rapd2/command_files/INTEGRATE__ncmuo.rapd'
message = json.loads(open(command_file, 'r').read())
cluster = load_module(seek_module="sites.cluster.necat")
# Get the new working directory
work_dir = message["directories"]["work"]
# Set the site tag from input
site_tag = 'NECAT'
# The command to launch the job
command_line = "rapd.launch -vs %s %s" % (site_tag, command_file)
# Parse a label for qsub job from the command_file name
qsub_label = os.path.basename(command_file).replace(".rapd", "")
nproc = 1
queue = 'phase1.q,general.q'
def __init__(self, inp, output, logger=None):
logger.info('FindBeamCenter.__init__')
self.st = time.time()
self.input,self.cpu,self.verbose,self.clean = inp
self.output = output
self.logger = logger
# Store passed-in variables
#self.site = site
#self.command = command
self.reply_address = self.input["return_address"]
# Setting up data input
self.setup = self.input["directories"]
self.header = self.input["header1"]
self.header2 = self.input.get("header2", False)
self.site_parameters = self.input.get("site_parameters")
self.preferences = self.input.get("preferences", {})
self.controller_address = self.reply_address
# Setup the weird named rapd_agent_index+strategy pipeline
self.Labelit = load_module(seek_module="rapd_agent_index+strategy",
directories=('agents'),
logger=self.logger)
# Setup the proper computer cluster for site.
if self.site_parameters.get('cluster_site', False):
self.Cluster = load_module(seek_module=self.site_parameters.get('cluster_site') ,
directories=('sites.cluster'),
logger=self.logger)
else:
self.Cluster = load_module(seek_module=self.site_parameters.get('img_site') ,
directories=('sites.cluster'),
logger=self.logger)
#Number of labelit runs to fine tune beam center and spacing from original
#Should be even number for symmetry around original beam center
self.iterations2 = ITERATIONS
self.iterations2_space = 0.3
#self.iterations2 = 15
#self.iterations2_space = 0.1
#Sets the limit of the std dev (mm) of either x or y beam position in the top 10% of the solutions.
#self.std_dev = 0.1
self.std_dev = 0.25
#Sets settings to check output on my machine and does not send results to database.
self.cluster_use = self.Cluster.check_cluster()
#self.test = True
if self.cluster_use:
self.test = False
else:
self.test = True
#this is where I have chosen to place my results
self.labelit_results = {}
self.labelit_summary = False
#Labelit settings
self.twotheta = False
self.multiproc = True
self.labelit_jobs = {}
self.results = {}
self.pool = {}
self.new_input = {}
#Settings for all programs
self.x_beam = self.header.get('beam_center_x')
self.y_beam = self.header.get('beam_center_y')
self.working_dir = self.setup.get('work')
self.sg = self.header.get('spacegroup','None')
self.distance = os.path.basename(self.working_dir)
self.vendortype = self.header.get('vendortype')
Process.__init__(self,name='FindBeamCenter')
self.start()
def main():
"""
The main process
Setup logging and instantiate the model"""
# Get the commandline args
commandline_args = get_commandline()
# Output log file is always verbose
log_level = 10
# Set up logging
logger = utils.log.get_logger(logfile_dir="./",
logfile_id="rapd_pdbquery",
level=log_level,
console=commandline_args.test)
# Set up terminal printer
# Verbosity
if commandline_args.json:
terminal_log_level = 100
elif commandline_args.verbose:
terminal_log_level = 10
else:
terminal_log_level = 50
tprint = utils.log.get_terminal_printer(verbosity=terminal_log_level,
no_color=commandline_args.no_color,
progress=commandline_args.progress)
print_welcome_message(tprint)
logger.debug("Commandline arguments:")
tprint(arg="\nCommandline arguments:", level=10, color="blue")
for pair in commandline_args._get_kwargs():
logger.debug(" arg:%s val:%s", pair[0], pair[1])
tprint(arg=" arg:%-20s val:%s" % (pair[0], pair[1]), level=10, color="white")
# Get the environmental variables
environmental_vars = utils.site.get_environmental_variables()
logger.debug("" + text.info + "Environmental variables" + text.stop)
tprint("\nEnvironmental variables", level=10, color="blue")
for key, val in environmental_vars.iteritems():
logger.debug(" " + key + " : " + val)
tprint(arg=" arg:%-20s val:%s" % (key, val), level=10, color="white")
# Should working directory go up or down?
if environmental_vars.get("RAPD_DIR_INCREMENT") in ("up", "UP"):
commandline_args.dir_up = True
else:
commandline_args.dir_up = False
# Construct the command
command = construct_command(commandline_args=commandline_args)
# Load the plugin
plugin = modules.load_module(seek_module="plugin",
directories=["plugins.pdbquery"],
logger=logger)
# Print out plugin info
tprint(arg="\nPlugin information", level=10, color="blue")
tprint(arg=" Plugin type: %s" % plugin.PLUGIN_TYPE, level=10, color="white")
tprint(arg=" Plugin subtype: %s" % plugin.PLUGIN_SUBTYPE, level=10, color="white")
tprint(arg=" Plugin version: %s" % plugin.VERSION, level=10, color="white")
tprint(arg=" Plugin id: %s" % plugin.ID, level=10, color="white")
# Run the plugin
plugin.RapdPlugin(command, tprint, logger)
def main():
"""
The main process
Setup logging, gather information, and run the plugin
"""
# Get the commandline args
commandline_args = get_commandline()
# Output log file is always verbose
log_level = 10
# Set up logging
logger = utils.log.get_logger(logfile_dir="./",
logfile_id="rapd_integrate",
level=log_level,
console=commandline_args.test)
# Set up terminal printer
# Verbosity
if commandline_args.verbose:
terminal_log_level = 10
elif commandline_args.json:
terminal_log_level = 100
else:
terminal_log_level = 30
tprint = utils.log.get_terminal_printer(verbosity=terminal_log_level,
no_color=commandline_args.no_color,
progress=commandline_args.progress)
print_welcome_message(tprint)
# Print out commandline arguments
logger.debug("Commandline arguments:")
tprint(arg="\nCommandline arguments:", level=10, color="blue")
for pair in commandline_args._get_kwargs():
logger.debug(" arg:%s val:%s", pair[0], pair[1])
tprint(arg=" arg:%-20s val:%s" % (pair[0], pair[1]), level=10, color="white")
# Get the environmental variables
environmental_vars = utils.site.get_environmental_variables()
logger.debug("\n" + text.info + "Environmental variables" + text.stop)
tprint("\nEnvironmental variables", level=10, color="blue")
for key, val in environmental_vars.iteritems():
logger.debug(" " + key + " : " + val)
tprint(arg=" arg:%-20s val:%s" % (key, val), level=10, color="white")
# Should working directory go up or down?
if environmental_vars.get("RAPD_DIR_INCREMENT") in ("up", "UP"):
commandline_args.dir_up = True
else:
commandline_args.dir_up = False
# List sites?
if commandline_args.listsites:
tprint(arg="\nAvailable sites", level=99, color="blue")
commandline_utils.print_sites(left_buffer=" ")
if not commandline_args.listdetectors:
sys.exit()
# List detectors?
if commandline_args.listdetectors:
print "\n" + text.info + "Available detectors:" + text.stop
commandline_utils.print_detectors(left_buffer=" ")
sys.exit()
# Look for data based on the input template
data_files = commandline_utils.analyze_data_sources(
commandline_args.template,
mode="integrate",
start_image=commandline_args.start_image,
end_image=commandline_args.end_image,
timeout=3)
# Change hdf5 to cbf
if "hdf5_files" in data_files:
logger.debug("HDF5 source file(s)")
tprint(arg="\nHDF5 source file(s)", level=99, color="blue")
logger.debug(data_files["hdf5_files"])
for data_file in data_files["hdf5_files"]:
tprint(arg=" " + data_file, level=99, color="white")
logger.debug("CBF file(s) from HDF5 file(s)")
tprint(arg="\nData files", level=99, color="blue")
# logger.debug("Data to be integrated")
# tprint(arg="\nData to be integrated", level=99, color="blue")
# tprint(arg=" From %s" % data_files["data_files"][0], level=99, color="white")
# tprint(arg=" To %s" % data_files["data_files"][-1], level=99, color="white")
# No images match - assume that images will be arriving soon
if len(data_files) == 0 and commandline_args.test == False:
raise Exception("No files found for integration.")
# Get site - commandline wins over the environmental variable
site = False
site_module = False
detector = {}
detector_module = False
if commandline_args.site:
site = commandline_args.site
elif environmental_vars.has_key("RAPD_SITE"):
site = environmental_vars["RAPD_SITE"]
if commandline_args.detector:
detector = commandline_args.detector
detector_module = detector_utils.load_detector(detector)
"""
# If no site or detector, try to figure out the detector
if not (site or detector):
detector = detector_utils.get_detector_file(data_files["data_files"][0])
if isinstance(detector, dict):
if detector.has_key("site"):
site_target = detector.get("site")
site_file = utils.site.determine_site(site_arg=site_target)
# print site_file
site_module = importlib.import_module(site_file)
detector_target = site_module.DETECTOR.lower()
detector_module = detector_utils.load_detector(detector_target)
elif detector.has_key("detector"):
site_module = False
detector_target = detector.get("detector")
detector_module = detector_utils.load_detector(detector_target)
"""
if not detector:
detector = detector_utils.get_detector_file(data_files["data_files"][0])
if isinstance(detector, dict):
if detector.has_key("site"):
site_target = detector.get("site")
site_file = utils.site.determine_site(site_arg=site_target)
site_module = importlib.import_module(site_file)
detector_target = site_module.DETECTOR.lower()
detector_module = detector_utils.load_detector(detector_target)
elif detector.has_key("detector"):
site_module = False
detector_target = detector.get("detector")
detector_module = detector_utils.load_detector(detector_target)
# If someone specifies the site or found in env.
if site and not site_module:
site_file = utils.site.determine_site(site_arg=site)
site_module = importlib.import_module(site_file)
# Have a detector - read in file data
if not detector_module:
raise Exception("No detector identified")
# Get header information
image_0_data = get_image_data(data_file=data_files["data_files"][0],
detector_module=detector_module,
site_module=site_module,
site=site)
image_n_data = get_image_data(data_file=data_files["data_files"][-1],
detector_module=detector_module,
site_module=site_module,
site=site)
# # Have an end image set
# if commandline_args.end_image:
# # End image not yet present
# if image_n_data["image_number"] < commandline_args.end_image:
# pass
# # End image present
# else:
# pass
# # No end image set
# else:
# pass
logger.debug("First image header: %s", image_0_data)
tprint(arg="\nFirst image header", level=10, color="blue")
keys = image_0_data.keys()
keys.sort()
tprint(arg=" %s" % image_0_data["fullname"], level=10, color="white")
for key in keys:
tprint(arg=" arg:%-22s val:%s" % (key, image_0_data[key]),
level=10,
color="white")
# Get the run data
run_data = get_run_data(detector_module, image_0_data, image_n_data, commandline_args)
logger.debug("Run data: %s", run_data)
tprint(arg="\nRun data", level=10, color="blue")
keys = run_data.keys()
keys.sort()
for key in keys:
tprint(arg=" arg:%-22s val:%s" % (key, run_data[key]), level=10, color="white")
# Construct the command for the plugin
command = construct_command(image_0_data,
run_data,
commandline_args,
detector_module)
# Load the plugin
plugin = load_module(seek_module="plugin",
directories=["plugins.integrate"],
logger=logger)
tprint(arg="\nPlugin information", level=10, color="blue")
tprint(arg=" Plugin type: %s" % plugin.PLUGIN_TYPE, level=10, color="white")
tprint(arg=" Plugin subtype: %s" % plugin.PLUGIN_SUBTYPE, level=10, color="white")
tprint(arg=" Plugin version: %s" % plugin.VERSION, level=10, color="white")
tprint(arg=" Plugin id: %s" % plugin.ID, level=10, color="white")
# Instantiate the plugin
plugin_instance = plugin.RapdPlugin(site_module,
command=command,
tprint=tprint,
logger=logger)
plugin_instance.start()
def main():
"""
The main process
Setup logging and instantiate the model"""
# Get the commandline args
commandline_args = get_commandline()
# Output log file is always verbose
log_level = 10
# Set up logging
logger = utils.log.get_logger(logfile_dir="./",
logfile_id="rapd_pdbquery",
level=log_level,
console=commandline_args.test)
# Set up terminal printer
# Verbosity
if commandline_args.json:
terminal_log_level = 100
elif commandline_args.verbose:
terminal_log_level = 10
else:
terminal_log_level = 50
tprint = utils.log.get_terminal_printer(verbosity=terminal_log_level,
no_color=commandline_args.no_color,
progress=commandline_args.progress)
print_welcome_message(tprint)
logger.debug("Commandline arguments:")
tprint(arg="\nCommandline arguments:", level=10, color="blue")
for pair in commandline_args._get_kwargs():
logger.debug(" arg:%s val:%s", pair[0], pair[1])
tprint(arg=" arg:%-20s val:%s" % (pair[0], pair[1]),
level=10,
color="white")
# Get the environmental variables
environmental_vars = utils.site.get_environmental_variables()
logger.debug("" + text.info + "Environmental variables" + text.stop)
tprint("\nEnvironmental variables", level=10, color="blue")
for key, val in environmental_vars.iteritems():
logger.debug(" " + key + " : " + val)
tprint(arg=" arg:%-20s val:%s" % (key, val), level=10, color="white")
# Should working directory go up or down?
if environmental_vars.get("RAPD_DIR_INCREMENT") in ("up", "UP"):
commandline_args.dir_up = True
else:
commandline_args.dir_up = False
# Construct the command
command = construct_command(commandline_args=commandline_args)
#print command
# Load the plugin
plugin = modules.load_module(seek_module="plugin",
directories=["plugins.pdbquery"],
logger=logger)
# Print out plugin info
tprint(arg="\nPlugin information", level=10, color="blue")
tprint(arg=" Plugin type: %s" % plugin.PLUGIN_TYPE,
level=10,
color="white")
tprint(arg=" Plugin subtype: %s" % plugin.PLUGIN_SUBTYPE,
level=10,
color="white")
tprint(arg=" Plugin version: %s" % plugin.VERSION,
level=10,
color="white")
tprint(arg=" Plugin id: %s" % plugin.ID, level=10, color="white")
# Run the plugin
P = plugin.RapdPlugin(site=False,
command=command,
processed_results=False,
tprint=tprint,
logger=logger,
verbosity=commandline_args.verbose)
P.run()
def main():
""" The main process
Setup logging and instantiate the model"""
# Get the commandline args
commandline_args = get_commandline()
# Output log file is always verbose
log_level = 10
# Set up logging
if commandline_args.logging:
logger = utils.log.get_logger(logfile_dir="./",
logfile_id="rapd_index",
level=log_level,
console=commandline_args.test)
# Set up terminal printer
# Verbosity
if commandline_args.verbose:
terminal_log_level = 10
elif commandline_args.json:
terminal_log_level = 100
elif commandline_args.json_fd:
terminal_log_level = 100
else:
terminal_log_level = 30
tprint = utils.log.get_terminal_printer(verbosity=terminal_log_level,
no_color=commandline_args.no_color,
progress=commandline_args.progress,
progress_fd=commandline_args.progress_fd)
print_welcome_message(tprint)
logger.debug("Commandline arguments:")
tprint(arg="\nCommandline arguments:", level=10, color="blue")
for pair in commandline_args._get_kwargs():
logger.debug(" arg:%s val:%s", pair[0], pair[1])
tprint(arg=" arg:%-20s val:%s" % (pair[0], pair[1]), level=10, color="white")
# Get the environmental variables
environmental_vars = utils.site.get_environmental_variables()
logger.debug("\n" + text.info + "Environmental variables" + text.stop)
tprint("\nEnvironmental variables", level=10, color="blue")
for key, val in environmental_vars.iteritems():
logger.debug(" " + key + " : " + val)
tprint(arg=" arg:%-20s val:%s" % (key, val), level=10, color="white")
# Should working directory go up or down?
if environmental_vars.get("RAPD_DIR_INCREMENT") in ("up", "UP"):
commandline_args.dir_up = True
else:
commandline_args.dir_up = False
# List sites?
if commandline_args.listsites:
tprint(arg="\nAvailable sites", level=98, color="blue")
commandline_utils.print_sites(left_buffer=" ")
if not commandline_args.listdetectors:
sys.exit()
# List detectors?
if commandline_args.listdetectors:
tprint(arg="Available detectors", level=98, color="blue")
commandline_utils.print_detectors(left_buffer=" ")
sys.exit()
# Get the data files
data_files = commandline_utils.analyze_data_sources(sources=commandline_args.sources,
mode="index",
hdf5_image_range=commandline_args.hdf5_image_range,
hdf5_wedge_range=commandline_args.hdf5_wedge_range)
if "hdf5_files" in data_files:
logger.debug("HDF5 source file(s)")
tprint(arg="\nHDF5 source file(s)", level=98, color="blue")
logger.debug(data_files["hdf5_files"])
for data_file in data_files["hdf5_files"]:
tprint(arg=" " + data_file, level=98, color="white")
logger.debug("CBF file(s) from HDF5 file(s)")
tprint(arg="\nData files", level=98, color="blue")
else:
logger.debug("Data file(s)")
tprint(arg="\nData file(s)", level=98, color="blue")
if len(data_files) == 0:
tprint(arg=" None", level=98, color="white")
else:
logger.debug(data_files["files"])
for data_file in data_files["files"]:
tprint(arg=" " + data_file, level=98, color="white")
# Need data
if len(data_files) == 0 and commandline_args.test == False:
if logger:
logger.exception("No files input for indexing.")
raise Exception, "No files input for indexing."
# Too much data?
if len(data_files) > 2:
if logger:
logger.exception("Too many files for indexing. 1 or 2 images accepted")
raise Exception, "Too many files for indexing. 1 or 2 images accepted"
# Get site - commandline wins over the environmental variable
site = False
site_module = False
#detector = {}
detector = False
detector_module = False
if commandline_args.site:
site = commandline_args.site
elif environmental_vars.has_key("RAPD_SITE"):
site = environmental_vars["RAPD_SITE"]
# Detector is defined by the user
if commandline_args.detector:
detector = commandline_args.detector
detector_module = detector_utils.load_detector(detector)
# If no site or detector, try to figure out the detector
# if not (site or detector):
if not detector:
detector = detector_utils.get_detector_file(data_files["files"][0])
if isinstance(detector, dict):
if detector.has_key("site"):
site_target = detector.get("site")
site_file = utils.site.determine_site(site_arg=site_target)
site_module = importlib.import_module(site_file)
detector_target = site_module.DETECTOR.lower()
detector_module = detector_utils.load_detector(detector_target)
elif detector.has_key("detector"):
site_module = False
detector_target = detector.get("detector")
detector_module = detector_utils.load_detector(detector_target)
# If someone specifies the site or found in env.
if site and not site_module:
site_file = utils.site.determine_site(site_arg=site)
site_module = importlib.import_module(site_file)
# Have a detector - read in file data
if detector_module:
image_headers = {}
for index, data_file in enumerate(data_files["files"]):
if site_module:
image_headers[data_file] = detector_module.read_header(data_file,
#site_module.BEAM_SETTINGS)
site_module.BEAM_INFO.get(site.upper(), {}))
else:
image_headers[data_file] = detector_module.read_header(data_file)
# If this image is derived from an hdf5 master file, tag it
if "hdf5_files" in data_files:
if len(data_files["hdf5_files"]) == len(data_files["files"]):
image_headers[data_file]["hdf5_source"] = data_files["hdf5_files"][index]
else:
image_headers[data_file]["hdf5_source"] = data_files["hdf5_files"][0]
logger.debug("Image headers: %s", image_headers)
print_headers(tprint, image_headers)
command = construct_command(image_headers=image_headers,
commandline_args=commandline_args,
detector_module=detector_module)
else:
if logger:
logger.exception("No detector module found")
raise Exception("No detector module found")
# If no site, error
# if site == False:
# print text.error+"Could not determine a site. Exiting."+text.stop
# sys.exit(9)
# Determine the site_file
# site_file = utils.site.determine_site(site_arg=site)
# Error out if no site_file to import
# if site_file == False:
# print text.error+"Could not find a site file. Exiting."+text.stop
# sys.exit(9)
# Import the site settings
# print "Importing %s" % site_file
# site_module = importlib.import_module(site_file)
# Single process lock?
# utils.lock.file_lock(site_module.CONTROL_LOCK_FILE)
# Instantiate the plugin
# Load the plugin from directories defined in site file
# for d in sys.path:
# if d.endswith("src"):
# toplevel_dir = d+".plugins"
plugin = load_module(seek_module="plugin",
directories=["plugins.index"],
logger=logger)
tprint(arg="\nPlugin information", level=10, color="blue")
tprint(arg=" Plugin type: %s" % plugin.PLUGIN_TYPE, level=10, color="white")
tprint(arg=" Plugin subtype: %s" % plugin.PLUGIN_SUBTYPE, level=10, color="white")
tprint(arg=" Plugin version: %s" % plugin.VERSION, level=10, color="white")
tprint(arg=" Plugin id: %s" % plugin.ID, level=10, color="white")
#plugin_instance = plugin.RapdPlugin(None, command, tprint, logger)
plugin_instance = plugin.RapdPlugin(site_module, command, tprint, logger)
plugin_instance.start()
def main():
"""
The main process
Setup logging and instantiate the model"""
# Get the commandline args
commandline_args = get_commandline()
# Output log file is always verbose
log_level = 10
# Set up logging
logger = utils.log.get_logger(logfile_dir="./",
logfile_id="rapd_echo",
level=log_level,
console=commandline_args.test)
# Set up terminal printer
# tprint prints to the terminal, taking the arguments
# arg - the string to be printed (if using level of "progress" this needs to be an int)
# level - value 0 to 99 or "progress". Do NOT use a value of 100 or above
# 50 - alert
# 40 - error
# 30 - warning
# 20 - info
# 10 - debug
# color - color to print ("red" and so forth. See list in utils/text)
# newline - put in False if you don't want a newline at the end of your print
#
# Verbosity
if commandline_args.verbose:
terminal_log_level = 10
elif commandline_args.json:
terminal_log_level = 100
else:
terminal_log_level = 50
tprint = utils.log.get_terminal_printer(verbosity=terminal_log_level,
no_color=commandline_args.no_color,
progress=commandline_args.progress)
print_welcome_message(tprint)
logger.debug("Commandline arguments:")
tprint(arg="\nCommandline arguments:", level=10, color="blue")
for pair in commandline_args._get_kwargs():
logger.debug(" arg:%s val:%s", pair[0], pair[1])
tprint(arg=" arg:%-20s val:%s" % (pair[0], pair[1]),
level=10,
color="white")
# Get the environmental variables
environmental_vars = utils.site.get_environmental_variables()
logger.debug("" + text.info + "Environmental variables" + text.stop)
tprint("\nEnvironmental variables", level=10, color="blue")
for key, val in environmental_vars.iteritems():
logger.debug(" " + key + " : " + val)
tprint(arg=" arg:%-20s val:%s" % (key, val), level=10, color="white")
# Should working directory go up or down?
if environmental_vars.get("RAPD_DIR_INCREMENT") in ("up", "UP"):
commandline_args.dir_up = True
else:
commandline_args.dir_up = False
# Construct the command
command = construct_command(commandline_args=commandline_args,
logger=logger)
# Load the plugin
plugin = modules.load_module(seek_module="plugin",
directories=["plugins.echo"],
logger=logger)
# Print plugin info
tprint(arg="\nPlugin information", level=10, color="blue")
tprint(arg=" Plugin type: %s" % plugin.PLUGIN_TYPE,
level=10,
color="white")
tprint(arg=" Plugin subtype: %s" % plugin.PLUGIN_SUBTYPE,
level=10,
color="white")
tprint(arg=" Plugin version: %s" % plugin.VERSION,
level=10,
color="white")
tprint(arg=" Plugin id: %s" % plugin.ID, level=10, color="white")
# Run the plugin
plugin.RapdPlugin(command, tprint, logger)
def main():
"""
The main process
Setup logging and instantiate the model"""
# Get the commandline args
commandline_args = get_commandline()
# Output log file is always verbose
log_level = 10
# Set up logging
logger = utils.log.get_logger(logfile_dir="./",
logfile_id="rapd_get_cif",
level=log_level,
console=commandline_args.test)
# Set up terminal printer
# Verbosity
if commandline_args.verbose:
terminal_log_level = 10
elif commandline_args.json:
terminal_log_level = 100
else:
terminal_log_level = 50
tprint = utils.log.get_terminal_printer(verbosity=terminal_log_level,
no_color=commandline_args.no_color)
print_welcome_message(tprint)
logger.debug("Commandline arguments:")
tprint(arg="\nCommandline arguments:", level=10, color="blue")
for pair in commandline_args._get_kwargs():
logger.debug(" arg:%s val:%s", pair[0], pair[1])
tprint(arg=" arg:%-20s val:%s" % (pair[0], pair[1]), level=10, color="white")
# Get the environmental variables
environmental_vars = utils.site.get_environmental_variables()
logger.debug("" + text.info + "Environmental variables" + text.stop)
tprint("\nEnvironmental variables", level=10, color="blue")
for key, val in environmental_vars.iteritems():
logger.debug(" " + key + " : " + val)
tprint(arg=" arg:%-20s val:%s" % (key, val), level=10, color="white")
# Construct the command
command = construct_command(commandline_args=commandline_args,
logger=logger)
# Load the plugin
plugin = modules.load_module(seek_module="plugin",
directories=["plugins.get_cif"],
logger=logger)
# Print plugin info
tprint(arg="\nPlugin information", level=10, color="blue")
tprint(arg=" Plugin type: %s" % plugin.PLUGIN_TYPE, level=10, color="white")
tprint(arg=" Plugin subtype: %s" % plugin.PLUGIN_SUBTYPE, level=10, color="white")
tprint(arg=" Plugin version: %s" % plugin.VERSION, level=10, color="white")
tprint(arg=" Plugin id: %s" % plugin.ID, level=10, color="white")
# Run the plugin
plugin.RapdPlugin(command, tprint, logger)