def init_detectors(self): """Set up the detectors""" self.logger.debug("Setting up the detectors") # Shorten variable names site = self.site import sites.detectors.necat_dectris_eiger16m # A single detector if site.DETECTOR: #detector, suffix = site.DETECTOR detector = site.DETECTOR detector = detector.lower() self.detectors[self.site_ids[0].upper()] = load_module( seek_module=detector, directories=("sites.detectors", "detectors"), logger=self.logger) # Multiple detectors elif site.DETECTORS: for site_id in self.site_ids: detector, suffix = site.DETECTORS[site_id] detector = detector.lower() self.detectors[site_id.upper()] = load_module( seek_module=detector, directories=("sites.detectors", "detectors"))
def run(self): """ Orchestrate the adapter's actions """ # Check if command is ECHO if self.message['command'] == 'ECHO': # Load the simple_echo module echo = load_module( seek_module='launch.launcher_adapters.echo_simple') # send message to simple_echo echo.LauncherAdapter(self.site, self.message, self.settings) else: # Adjust the message to this site #self.fix_command() self.message = launch_tools.fix_command(self.message) # Put the command into a file #command_file = launch_tools.write_command_file(self.settings["launch_dir"], command_file = launch_tools.write_command_file( os.path.join(self.settings["launch_dir"], 'command_files'), self.message["command"], self.message) # Set the site tag from input site_tag = launch_tools.get_site_tag(self.message).split('_')[0] # Call the launch process on the command file self.logger.debug("rapd.launch -s %s %s", site_tag, command_file) Popen(["rapd.launch", "-s", site_tag, command_file])
def run(self): """ Orchestrate the adapter's actions """ # Check if command is ECHO if self.message['command'] == 'ECHO': # Load the simple_echo module echo = load_module(seek_module='launch.launcher_adapters.echo_simple') # send message to simple_echo echo.LauncherAdapter(self.site, self.message, self.settings) else: # Adjust the message to this site #self.fix_command() self.message = launch_tools.fix_command(self.message) # Put the command into a file command_file = launch_tools.write_command_file(self.settings["launch_dir"], self.message["command"], self.message) # Set the site tag from input site_tag = launch_tools.get_site_tag(self.message).split('_')[0] # Call the launch process on the command file self.logger.debug("rapd.launch -s %s %s", site_tag, command_file) Popen(["rapd.launch", "-s",site_tag, command_file])
def process_labelit(self, inp=False): """ Initiate Labelit runs. """ if self.verbose: self.logger.debug("LabelitPP::process_labelit") try: queue = Queue() params = {} params["test"] = self.test params["cluster"] = False # self.cluster_use # TODO params["verbose"] = self.verbose args1 = {} if inp: args1["input"] = inp else: args1["input"] = self.images args1["output"] = queue args1["params"] = params args1["logger"] = self.logger # Import the RunLabelit class agent = load_module(seek_module="rapd_agent_index+strategy", directories=self.agent_directories, logger=self.logger) Process(target=agent.RunLabelit, kwargs=args1).start() self.labelit_results = queue.get() self.labelit_log = queue.get() except: self.logger.exception("**Error in LabelitPP.process_labelit**")
def load_agent(self, command): """Load the agent file for this command""" # Agent we are looking for seek_module = "rapd_agent_%s" % command.lower() self.agent = load_module(seek_module=seek_module, directories=self.site.RAPD_AGENT_DIRECTORIES)
def load_adapter(self): """Find and load the adapter""" # Import the database adapter as database module self.adapter = load_module( seek_module=self.launcher["adapter"], directories=self.site.LAUNCHER_SETTINGS["RAPD_LAUNCHER_ADAPTER_DIRECTORIES"]).LauncherAdapter if self.logger: self.logger.debug(self.adapter)
def load_adapter(self): """Find and load the adapter""" # Import the database adapter as database module self.adapter = load_module( seek_module=self.specifications["adapter"], directories=self.site.LAUNCHER_SETTINGS[ "RAPD_LAUNCHER_ADAPTER_DIRECTORIES"]).LauncherAdapter self.logger.debug(self.adapter)
def load_cluster(image_dir): """ Load the computer cluster module so the job can be submitted. """ # Determine the site and load cluster. May be different from where images are from... # Check the images imgs = sort_dir(image_dir) site = Utils.get_site(imgs[0],False) cluster_site_name = os.getenv('RAPD_SITE1', False) # Determine the site by env variable first. if cluster_site_name: cluster_site_name = cluster_site_name.lower() cluster_site = load_module(seek_module=cluster_site_name, directories='sites') else: # Otherwise use the image origin as site. cluster_site = load_module(seek_module=site[0], directories='sites') cluster = load_module(cluster_site.CLUSTER_ADAPTER) out = (site[1], site[0], cluster_site_name, imgs) return ((out, cluster))
def load_cluster(image_dir): """ Load the computer cluster module so the job can be submitted. """ # Determine the site and load cluster. May be different from where images are from... # Check the images imgs = sort_dir(image_dir) site = Utils.get_site(imgs[0], False) cluster_site_name = os.getenv('RAPD_SITE1', False) # Determine the site by env variable first. if cluster_site_name: cluster_site_name = cluster_site_name.lower() cluster_site = load_module(seek_module=cluster_site_name, directories='sites') else: # Otherwise use the image origin as site. cluster_site = load_module(seek_module=site[0], directories='sites') cluster = load_module(cluster_site.CLUSTER_ADAPTER) out = (site[1], site[0], cluster_site_name, imgs) return ((out, cluster))
def load_plugin(self, command): """ Load the plugin file for this command Keyword arguments command -- the command to be run (index+strategy for example) """ # plugin directories we are looking for directories = [] for directory in self.site.RAPD_PLUGIN_DIRECTORIES: directories.append(directory+".%s" % self.command.get("command").lower()) # Load the plugin from directories defined in site file self.plugin = load_module(seek_module="plugin", directories=directories, logger=self.new_logger)
def load_plugin(self, command): """ Load the plugin file for this command Keyword arguments command -- the command to be run (index+strategy for example) """ # plugin directories we are looking for directories = [] for directory in self.site.RAPD_PLUGIN_DIRECTORIES: directories.append(directory + ".%s" % self.command.get("command").lower()) # Load the plugin from directories defined in site file self.plugin = load_module(seek_module="plugin", directories=directories, logger=self.new_logger)
def Launcher_Adapter(site, message, settings): # Check if command is ECHO if message['command'] == 'ECHO': # Load the simple_echo module echo = load_module(seek_module='launch.launcher_adapters.echo_simple') # send message to simple_echo echo.LauncherAdapter(site, message, settings) else: # Adjust the message to this site #self.fix_command() message = launch_tools.fix_command(message) # Put the command into a file command_file = launch_tools.write_command_file(settings["launch_dir"], message["command"], message) # Set the site tag from input site_tag = launch_tools.get_site_tag(message).split('_')[0] # Call the launch process on the command file #self.logger.debug("rapd.launch -s %s %s", site_tag, command_file) Popen(["rapd.launch", "-s", site_tag, command_file], shell=True)
def main(): """ The main process Setup logging, gather information, and run the plugin """ # Get the commandline args commandline_args = get_commandline() # Output log file is always verbose log_level = 10 # Set up logging logger = utils.log.get_logger(logfile_dir="./", logfile_id="rapd_integrate", level=log_level, console=commandline_args.test) # Set up terminal printer # Verbosity if commandline_args.verbose: terminal_log_level = 10 elif commandline_args.json: terminal_log_level = 100 else: terminal_log_level = 50 tprint = utils.log.get_terminal_printer(verbosity=terminal_log_level, no_color=commandline_args.no_color) print_welcome_message(tprint) # Print out commandline arguments logger.debug("Commandline arguments:") tprint(arg="\nCommandline arguments:", level=10, color="blue") for pair in commandline_args._get_kwargs(): logger.debug(" arg:%s val:%s", pair[0], pair[1]) tprint(arg=" arg:%-20s val:%s" % (pair[0], pair[1]), level=10, color="white") # Get the environmental variables environmental_vars = utils.site.get_environmental_variables() logger.debug("\n" + text.info + "Environmental variables" + text.stop) tprint("\nEnvironmental variables", level=10, color="blue") for key, val in environmental_vars.iteritems(): logger.debug(" " + key + " : " + val) tprint(arg=" arg:%-20s val:%s" % (key, val), level=10, color="white") # List sites? if commandline_args.listsites: tprint(arg="\nAvailable sites", level=99, color="blue") commandline_utils.print_sites(left_buffer=" ") if not commandline_args.listdetectors: sys.exit() # List detectors? if commandline_args.listdetectors: print "\n" + text.info + "Available detectors:" + text.stop commandline_utils.print_detectors(left_buffer=" ") sys.exit() # Look for data based on the input template data_files = commandline_utils.analyze_data_sources( commandline_args.template, mode="integrate", start_image=commandline_args.start_image, end_image=commandline_args.end_image) if "hdf5_files" in data_files: logger.debug("HDF5 source file(s)") tprint(arg="\nHDF5 source file(s)", level=99, color="blue") logger.debug(data_files["hdf5_files"]) for data_file in data_files["hdf5_files"]: tprint(arg=" " + data_file, level=99, color="white") logger.debug("CBF file(s) from HDF5 file(s)") tprint(arg="\nData files", level=99, color="blue") else: logger.debug("Data to be integrated") tprint(arg="\nData to be integrated", level=99, color="blue") tprint(arg=" From %s" % data_files["data_files"][0], level=99, color="white") tprint(arg=" To %s" % data_files["data_files"][-1], level=99, color="white") # Need data if len(data_files) == 0 and commandline_args.test == False: raise Exception("No files input for integration.") # Get site - commandline wins over the environmental variable site = False site_module = False detector = {} detector_module = False if commandline_args.site: site = commandline_args.site elif environmental_vars.has_key("RAPD_SITE"): site = environmental_vars["RAPD_SITE"] if commandline_args.detector: detector = commandline_args.detector detector_module = detector_utils.load_detector(detector) # If no site or detector, try to figure out the detector if not (site or detector): detector = detector_utils.get_detector_file( data_files["data_files"][0]) if isinstance(detector, dict): if detector.has_key("site"): site_target = detector.get("site") site_file = utils.site.determine_site(site_arg=site_target) # print site_file site_module = importlib.import_module(site_file) detector_target = site_module.DETECTOR.lower() detector_module = detector_utils.load_detector(detector_target) elif detector.has_key("detector"): site_module = False detector_target = detector.get("detector") detector_module = detector_utils.load_detector(detector_target) # Have a detector - read in file data if not detector_module: raise Exception("No detector identified") # Get header information image_0_data = get_image_data(data_file=data_files["data_files"][0], detector_module=detector_module, site_module=site_module) image_n_data = get_image_data(data_file=data_files["data_files"][-1], detector_module=detector_module, site_module=site_module) logger.debug("Image header: %s, %s", image_0_data, image_n_data) tprint(arg="\nImage headers", level=10, color="blue") count = 0 for header in (image_0_data, image_n_data): keys = header.keys() keys.sort() if count > 0: tprint(arg="", level=10, color="white") tprint(arg=" %s" % header["fullname"], level=10, color="white") for key in keys: tprint(arg=" arg:%-22s val:%s" % (key, header[key]), level=10, color="white") count += 1 # Get the run data run_data = get_run_data(detector_module, image_0_data, image_n_data, commandline_args) logger.debug("Run data: %s", run_data) tprint(arg="\nRun data", level=10, color="blue") keys = run_data.keys() keys.sort() for key in keys: tprint(arg=" arg:%-22s val:%s" % (key, run_data[key]), level=10, color="white") # Construct the command for the plugin command = construct_command(image_0_data, run_data, commandline_args, detector_module) # Load the plugin plugin = load_module(seek_module="plugin", directories=["plugins.integrate"], logger=logger) tprint(arg="\nPlugin information", level=10, color="blue") tprint(arg=" Plugin type: %s" % plugin.PLUGIN_TYPE, level=10, color="white") tprint(arg=" Plugin subtype: %s" % plugin.PLUGIN_SUBTYPE, level=10, color="white") tprint(arg=" Plugin version: %s" % plugin.VERSION, level=10, color="white") tprint(arg=" Plugin id: %s" % plugin.ID, level=10, color="white") # Instantiate the plugin plugin.RapdPlugin(site=None, command=command, tprint=tprint, logger=logger)
def run(self): """ Orchestrate the adapter's actions """ if self.message['command'] == 'ECHO': echo = load_module(seek_module='launch.launcher_adapters.echo_simple') # send message to simple_echo echo.LauncherAdapter(self.site, self.message, self.settings) else: # Load the cluster adapter for the site. cluster = load_module(seek_module=self.site.CLUSTER_ADAPTER) # Adjust the message to this site # Get the new working directory. Message only has second part of the path. #self.message = cluster.fix_command(self.message) self.message = launch_tools.fix_command(self.message) # Get the new working directory work_dir = self.message["directories"]["work"] # Get the launcher directory - Add command_files to keep files isolated qsub_dir = self.message["directories"]["launch_dir"]+"/command_files" # Put the message into a rapd-readable file command_file = launch_tools.write_command_file(qsub_dir, self.message["command"], self.message) # Set the site tag from input site_tag = launch_tools.get_site_tag(self.message).split('_')[0] # The command to launch the job command_line = "rapd.launch -vs %s %s" % (site_tag, command_file) # Parse a label for qsub job from the command_file name qsub_label = os.path.basename(command_file).replace(".rapd", "") # Determine the number of precessors to request for job if hasattr(cluster, 'determine_nproc'): nproc = cluster.determine_nproc(self.message['command']) else: nproc = 1 # Determine which cluster queue to run if hasattr(cluster, 'check_queue'): queue = cluster.check_queue(self.message['command']) else: queue = False Thread(target=cluster.process_cluster, kwargs={'command':command_line, 'work_dir':work_dir, 'logfile':False, 'batch_queue':queue, 'nproc':nproc, 'logger':self.logger, 'name':qsub_label, 'mp_event':False, 'timeout':False, 'pid_queue':False, }).start() """
def main(): """ The main process Setup logging and instantiate the model""" # Get the commandline args commandline_args = get_commandline() # Output log file is always verbose log_level = 10 # Set up logging logger = utils.log.get_logger( logfile_dir="./", logfile_id="rapd_mr", level=log_level, #console=commandline_args.test console=False) # Set up terminal printer # Verbosity if commandline_args.json: terminal_log_level = 100 elif commandline_args.verbose: terminal_log_level = 10 else: terminal_log_level = 50 tprint = utils.log.get_terminal_printer(verbosity=terminal_log_level, no_color=commandline_args.no_color, progress=commandline_args.progress) print_welcome_message(tprint) logger.debug("Commandline arguments:") tprint(arg="\nCommandline arguments:", level=10, color="blue") for pair in commandline_args._get_kwargs(): logger.debug(" arg:%s val:%s", pair[0], pair[1]) tprint(arg=" arg:%-20s val:%s" % (pair[0], pair[1]), level=10, color="white") # Get the environmental variables environmental_vars = utils.site.get_environmental_variables() logger.debug("" + text.info + "Environmental variables" + text.stop) tprint("\nEnvironmental variables", level=10, color="blue") for key, val in environmental_vars.iteritems(): logger.debug(" " + key + " : " + val) tprint(arg=" arg:%-20s val:%s" % (key, val), level=10, color="white") # Should working directory go up or down? if environmental_vars.get("RAPD_DIR_INCREMENT") in ("up", "UP"): commandline_args.dir_up = True else: commandline_args.dir_up = False # Get site - commandline wins over the environmental variable site = False site_module = False if commandline_args.site: site = commandline_args.site elif environmental_vars.has_key("RAPD_SITE"): site = environmental_vars["RAPD_SITE"] # If someone specifies the site or found in env. if site and not site_module: site_file = utils.site.determine_site(site_arg=site) site_module = importlib.import_module(site_file) # Construct the command command = construct_command(commandline_args=commandline_args) # Load the plugin plugin = modules.load_module(seek_module="plugin", directories=["plugins.mr"], logger=logger) # Print out plugin info tprint(arg="\nPlugin information", level=10, color="blue") tprint(arg=" Plugin type: %s" % plugin.PLUGIN_TYPE, level=10, color="white") tprint(arg=" Plugin subtype: %s" % plugin.PLUGIN_SUBTYPE, level=10, color="white") tprint(arg=" Plugin version: %s" % plugin.VERSION, level=10, color="white") tprint(arg=" Plugin id: %s" % plugin.ID, level=10, color="white") # Run the plugin # Instantiate the plugin plugin_instance = plugin.RapdPlugin(command=command, site=site_module, tprint=tprint, logger=logger) plugin_instance.start()
def main(): """ The main process Setup logging and instantiate the model""" # Get the commandline args commandline_args = get_commandline() # Output log file is always verbose log_level = 10 # Set up logging if commandline_args.logging: logger = utils.log.get_logger(logfile_dir="./", logfile_id="rapd_index", level=log_level, console=commandline_args.test) # Set up terminal printer # Verbosity if commandline_args.verbose: terminal_log_level = 10 elif commandline_args.json: terminal_log_level = 100 elif commandline_args.json_fd: terminal_log_level = 100 else: terminal_log_level = 30 tprint = utils.log.get_terminal_printer(verbosity=terminal_log_level, no_color=commandline_args.no_color, progress=commandline_args.progress, progress_fd=commandline_args.progress_fd) print_welcome_message(tprint) logger.debug("Commandline arguments:") tprint(arg="\nCommandline arguments:", level=10, color="blue") for pair in commandline_args._get_kwargs(): logger.debug(" arg:%s val:%s", pair[0], pair[1]) tprint(arg=" arg:%-20s val:%s" % (pair[0], pair[1]), level=10, color="white") # Get the environmental variables environmental_vars = utils.site.get_environmental_variables() logger.debug("\n" + text.info + "Environmental variables" + text.stop) tprint("\nEnvironmental variables", level=10, color="blue") for key, val in environmental_vars.iteritems(): logger.debug(" " + key + " : " + val) tprint(arg=" arg:%-20s val:%s" % (key, val), level=10, color="white") # Should working directory go up or down? if environmental_vars.get("RAPD_DIR_INCREMENT") in ("up", "UP"): commandline_args.dir_up = True else: commandline_args.dir_up = False # List sites? if commandline_args.listsites: tprint(arg="\nAvailable sites", level=98, color="blue") commandline_utils.print_sites(left_buffer=" ") if not commandline_args.listdetectors: sys.exit() # List detectors? if commandline_args.listdetectors: tprint(arg="Available detectors", level=98, color="blue") commandline_utils.print_detectors(left_buffer=" ") sys.exit() # Get the data files data_files = commandline_utils.analyze_data_sources(sources=commandline_args.sources, mode="index", hdf5_image_range=commandline_args.hdf5_image_range, hdf5_wedge_range=commandline_args.hdf5_wedge_range) if "hdf5_files" in data_files: logger.debug("HDF5 source file(s)") tprint(arg="\nHDF5 source file(s)", level=98, color="blue") logger.debug(data_files["hdf5_files"]) for data_file in data_files["hdf5_files"]: tprint(arg=" " + data_file, level=98, color="white") logger.debug("CBF file(s) from HDF5 file(s)") tprint(arg="\nData files", level=98, color="blue") else: logger.debug("Data file(s)") tprint(arg="\nData file(s)", level=98, color="blue") if len(data_files) == 0: tprint(arg=" None", level=98, color="white") else: logger.debug(data_files["files"]) for data_file in data_files["files"]: tprint(arg=" " + data_file, level=98, color="white") # Need data if len(data_files) == 0 and commandline_args.test == False: if logger: logger.exception("No files input for indexing.") raise Exception, "No files input for indexing." # Too much data? if len(data_files) > 2: if logger: logger.exception("Too many files for indexing. 1 or 2 images accepted") raise Exception, "Too many files for indexing. 1 or 2 images accepted" # Get site - commandline wins over the environmental variable site = False site_module = False #detector = {} detector = False detector_module = False if commandline_args.site: site = commandline_args.site elif environmental_vars.has_key("RAPD_SITE"): site = environmental_vars["RAPD_SITE"] # Detector is defined by the user if commandline_args.detector: detector = commandline_args.detector detector_module = detector_utils.load_detector(detector) # If no site or detector, try to figure out the detector # if not (site or detector): if not detector: detector = detector_utils.get_detector_file(data_files["files"][0]) if isinstance(detector, dict): if detector.has_key("site"): site_target = detector.get("site") site_file = utils.site.determine_site(site_arg=site_target) site_module = importlib.import_module(site_file) detector_target = site_module.DETECTOR.lower() detector_module = detector_utils.load_detector(detector_target) elif detector.has_key("detector"): site_module = False detector_target = detector.get("detector") detector_module = detector_utils.load_detector(detector_target) # If someone specifies the site or found in env. if site and not site_module: site_file = utils.site.determine_site(site_arg=site) site_module = importlib.import_module(site_file) # Have a detector - read in file data if detector_module: image_headers = {} for index, data_file in enumerate(data_files["files"]): if site_module: image_headers[data_file] = detector_module.read_header(data_file, #site_module.BEAM_SETTINGS) site_module.BEAM_INFO.get(site.upper(), {})) else: image_headers[data_file] = detector_module.read_header(data_file) # If this image is derived from an hdf5 master file, tag it if "hdf5_files" in data_files: if len(data_files["hdf5_files"]) == len(data_files["files"]): image_headers[data_file]["hdf5_source"] = data_files["hdf5_files"][index] else: image_headers[data_file]["hdf5_source"] = data_files["hdf5_files"][0] logger.debug("Image headers: %s", image_headers) print_headers(tprint, image_headers) command = construct_command(image_headers=image_headers, commandline_args=commandline_args, detector_module=detector_module) else: if logger: logger.exception("No detector module found") raise Exception("No detector module found") # If no site, error # if site == False: # print text.error+"Could not determine a site. Exiting."+text.stop # sys.exit(9) # Determine the site_file # site_file = utils.site.determine_site(site_arg=site) # Error out if no site_file to import # if site_file == False: # print text.error+"Could not find a site file. Exiting."+text.stop # sys.exit(9) # Import the site settings # print "Importing %s" % site_file # site_module = importlib.import_module(site_file) # Single process lock? # utils.lock.file_lock(site_module.CONTROL_LOCK_FILE) # Instantiate the plugin # Load the plugin from directories defined in site file # for d in sys.path: # if d.endswith("src"): # toplevel_dir = d+".plugins" plugin = load_module(seek_module="plugin", directories=["plugins.index"], logger=logger) tprint(arg="\nPlugin information", level=10, color="blue") tprint(arg=" Plugin type: %s" % plugin.PLUGIN_TYPE, level=10, color="white") tprint(arg=" Plugin subtype: %s" % plugin.PLUGIN_SUBTYPE, level=10, color="white") tprint(arg=" Plugin version: %s" % plugin.VERSION, level=10, color="white") tprint(arg=" Plugin id: %s" % plugin.ID, level=10, color="white") #plugin_instance = plugin.RapdPlugin(None, command, tprint, logger) plugin_instance = plugin.RapdPlugin(site_module, command, tprint, logger) plugin_instance.start()
def run(self): """ Orchestrate the adapter's actions """ if self.message['command'] == 'ECHO': echo = load_module( seek_module='launch.launcher_adapters.echo_simple') # send message to simple_echo echo.LauncherAdapter(self.site, self.message, self.settings) else: # Load the cluster adapter for the site. cluster = load_module(seek_module=self.site.CLUSTER_ADAPTER) # Adjust the message to this site # Get the new working directory. Message only has second part of the path. #self.message = cluster.fix_command(self.message) self.message = launch_tools.fix_command(self.message) # Get the new working directory work_dir = self.message["directories"]["work"] # Get the launcher directory - Add command_files to keep files isolated qsub_dir = self.message["directories"][ "launch_dir"] + "/command_files" # Put the message into a rapd-readable file command_file = launch_tools.write_command_file( qsub_dir, self.message["command"], self.message) # Set the site tag from input site_tag = launch_tools.get_site_tag(self.message).split('_')[0] # The command to launch the job command_line = "rapd.launch -vs %s %s" % (site_tag, command_file) # Parse a label for qsub job from the command_file name qsub_label = os.path.basename(command_file).replace(".rapd", "") # Determine the number of precessors to request for job if hasattr(cluster, 'determine_nproc'): nproc = cluster.determine_nproc(self.message['command']) else: nproc = 1 # Determine which cluster queue to run if hasattr(cluster, 'check_queue'): queue = cluster.check_queue(self.message['command']) else: queue = False Thread(target=cluster.process_cluster, kwargs={ 'command': command_line, 'work_dir': work_dir, 'logfile': False, 'batch_queue': queue, 'nproc': nproc, 'logger': self.logger, 'name': qsub_label, 'mp_event': False, 'timeout': False, 'pid_queue': False, }).start() """
def main(): """ The main process Setup logging and instantiate the model""" # Get the commandline args commandline_args = get_commandline() # Output log file is always verbose log_level = 10 # Set up logging logger = utils.log.get_logger(logfile_dir="./", logfile_id="rapd_echo", level=log_level, console=commandline_args.test) # Set up terminal printer # tprint prints to the terminal, taking the arguments # arg - the string to be printed (if using level of "progress" this needs to be an int) # level - value 0 to 99 or "progress". Do NOT use a value of 100 or above # 50 - alert # 40 - error # 30 - warning # 20 - info # 10 - debug # color - color to print ("red" and so forth. See list in utils/text) # newline - put in False if you don't want a newline at the end of your print # # Verbosity if commandline_args.verbose: terminal_log_level = 10 elif commandline_args.json: terminal_log_level = 100 else: terminal_log_level = 50 tprint = utils.log.get_terminal_printer(verbosity=terminal_log_level, no_color=commandline_args.no_color, progress=commandline_args.progress) print_welcome_message(tprint) logger.debug("Commandline arguments:") tprint(arg="\nCommandline arguments:", level=10, color="blue") for pair in commandline_args._get_kwargs(): logger.debug(" arg:%s val:%s", pair[0], pair[1]) tprint(arg=" arg:%-20s val:%s" % (pair[0], pair[1]), level=10, color="white") # Get the environmental variables environmental_vars = utils.site.get_environmental_variables() logger.debug("" + text.info + "Environmental variables" + text.stop) tprint("\nEnvironmental variables", level=10, color="blue") for key, val in environmental_vars.iteritems(): logger.debug(" " + key + " : " + val) tprint(arg=" arg:%-20s val:%s" % (key, val), level=10, color="white") # Should working directory go up or down? if environmental_vars.get("RAPD_DIR_INCREMENT") in ("up", "UP"): commandline_args.dir_up = True else: commandline_args.dir_up = False # Construct the command command = construct_command(commandline_args=commandline_args, logger=logger) # Load the plugin plugin = modules.load_module(seek_module="plugin", directories=["plugins.echo"], logger=logger) # Print plugin info tprint(arg="\nPlugin information", level=10, color="blue") tprint(arg=" Plugin type: %s" % plugin.PLUGIN_TYPE, level=10, color="white") tprint(arg=" Plugin subtype: %s" % plugin.PLUGIN_SUBTYPE, level=10, color="white") tprint(arg=" Plugin version: %s" % plugin.VERSION, level=10, color="white") tprint(arg=" Plugin id: %s" % plugin.ID, level=10, color="white") # Run the plugin plugin.RapdPlugin(command, tprint, logger)
def __init__(self, inp, output, logger=None): logger.info('FindBeamCenter.__init__') self.st = time.time() self.input, self.cpu, self.verbose, self.clean = inp self.output = output self.logger = logger # Store passed-in variables #self.site = site #self.command = command self.reply_address = self.input["return_address"] # Setting up data input self.setup = self.input["directories"] self.header = self.input["header1"] self.header2 = self.input.get("header2", False) self.site_parameters = self.input.get("site_parameters") self.preferences = self.input.get("preferences", {}) self.controller_address = self.reply_address # Setup the weird named rapd_agent_index+strategy pipeline self.Labelit = load_module(seek_module="rapd_agent_index+strategy", directories=('agents'), logger=self.logger) # Setup the proper computer cluster for site. if self.site_parameters.get('cluster_site', False): self.Cluster = load_module( seek_module=self.site_parameters.get('cluster_site'), directories=('sites.cluster'), logger=self.logger) else: self.Cluster = load_module( seek_module=self.site_parameters.get('img_site'), directories=('sites.cluster'), logger=self.logger) #Number of labelit runs to fine tune beam center and spacing from original #Should be even number for symmetry around original beam center self.iterations2 = ITERATIONS self.iterations2_space = 0.3 #self.iterations2 = 15 #self.iterations2_space = 0.1 #Sets the limit of the std dev (mm) of either x or y beam position in the top 10% of the solutions. #self.std_dev = 0.1 self.std_dev = 0.25 #Sets settings to check output on my machine and does not send results to database. self.cluster_use = self.Cluster.check_cluster() #self.test = True if self.cluster_use: self.test = False else: self.test = True #this is where I have chosen to place my results self.labelit_results = {} self.labelit_summary = False #Labelit settings self.twotheta = False self.multiproc = True self.labelit_jobs = {} self.results = {} self.pool = {} self.new_input = {} #Settings for all programs self.x_beam = self.header.get('beam_center_x') self.y_beam = self.header.get('beam_center_y') self.working_dir = self.setup.get('work') self.sg = self.header.get('spacegroup', 'None') self.distance = os.path.basename(self.working_dir) self.vendortype = self.header.get('vendortype') Process.__init__(self, name='FindBeamCenter') self.start()
#logger.addHandler(handler) # Set up logging logger = utils.log.get_logger( logfile_dir="/gpfs6/users/necat/Jon/RAPD_test/Output", logfile_id="rapd_mr", level=1, #console=commandline_args.test console=False) # Setup cluster import sites.necat as site from utils.modules import load_module cluster_launcher = load_module(site.CLUSTER_ADAPTER) launcher = cluster_launcher.process_cluster # Setup local_subprocess #launcher = local_subprocess # Setup redis redis_database = importlib.import_module('database.redis_adapter') #redis_database = redis_database.Database(settings=site.CONTROL_DATABASE_SETTINGS) redis = redis_database.Database(settings=site.CONTROL_DATABASE_SETTINGS) #redis = redis_database.connect_to_redis() """ cif = '/gpfs6/users/necat/rapd2/integrate/2018-06-06/JDO_PUCK2_A14_Run4_1/rapd_pdbquery_JDO_PUCK2_A14_Run4_1_free/Phaser_1Z7E/1z7e.cif' l = ['2FGE_E', '2FGE'] for i in l: print i.split('_')[0] print len(i.split('_'))
#from multiprocessing import Process #from multiprocessing import Queue as mp_Queue #from Queue import Queue as t_Queue from threading import Thread import json # RAPD imports #import utils.launch_tools as launch_tools from utils.modules import load_module #from utils.text import json #from bson.objectid import ObjectId command_file = '/gpfs6/users/necat/rapd2/command_files/INTEGRATE__ncmuo.rapd' message = json.loads(open(command_file, 'r').read()) cluster = load_module(seek_module="sites.cluster.necat") # Get the new working directory work_dir = message["directories"]["work"] # Set the site tag from input site_tag = 'NECAT' # The command to launch the job command_line = "rapd.launch -vs %s %s" % (site_tag, command_file) # Parse a label for qsub job from the command_file name qsub_label = os.path.basename(command_file).replace(".rapd", "") nproc = 1 queue = 'phase1.q,general.q'
def __init__(self, inp, output, logger=None): logger.info('FindBeamCenter.__init__') self.st = time.time() self.input,self.cpu,self.verbose,self.clean = inp self.output = output self.logger = logger # Store passed-in variables #self.site = site #self.command = command self.reply_address = self.input["return_address"] # Setting up data input self.setup = self.input["directories"] self.header = self.input["header1"] self.header2 = self.input.get("header2", False) self.site_parameters = self.input.get("site_parameters") self.preferences = self.input.get("preferences", {}) self.controller_address = self.reply_address # Setup the weird named rapd_agent_index+strategy pipeline self.Labelit = load_module(seek_module="rapd_agent_index+strategy", directories=('agents'), logger=self.logger) # Setup the proper computer cluster for site. if self.site_parameters.get('cluster_site', False): self.Cluster = load_module(seek_module=self.site_parameters.get('cluster_site') , directories=('sites.cluster'), logger=self.logger) else: self.Cluster = load_module(seek_module=self.site_parameters.get('img_site') , directories=('sites.cluster'), logger=self.logger) #Number of labelit runs to fine tune beam center and spacing from original #Should be even number for symmetry around original beam center self.iterations2 = ITERATIONS self.iterations2_space = 0.3 #self.iterations2 = 15 #self.iterations2_space = 0.1 #Sets the limit of the std dev (mm) of either x or y beam position in the top 10% of the solutions. #self.std_dev = 0.1 self.std_dev = 0.25 #Sets settings to check output on my machine and does not send results to database. self.cluster_use = self.Cluster.check_cluster() #self.test = True if self.cluster_use: self.test = False else: self.test = True #this is where I have chosen to place my results self.labelit_results = {} self.labelit_summary = False #Labelit settings self.twotheta = False self.multiproc = True self.labelit_jobs = {} self.results = {} self.pool = {} self.new_input = {} #Settings for all programs self.x_beam = self.header.get('beam_center_x') self.y_beam = self.header.get('beam_center_y') self.working_dir = self.setup.get('work') self.sg = self.header.get('spacegroup','None') self.distance = os.path.basename(self.working_dir) self.vendortype = self.header.get('vendortype') Process.__init__(self,name='FindBeamCenter') self.start()
def main(): """ The main process Setup logging and instantiate the model""" # Get the commandline args commandline_args = get_commandline() # Output log file is always verbose log_level = 10 # Set up logging logger = utils.log.get_logger(logfile_dir="./", logfile_id="rapd_pdbquery", level=log_level, console=commandline_args.test) # Set up terminal printer # Verbosity if commandline_args.json: terminal_log_level = 100 elif commandline_args.verbose: terminal_log_level = 10 else: terminal_log_level = 50 tprint = utils.log.get_terminal_printer(verbosity=terminal_log_level, no_color=commandline_args.no_color, progress=commandline_args.progress) print_welcome_message(tprint) logger.debug("Commandline arguments:") tprint(arg="\nCommandline arguments:", level=10, color="blue") for pair in commandline_args._get_kwargs(): logger.debug(" arg:%s val:%s", pair[0], pair[1]) tprint(arg=" arg:%-20s val:%s" % (pair[0], pair[1]), level=10, color="white") # Get the environmental variables environmental_vars = utils.site.get_environmental_variables() logger.debug("" + text.info + "Environmental variables" + text.stop) tprint("\nEnvironmental variables", level=10, color="blue") for key, val in environmental_vars.iteritems(): logger.debug(" " + key + " : " + val) tprint(arg=" arg:%-20s val:%s" % (key, val), level=10, color="white") # Should working directory go up or down? if environmental_vars.get("RAPD_DIR_INCREMENT") in ("up", "UP"): commandline_args.dir_up = True else: commandline_args.dir_up = False # Construct the command command = construct_command(commandline_args=commandline_args) # Load the plugin plugin = modules.load_module(seek_module="plugin", directories=["plugins.pdbquery"], logger=logger) # Print out plugin info tprint(arg="\nPlugin information", level=10, color="blue") tprint(arg=" Plugin type: %s" % plugin.PLUGIN_TYPE, level=10, color="white") tprint(arg=" Plugin subtype: %s" % plugin.PLUGIN_SUBTYPE, level=10, color="white") tprint(arg=" Plugin version: %s" % plugin.VERSION, level=10, color="white") tprint(arg=" Plugin id: %s" % plugin.ID, level=10, color="white") # Run the plugin plugin.RapdPlugin(command, tprint, logger)
def main(): """ The main process Setup logging, gather information, and run the plugin """ # Get the commandline args commandline_args = get_commandline() # Output log file is always verbose log_level = 10 # Set up logging logger = utils.log.get_logger(logfile_dir="./", logfile_id="rapd_integrate", level=log_level, console=commandline_args.test) # Set up terminal printer # Verbosity if commandline_args.verbose: terminal_log_level = 10 elif commandline_args.json: terminal_log_level = 100 else: terminal_log_level = 30 tprint = utils.log.get_terminal_printer(verbosity=terminal_log_level, no_color=commandline_args.no_color, progress=commandline_args.progress) print_welcome_message(tprint) # Print out commandline arguments logger.debug("Commandline arguments:") tprint(arg="\nCommandline arguments:", level=10, color="blue") for pair in commandline_args._get_kwargs(): logger.debug(" arg:%s val:%s", pair[0], pair[1]) tprint(arg=" arg:%-20s val:%s" % (pair[0], pair[1]), level=10, color="white") # Get the environmental variables environmental_vars = utils.site.get_environmental_variables() logger.debug("\n" + text.info + "Environmental variables" + text.stop) tprint("\nEnvironmental variables", level=10, color="blue") for key, val in environmental_vars.iteritems(): logger.debug(" " + key + " : " + val) tprint(arg=" arg:%-20s val:%s" % (key, val), level=10, color="white") # Should working directory go up or down? if environmental_vars.get("RAPD_DIR_INCREMENT") in ("up", "UP"): commandline_args.dir_up = True else: commandline_args.dir_up = False # List sites? if commandline_args.listsites: tprint(arg="\nAvailable sites", level=99, color="blue") commandline_utils.print_sites(left_buffer=" ") if not commandline_args.listdetectors: sys.exit() # List detectors? if commandline_args.listdetectors: print "\n" + text.info + "Available detectors:" + text.stop commandline_utils.print_detectors(left_buffer=" ") sys.exit() # Look for data based on the input template data_files = commandline_utils.analyze_data_sources( commandline_args.template, mode="integrate", start_image=commandline_args.start_image, end_image=commandline_args.end_image, timeout=3) # Change hdf5 to cbf if "hdf5_files" in data_files: logger.debug("HDF5 source file(s)") tprint(arg="\nHDF5 source file(s)", level=99, color="blue") logger.debug(data_files["hdf5_files"]) for data_file in data_files["hdf5_files"]: tprint(arg=" " + data_file, level=99, color="white") logger.debug("CBF file(s) from HDF5 file(s)") tprint(arg="\nData files", level=99, color="blue") # logger.debug("Data to be integrated") # tprint(arg="\nData to be integrated", level=99, color="blue") # tprint(arg=" From %s" % data_files["data_files"][0], level=99, color="white") # tprint(arg=" To %s" % data_files["data_files"][-1], level=99, color="white") # No images match - assume that images will be arriving soon if len(data_files) == 0 and commandline_args.test == False: raise Exception("No files found for integration.") # Get site - commandline wins over the environmental variable site = False site_module = False detector = {} detector_module = False if commandline_args.site: site = commandline_args.site elif environmental_vars.has_key("RAPD_SITE"): site = environmental_vars["RAPD_SITE"] if commandline_args.detector: detector = commandline_args.detector detector_module = detector_utils.load_detector(detector) """ # If no site or detector, try to figure out the detector if not (site or detector): detector = detector_utils.get_detector_file(data_files["data_files"][0]) if isinstance(detector, dict): if detector.has_key("site"): site_target = detector.get("site") site_file = utils.site.determine_site(site_arg=site_target) # print site_file site_module = importlib.import_module(site_file) detector_target = site_module.DETECTOR.lower() detector_module = detector_utils.load_detector(detector_target) elif detector.has_key("detector"): site_module = False detector_target = detector.get("detector") detector_module = detector_utils.load_detector(detector_target) """ if not detector: detector = detector_utils.get_detector_file(data_files["data_files"][0]) if isinstance(detector, dict): if detector.has_key("site"): site_target = detector.get("site") site_file = utils.site.determine_site(site_arg=site_target) site_module = importlib.import_module(site_file) detector_target = site_module.DETECTOR.lower() detector_module = detector_utils.load_detector(detector_target) elif detector.has_key("detector"): site_module = False detector_target = detector.get("detector") detector_module = detector_utils.load_detector(detector_target) # If someone specifies the site or found in env. if site and not site_module: site_file = utils.site.determine_site(site_arg=site) site_module = importlib.import_module(site_file) # Have a detector - read in file data if not detector_module: raise Exception("No detector identified") # Get header information image_0_data = get_image_data(data_file=data_files["data_files"][0], detector_module=detector_module, site_module=site_module, site=site) image_n_data = get_image_data(data_file=data_files["data_files"][-1], detector_module=detector_module, site_module=site_module, site=site) # # Have an end image set # if commandline_args.end_image: # # End image not yet present # if image_n_data["image_number"] < commandline_args.end_image: # pass # # End image present # else: # pass # # No end image set # else: # pass logger.debug("First image header: %s", image_0_data) tprint(arg="\nFirst image header", level=10, color="blue") keys = image_0_data.keys() keys.sort() tprint(arg=" %s" % image_0_data["fullname"], level=10, color="white") for key in keys: tprint(arg=" arg:%-22s val:%s" % (key, image_0_data[key]), level=10, color="white") # Get the run data run_data = get_run_data(detector_module, image_0_data, image_n_data, commandline_args) logger.debug("Run data: %s", run_data) tprint(arg="\nRun data", level=10, color="blue") keys = run_data.keys() keys.sort() for key in keys: tprint(arg=" arg:%-22s val:%s" % (key, run_data[key]), level=10, color="white") # Construct the command for the plugin command = construct_command(image_0_data, run_data, commandline_args, detector_module) # Load the plugin plugin = load_module(seek_module="plugin", directories=["plugins.integrate"], logger=logger) tprint(arg="\nPlugin information", level=10, color="blue") tprint(arg=" Plugin type: %s" % plugin.PLUGIN_TYPE, level=10, color="white") tprint(arg=" Plugin subtype: %s" % plugin.PLUGIN_SUBTYPE, level=10, color="white") tprint(arg=" Plugin version: %s" % plugin.VERSION, level=10, color="white") tprint(arg=" Plugin id: %s" % plugin.ID, level=10, color="white") # Instantiate the plugin plugin_instance = plugin.RapdPlugin(site_module, command=command, tprint=tprint, logger=logger) plugin_instance.start()
def main(): """ The main process Setup logging and instantiate the model""" # Get the commandline args commandline_args = get_commandline() # Output log file is always verbose log_level = 10 # Set up logging logger = utils.log.get_logger(logfile_dir="./", logfile_id="rapd_pdbquery", level=log_level, console=commandline_args.test) # Set up terminal printer # Verbosity if commandline_args.json: terminal_log_level = 100 elif commandline_args.verbose: terminal_log_level = 10 else: terminal_log_level = 50 tprint = utils.log.get_terminal_printer(verbosity=terminal_log_level, no_color=commandline_args.no_color, progress=commandline_args.progress) print_welcome_message(tprint) logger.debug("Commandline arguments:") tprint(arg="\nCommandline arguments:", level=10, color="blue") for pair in commandline_args._get_kwargs(): logger.debug(" arg:%s val:%s", pair[0], pair[1]) tprint(arg=" arg:%-20s val:%s" % (pair[0], pair[1]), level=10, color="white") # Get the environmental variables environmental_vars = utils.site.get_environmental_variables() logger.debug("" + text.info + "Environmental variables" + text.stop) tprint("\nEnvironmental variables", level=10, color="blue") for key, val in environmental_vars.iteritems(): logger.debug(" " + key + " : " + val) tprint(arg=" arg:%-20s val:%s" % (key, val), level=10, color="white") # Should working directory go up or down? if environmental_vars.get("RAPD_DIR_INCREMENT") in ("up", "UP"): commandline_args.dir_up = True else: commandline_args.dir_up = False # Construct the command command = construct_command(commandline_args=commandline_args) #print command # Load the plugin plugin = modules.load_module(seek_module="plugin", directories=["plugins.pdbquery"], logger=logger) # Print out plugin info tprint(arg="\nPlugin information", level=10, color="blue") tprint(arg=" Plugin type: %s" % plugin.PLUGIN_TYPE, level=10, color="white") tprint(arg=" Plugin subtype: %s" % plugin.PLUGIN_SUBTYPE, level=10, color="white") tprint(arg=" Plugin version: %s" % plugin.VERSION, level=10, color="white") tprint(arg=" Plugin id: %s" % plugin.ID, level=10, color="white") # Run the plugin P = plugin.RapdPlugin(site=False, command=command, processed_results=False, tprint=tprint, logger=logger, verbosity=commandline_args.verbose) P.run()
def main(): """ The main process Setup logging and instantiate the model""" # Get the commandline args commandline_args = get_commandline() # Output log file is always verbose log_level = 10 # Set up logging logger = utils.log.get_logger(logfile_dir="./", logfile_id="rapd_get_cif", level=log_level, console=commandline_args.test) # Set up terminal printer # Verbosity if commandline_args.verbose: terminal_log_level = 10 elif commandline_args.json: terminal_log_level = 100 else: terminal_log_level = 50 tprint = utils.log.get_terminal_printer(verbosity=terminal_log_level, no_color=commandline_args.no_color) print_welcome_message(tprint) logger.debug("Commandline arguments:") tprint(arg="\nCommandline arguments:", level=10, color="blue") for pair in commandline_args._get_kwargs(): logger.debug(" arg:%s val:%s", pair[0], pair[1]) tprint(arg=" arg:%-20s val:%s" % (pair[0], pair[1]), level=10, color="white") # Get the environmental variables environmental_vars = utils.site.get_environmental_variables() logger.debug("" + text.info + "Environmental variables" + text.stop) tprint("\nEnvironmental variables", level=10, color="blue") for key, val in environmental_vars.iteritems(): logger.debug(" " + key + " : " + val) tprint(arg=" arg:%-20s val:%s" % (key, val), level=10, color="white") # Construct the command command = construct_command(commandline_args=commandline_args, logger=logger) # Load the plugin plugin = modules.load_module(seek_module="plugin", directories=["plugins.get_cif"], logger=logger) # Print plugin info tprint(arg="\nPlugin information", level=10, color="blue") tprint(arg=" Plugin type: %s" % plugin.PLUGIN_TYPE, level=10, color="white") tprint(arg=" Plugin subtype: %s" % plugin.PLUGIN_SUBTYPE, level=10, color="white") tprint(arg=" Plugin version: %s" % plugin.VERSION, level=10, color="white") tprint(arg=" Plugin id: %s" % plugin.ID, level=10, color="white") # Run the plugin plugin.RapdPlugin(command, tprint, logger)