def _setup_logging(self): config_ctrl = self.config["controlserver"] logfile = os.path.join( config_ctrl["log_path"], config_ctrl["log_name"].format(bl=self.beamline)) utils.check_writable(logfile) # Get queue self.log_queue = Queue(-1) handler = utils.get_log_handlers(logfile, config_ctrl["log_size"], config_ctrl["verbose"], config_ctrl["onscreen"]) # Start queue listener using the stream handler above self.log_queue_listener = utils.CustomQueueListener( self.log_queue, *handler) self.log_queue_listener.start() # Create log and set handler to queue handle self.log = utils.get_logger("ControlServer", self.log_queue) self.log.info("Init")
def setup(self, config): """Initializes parameters and creates sockets. Args: config (dict): All the configuration set either via config file or command line parameter. """ self.localhost = "127.0.0.1" self.current_pid = os.getpid() try: if config is None: self.config = get_config() else: self.config = config except Exception: self.log = logging self.ipc_dir = os.path.join(tempfile.gettempdir(), "hidra") raise config_gen = self.config["general"] config_df = self.config["datafetcher"] # change user # has to be done before logging is setup because otherwise the logfile # belongs to the wrong user user_info, user_was_changed = utils.change_user(config_gen) self.stop_request = multiprocessing.Event() # set up logging utils.check_writable(config_gen["log_file"]) self._setup_logging() utils.log_user_change(self.log, user_was_changed, user_info) # set process name # pylint: disable=no-member self.procname = config_gen["procname"] setproctitle.setproctitle(self.procname) self.log.info("Running as %s", config_gen["procname"]) self.log.info("DataManager started (PID %s).", self.current_pid) if (sys.version_info.major >= 3 and sys.version_info.minor >= 4): self.log.debug("Using multiprocessing start method '%s'", multiprocessing.get_start_method()) signal.signal(signal.SIGTERM, self.signal_term_handler) self.use_cleaner = (config_df["use_data_stream"] and config_df["remove_data"] == "with_confirmation") config_df["use_cleaner"] = self.use_cleaner self.use_statserver = config_gen["use_statserver"] self.number_of_streams = config_df["number_of_streams"] self.use_data_stream = config_df["use_data_stream"] self.log.info("Usage of data stream set to '%s'", self.use_data_stream) if "local_target" in config_df: self.log.info("Configured local_target: %s", config_df["local_target"]) else: config_df["local_target"] = None self.log.info("Version: %s", __version__) self.whitelist = config_gen["whitelist"] self.ldapuri = config_gen["ldapuri"] # IP and DNS name should be both in the whitelist self.whitelist = utils.extend_whitelist(self.whitelist, self.ldapuri, self.log) # Create zmq context # there should be only one context in one process self.context = zmq.Context() self.log.debug("Registering global ZMQ context") # set up endpoints and network config self._setup_network() self.receiver_communication = CheckReceiver(config=self.config, context=self.context, log_queue=self.log_queue) self._check_data_stream_targets()
def setup(self): """Initializes parameters, logging and transfer object. """ global _whitelist try: self.config = argument_parsing() except Exception: self.log = logging.getLogger("DataReceiver") raise config_gen = self.config["general"] config_recv = self.config["datareceiver"] # change user user_info, user_was_changed = utils.change_user(config_gen) # set up logging utils.check_writable(config_gen["log_file"]) self._setup_logging() utils.log_user_change(self.log, user_was_changed, user_info) # set process name # pylint: disable=no-member setproctitle.setproctitle(config_gen["procname"]) self.log.info("Version: %s", __version__) self.dirs_not_to_create = config_gen["dirs_not_to_create"] # for proper clean up if kill is called signal.signal(signal.SIGTERM, self.signal_term_handler) self.timeout = 2000 self.lock = threading.Lock() try: ldap_retry_time = config_gen["ldap_retry_time"] except KeyError: ldap_retry_time = 10 try: check_time = config_gen["netgroup_check_time"] except KeyError: check_time = 2 if config_gen["whitelist"] is not None: self.log.debug("config_gen['whitelist']=%s", config_gen["whitelist"]) with self.lock: _whitelist = utils.extend_whitelist(config_gen["whitelist"], config_gen["ldapuri"], self.log) self.log.info("Configured whitelist: %s", _whitelist) else: _whitelist = None # only start the thread if a netgroup was configured if (config_gen["whitelist"] is not None and isinstance(config_gen["whitelist"], str)): self.log.debug("Starting checking thread") try: self.checking_thread = CheckNetgroup(config_gen["whitelist"], self.lock, config_gen["ldapuri"], ldap_retry_time, check_time) self.checking_thread.start() except Exception: self.log.error("Could not start checking thread", exc_info=True) else: self.log.debug("Checking thread not started: %s", config_gen["whitelist"]) self.target_dir = os.path.normpath(config_recv["target_dir"]) self.data_ip = config_recv["data_stream_ip"] self.data_port = config_recv["data_stream_port"] self.log.info("Writing to directory '%s'", self.target_dir) self.transfer = Transfer(connection_type="STREAM", use_log=True, dirs_not_to_create=self.dirs_not_to_create) self._load_plugin()
def argument_parsing(): default_config = os.path.join(CONFIG_DIR, "nexusReceiver.conf") # ------------------------------------------------------------------------ # Get command line arguments # ------------------------------------------------------------------------ parser = argparse.ArgumentParser() parser.add_argument("--config_file", type=str, help="Location of the configuration file") parser.add_argument("--log_path", type=str, help="Path where logfile will be created") parser.add_argument("--log_name", type=str, help="Filename used for logging") parser.add_argument("--log_size", type=int, help="File size before rollover in B (linux only)") parser.add_argument("--verbose", help="More verbose output", action="store_true") parser.add_argument("--onscreen", type=str, help="Display logging on screen " "(options are CRITICAL, ERROR, WARNING, " "INFO, DEBUG)", default=False) parser.add_argument("--whitelist", type=str, help="List of hosts allowed to connect") parser.add_argument("--target_dir", type=str, help="Where incoming data will be stored to") parser.add_argument("--data_stream_ip", type=str, help="Ip of dataStream-socket to pull new files from") parser.add_argument("--data_stream_port", type=str, help="Port number of dataStream-socket to pull new " "files from") arguments = parser.parse_args() arguments.config_file = arguments.config_file \ or default_config # check if config_file exist utils.check_existence(arguments.config_file) # ------------------------------------------------------------------------ # Get arguments from config file # ------------------------------------------------------------------------ params = utils.set_parameters(base_config_file=arguments.config_file, config_file=None, arguments=arguments) if params["whitelist"] is not None and type(params["whitelist"]) == str: params["whitelist"] = utils.execute_ldapsearch(logging, params["whitelist"], params["ldapuri"]) # ------------------------------------------------------------------------ # Check given arguments # ------------------------------------------------------------------------ logfile = os.path.join(params["log_path"], params["log_name"]) # enable logging root = logging.getLogger() root.setLevel(logging.DEBUG) handlers = utils.get_log_handlers(logfile, params["log_size"], params["verbose"], params["onscreen"]) if type(handlers) == tuple: for h in handlers: root.addHandler(h) else: root.addHandler(handlers) # check target directory for existence utils.check_existence(params["target_dir"]) # check if logfile is writable params["log_file"] = os.path.join(params["log_path"], params["log_name"]) utils.check_writable(params["log_file"]) return params