def _setup_logging(self):
config_ctrl = self.config["controlserver"]
logfile = os.path.join(
config_ctrl["log_path"],
config_ctrl["log_name"].format(bl=self.beamline))
utils.check_writable(logfile)
# Get queue
self.log_queue = Queue(-1)
handler = utils.get_log_handlers(logfile, config_ctrl["log_size"],
config_ctrl["verbose"],
config_ctrl["onscreen"])
# Start queue listener using the stream handler above
self.log_queue_listener = utils.CustomQueueListener(
self.log_queue, *handler)
self.log_queue_listener.start()
# Create log and set handler to queue handle
self.log = utils.get_logger("ControlServer", self.log_queue)
self.log.info("Init")
def setup(self, config):
"""Initializes parameters and creates sockets.
Args:
config (dict): All the configuration set either via config file or
command line parameter.
"""
self.localhost = "127.0.0.1"
self.current_pid = os.getpid()
try:
if config is None:
self.config = get_config()
else:
self.config = config
except Exception:
self.log = logging
self.ipc_dir = os.path.join(tempfile.gettempdir(), "hidra")
raise
config_gen = self.config["general"]
config_df = self.config["datafetcher"]
# change user
# has to be done before logging is setup because otherwise the logfile
# belongs to the wrong user
user_info, user_was_changed = utils.change_user(config_gen)
self.stop_request = multiprocessing.Event()
# set up logging
utils.check_writable(config_gen["log_file"])
self._setup_logging()
utils.log_user_change(self.log, user_was_changed, user_info)
# set process name
# pylint: disable=no-member
self.procname = config_gen["procname"]
setproctitle.setproctitle(self.procname)
self.log.info("Running as %s", config_gen["procname"])
self.log.info("DataManager started (PID %s).", self.current_pid)
if (sys.version_info.major >= 3 and sys.version_info.minor >= 4):
self.log.debug("Using multiprocessing start method '%s'",
multiprocessing.get_start_method())
signal.signal(signal.SIGTERM, self.signal_term_handler)
self.use_cleaner = (config_df["use_data_stream"] and
config_df["remove_data"] == "with_confirmation")
config_df["use_cleaner"] = self.use_cleaner
self.use_statserver = config_gen["use_statserver"]
self.number_of_streams = config_df["number_of_streams"]
self.use_data_stream = config_df["use_data_stream"]
self.log.info("Usage of data stream set to '%s'", self.use_data_stream)
if "local_target" in config_df:
self.log.info("Configured local_target: %s",
config_df["local_target"])
else:
config_df["local_target"] = None
self.log.info("Version: %s", __version__)
self.whitelist = config_gen["whitelist"]
self.ldapuri = config_gen["ldapuri"]
# IP and DNS name should be both in the whitelist
self.whitelist = utils.extend_whitelist(self.whitelist, self.ldapuri,
self.log)
# Create zmq context
# there should be only one context in one process
self.context = zmq.Context()
self.log.debug("Registering global ZMQ context")
# set up endpoints and network config
self._setup_network()
self.receiver_communication = CheckReceiver(config=self.config,
context=self.context,
log_queue=self.log_queue)
self._check_data_stream_targets()
def setup(self):
"""Initializes parameters, logging and transfer object.
"""
global _whitelist
try:
self.config = argument_parsing()
except Exception:
self.log = logging.getLogger("DataReceiver")
raise
config_gen = self.config["general"]
config_recv = self.config["datareceiver"]
# change user
user_info, user_was_changed = utils.change_user(config_gen)
# set up logging
utils.check_writable(config_gen["log_file"])
self._setup_logging()
utils.log_user_change(self.log, user_was_changed, user_info)
# set process name
# pylint: disable=no-member
setproctitle.setproctitle(config_gen["procname"])
self.log.info("Version: %s", __version__)
self.dirs_not_to_create = config_gen["dirs_not_to_create"]
# for proper clean up if kill is called
signal.signal(signal.SIGTERM, self.signal_term_handler)
self.timeout = 2000
self.lock = threading.Lock()
try:
ldap_retry_time = config_gen["ldap_retry_time"]
except KeyError:
ldap_retry_time = 10
try:
check_time = config_gen["netgroup_check_time"]
except KeyError:
check_time = 2
if config_gen["whitelist"] is not None:
self.log.debug("config_gen['whitelist']=%s",
config_gen["whitelist"])
with self.lock:
_whitelist = utils.extend_whitelist(config_gen["whitelist"],
config_gen["ldapuri"],
self.log)
self.log.info("Configured whitelist: %s", _whitelist)
else:
_whitelist = None
# only start the thread if a netgroup was configured
if (config_gen["whitelist"] is not None
and isinstance(config_gen["whitelist"], str)):
self.log.debug("Starting checking thread")
try:
self.checking_thread = CheckNetgroup(config_gen["whitelist"],
self.lock,
config_gen["ldapuri"],
ldap_retry_time,
check_time)
self.checking_thread.start()
except Exception:
self.log.error("Could not start checking thread",
exc_info=True)
else:
self.log.debug("Checking thread not started: %s",
config_gen["whitelist"])
self.target_dir = os.path.normpath(config_recv["target_dir"])
self.data_ip = config_recv["data_stream_ip"]
self.data_port = config_recv["data_stream_port"]
self.log.info("Writing to directory '%s'", self.target_dir)
self.transfer = Transfer(connection_type="STREAM",
use_log=True,
dirs_not_to_create=self.dirs_not_to_create)
self._load_plugin()
def argument_parsing():
default_config = os.path.join(CONFIG_DIR, "nexusReceiver.conf")
# ------------------------------------------------------------------------
# Get command line arguments
# ------------------------------------------------------------------------
parser = argparse.ArgumentParser()
parser.add_argument("--config_file",
type=str,
help="Location of the configuration file")
parser.add_argument("--log_path",
type=str,
help="Path where logfile will be created")
parser.add_argument("--log_name",
type=str,
help="Filename used for logging")
parser.add_argument("--log_size",
type=int,
help="File size before rollover in B (linux only)")
parser.add_argument("--verbose",
help="More verbose output",
action="store_true")
parser.add_argument("--onscreen",
type=str,
help="Display logging on screen "
"(options are CRITICAL, ERROR, WARNING, "
"INFO, DEBUG)",
default=False)
parser.add_argument("--whitelist",
type=str,
help="List of hosts allowed to connect")
parser.add_argument("--target_dir",
type=str,
help="Where incoming data will be stored to")
parser.add_argument("--data_stream_ip",
type=str,
help="Ip of dataStream-socket to pull new files from")
parser.add_argument("--data_stream_port",
type=str,
help="Port number of dataStream-socket to pull new "
"files from")
arguments = parser.parse_args()
arguments.config_file = arguments.config_file \
or default_config
# check if config_file exist
utils.check_existence(arguments.config_file)
# ------------------------------------------------------------------------
# Get arguments from config file
# ------------------------------------------------------------------------
params = utils.set_parameters(base_config_file=arguments.config_file,
config_file=None,
arguments=arguments)
if params["whitelist"] is not None and type(params["whitelist"]) == str:
params["whitelist"] = utils.execute_ldapsearch(logging,
params["whitelist"],
params["ldapuri"])
# ------------------------------------------------------------------------
# Check given arguments
# ------------------------------------------------------------------------
logfile = os.path.join(params["log_path"], params["log_name"])
# enable logging
root = logging.getLogger()
root.setLevel(logging.DEBUG)
handlers = utils.get_log_handlers(logfile, params["log_size"],
params["verbose"], params["onscreen"])
if type(handlers) == tuple:
for h in handlers:
root.addHandler(h)
else:
root.addHandler(handlers)
# check target directory for existence
utils.check_existence(params["target_dir"])
# check if logfile is writable
params["log_file"] = os.path.join(params["log_path"], params["log_name"])
utils.check_writable(params["log_file"])
return params