def __init__(self, datafetcher_base_config):
DataFetcherBase.__init__(self, datafetcher_base_config, name=__name__)
# base class sets
# self.config_all - all configurations
# self.config_df - the config of the datafetcher
# self.config - the module specific config
# self.df_type - the name of the datafetcher module
# self.log_queue
# self.log
self.ipc_addresses = None
self.endpoints = None
self.source_file = None
if utils.is_windows():
self.required_params = {
"network": ["ext_ip"],
"datafetcher": {
self.df_type: ["internal_com_endpoint"]
}
}
else:
self.required_params = {
"network": ["ipc_dir"],
"datafetcher": {
self.df_type: ["internal_com_endpoint"]
}
}
self._setup()
def get_ipc_addresses(config):
"""Build the addresses used for IPC.
The addresses are only set if called on Linux. On windows they are set
to None.
Args:
config (dict): A dictionary conaining the ipc base directory and the
main PID.
Returns:
An IpcAddresses object.
"""
if utils.is_windows():
addrs = None
else:
ipc_ip = "{}/{}".format(config["network"]["ipc_dir"],
config["network"]["main_pid"])
out = "{}_{}".format(ipc_ip, "out")
mon = "{}_{}".format(ipc_ip, "mon")
addrs = IpcAddresses(out=out, mon=mon)
return addrs
def get_tcp_addresses(config):
"""Build the addresses used for TCP communication.
The addresses are only set if called on Windows. For Linux they are set
to None.
Args:
config (dict): A dictionary containing the IPs to bind and to connect
to as well as the ports. Usually con_ip is teh DNS name.
Returns:
A TcpAddresses object.
"""
if utils.is_windows():
ext_ip = config["network"]["ext_ip"]
con_ip = config["network"]["con_ip"]
df_type = config["datafetcher"]["type"]
port = config["datafetcher"][df_type]["datafetcher_port"]
datafetch_bind = "{}:{}".format(ext_ip, port)
datafetch_con = "{}:{}".format(con_ip, port)
addrs = TcpAddresses(datafetch_bind=datafetch_bind,
datafetch_con=datafetch_con)
else:
addrs = None
return addrs
def set_required_params(self):
"""
Defines the parameters to be in configuration to run this datafetcher.
Depending if on Linux or Windows other parameters are required.
"""
# self.required_params = ["context", "ext_ip",
# "con_ip", "ext_data_port"]
# if utils.is_windows():
# self.required_params += ["eventdetector_port", "datafetcher_port"]
# else:
# self.required_params += ["ipc_dir", "main_pid"]
self.required_params = {
"eventdetector": {
self.ed_type: ["ext_data_port"]
},
"network": ["ext_ip", "con_ip", "context"]
}
if utils.is_windows():
ed_params = self.required_params["eventdetector"][self.ed_type]
ed_params += ["eventdetector_port", "datafetcher_port"]
else:
self.required_params["network"] += ["ipc_dir", "main_pid"]
def __init__(self, datafetcher_base_config):
datafetcher_base_config["check_dep"] = False
DataFetcherBase.__init__(self, datafetcher_base_config, name=__name__)
# base class sets
# self.config_all - all configurations
# self.config_df - the config of the datafetcher
# self.config - the module specific config
# self.df_type - the name of the datafetcher module
# self.log_queue
# self.log
self.ipc_addresses = None
self.endpoints = None
if utils.is_windows():
self.required_params = {
"network": ["ext_ip"],
"datafetcher": {
self.df_type: ["datafetcher_port"]
}
}
else:
self.required_params = {"network": ["ipc_dir"]}
# check that the required_params are set inside of module specific
# config
self.check_config()
self._setup()
def get_tcp_addresses(config):
"""Build the addresses used for TCP communication.
The addresses are only set if called on Windows. For Linux they are set
to None.
Args:
config (dict): A dictionary containing the IPs to bind and to connect
to as well as the ports. Usually con_ip is the DNS name.
Returns:
A TcpAddresses object.
"""
if utils.is_windows():
ext_ip = config["network"]["ext_ip"]
con_ip = config["network"]["con_ip"]
port = config["eventdetector"]["zmq_events"]["eventdetector_port"]
eventdet_bind = "{}:{}".format(ext_ip, port)
eventdet_con = "{}:{}".format(con_ip, port)
addrs = TcpAddresses(eventdet_bind=eventdet_bind,
eventdet_con=eventdet_con)
else:
addrs = None
return addrs
def _setup(self):
"""
Sets static configuration parameters and which finish method to use.
"""
self.config["send_timeout"] = -1 # 10
self.config["remove_flag"] = False
try:
self.windows_handle_path = self.config["windows_handle_path"]
except KeyError:
pass
self.is_windows = utils.is_windows()
if self.config_df["use_cleaner"]:
self.log.debug("Set finish to finish_with_cleaner")
self.finish = self.finish_with_cleaner
else:
self.finish = self.finish_without_cleaner
def get_endpoints(config, ipc_addresses):
"""Configures the ZMQ endpoints depending on the protocol.
Args:
config (dict): A dictionary containing the IPs to bind and to connect
to as well as the ports. Usually con_ip is teh DNS name.
ipc_addresses: The addresses used for the inter-process communication
(ipc) protocol.
Returns:
An Endpoints object containing the bind and connection endpoints.
"""
ext_ip = config["network"]["ext_ip"]
con_ip = config["network"]["con_ip"]
port = config["eventdetector"]["hidra_events"]["ext_data_port"]
in_bind = "tcp://{}:{}".format(ext_ip, port)
in_con = "tcp://{}:{}".format(con_ip, port)
if utils.is_windows():
port = config["datafetcher_port"]
out_bind = "tcp://{}:{}".format(ext_ip, port)
out_con = "tcp://{}:{}".format(con_ip, port)
port = config["eventdetector_port"]
mon_bind = "tcp://{}:{}".format(ext_ip, port)
mon_con = "tcp://{}:{}".format(con_ip, port)
else:
out_bind = "ipc://{}".format(ipc_addresses.out)
out_con = out_bind
mon_bind = "ipc://{}".format(ipc_addresses.mon)
mon_con = mon_bind
return Endpoints(in_bind=in_bind,
in_con=in_con,
out_bind=out_bind,
out_con=out_con,
mon_bind=mon_bind,
mon_con=mon_con)
def set_required_params(self):
"""
Defines the parameters to be in configuration to run this data fetcher.
Depending if on Linux or Windows other parameters are required.
"""
self.required_params = {
"network": ["ext_ip"],
}
df_params = [
"status_check_resp_port", "confirmation_resp_port", "context"
]
if utils.is_windows():
df_params += ["datafetcher_port"]
else:
self.required_params["network"] += ["ipc_dir", "main_pid"]
self.required_params["datafetcher"] = [
"store_data", {
self.df_type: df_params
}
]
def _setup_network(self):
config_gen = self.config["general"]
config_df = self.config["datafetcher"]
self.ipc_dir = os.path.join(tempfile.gettempdir(), "hidra")
self.log.info("Configured ipc_dir: %s", self.ipc_dir)
if not os.path.exists(self.ipc_dir):
os.mkdir(self.ipc_dir)
# the permission have to be changed explicitly because
# on some platform they are ignored when called within mkdir
os.chmod(self.ipc_dir, self.ipc_dir_permissions)
self.log.info("Creating directory for IPC communication: %s",
self.ipc_dir)
# Enable specification via IP and DNS name
# TODO make this IPv6 compatible
if config_gen["ext_ip"] == "0.0.0.0":
self.ext_ip = config_gen["ext_ip"]
else:
self.ext_ip = socket.gethostbyaddr(config_gen["ext_ip"])[2][0]
self.con_ip = socket.getfqdn()
ports = {
"com": config_gen["com_port"],
"request": config_gen["request_port"],
"request_fw": config_gen["request_fw_port"],
"control_pub": config_gen["control_pub_port"],
"control_sub": config_gen["control_sub_port"],
"router": config_df["router_port"],
"cleaner": config_df["cleaner_port"],
"cleaner_trigger": config_df["cleaner_trigger_port"],
"confirmation": config_df["confirmation_port"],
}
self.ipc_addresses = utils.set_ipc_addresses(
ipc_dir=self.ipc_dir,
main_pid=self.current_pid,
use_cleaner=self.use_cleaner)
if self.use_data_stream:
data_stream_target = config_df["data_stream_targets"][0][0]
confirm_ips = [
socket.gethostbyaddr(data_stream_target)[2][0],
data_stream_target
]
else:
confirm_ips = [self.ext_ip, self.con_ip]
self.endpoints = utils.set_endpoints(ext_ip=self.ext_ip,
con_ip=self.con_ip,
ports=ports,
ipc_addresses=self.ipc_addresses,
confirm_ips=confirm_ips,
use_cleaner=self.use_cleaner)
if utils.is_windows():
self.log.info("Using tcp for internal communication.")
else:
self.log.info("Using ipc for internal communication.")
# Make ipc_dir accessible for modules
self.config["network"] = {
"ext_ip": self.ext_ip,
"con_ip": self.con_ip,
"ipc_dir": self.ipc_dir,
"main_pid": self.current_pid,
"endpoints": self.endpoints,
# TODO: this should not be set here (it belong to the modules)
"context": None,
"session": None
}