def main():
"""Get the hidra version on the sending side.
"""
# enable logging
logfile_path = os.path.join(BASE_DIR, "logs")
logfile = os.path.join(logfile_path, "example_get_remote_version.log")
utils.init_logging(logfile, True, "DEBUG")
parser = argparse.ArgumentParser()
parser.add_argument("--signal_host",
type=str,
help="Host where HiDRA is running",
default=socket.getfqdn())
parser.add_argument("--target_host",
type=str,
help="Host where the data should be send to",
default=socket.getfqdn())
arguments = parser.parse_args()
transfer_type = "QUERY_NEXT"
query = Transfer(transfer_type, arguments.signal_host, use_log=None)
version = query.get_remote_version()
print("Remote version: {0}".format(version))
class Worker(multiprocessing.Process):
def __init__(self, worker_id, transfer_type, signal_host, target_host,
port, number_of_files):
super().__init__()
self.id = worker_id
self.basepath = os.path.join(BASE_DIR, "data", "target")
self.number_of_files = number_of_files
self.port = port
print("start Transfer on port", port)
self.query = Transfer(transfer_type, signal_host, use_log=None)
self.query.start([target_host, port])
self.run()
def run(self):
try:
while True:
# print("Worker-{0}: waiting".format(self.id))
[metadata, data] = self.query.get()
if metadata and data:
self.number_of_files.value += 1
finally:
self.stop()
def stop(self):
self.query.stop()
def __exit__(self):
self.stop()
def __del__(self):
self.stop()
def __init__(self):
self.transfer = None
try:
params = argument_parsing()
except Exception:
self.log = self.get_logger()
raise
self.log = self.get_logger()
self.whitelist = params["whitelist"]
self.log.info("Configured whitelist: {0}".format(self.whitelist))
self.target_dir = os.path.normpath(params["target_dir"])
self.data_ip = params["data_stream_ip"]
self.data_port = params["data_stream_port"]
self.log.info("Writing to directory '{0}'".format(self.target_dir))
self.transfer = Transfer("NEXUS", use_log=True)
try:
self.run()
except KeyboardInterrupt:
pass
except Exception:
self.log.error("Stopping due to unknown error condition",
exc_info=True)
finally:
self.stop()
def _setup(self):
"""Sets up and configures the transfer.
"""
self.transfer = Transfer("STREAM", use_log=self.log_queue)
config_net = self.config_all["network"]
endpoint = "{}_{}".format(config_net["main_pid"], "out")
self.transfer.start([config_net["ipc_dir"], endpoint],
protocol="ipc",
data_con_style="connect")
# enable status check requests from any sender
self.transfer.setopt(option="status_check",
value=[
config_net["ext_ip"],
self.config["status_check_resp_port"]
])
# enable confirmation reply if this is requested in a received data
# packet
self.transfer.setopt(option="confirmation",
value=[
config_net["ext_ip"],
self.config["confirmation_resp_port"]
])
def __init__(self, identifier, transfer_type, signal_host, target_host,
port): # noqa F811
super().__init__()
self.identifier = identifier
self.port = port
self.query = Transfer(transfer_type, signal_host, use_log=False)
# Set up ZeroMQ for this worker
print("start Transfer on port {0}".format(port))
self.query.start([target_host, port])
self.run()
def __init__(self, worker_id, transfer_type, signal_host, target_host,
port, number_of_files):
super().__init__()
self.id = worker_id
self.basepath = os.path.join(BASE_DIR, "data", "target")
self.number_of_files = number_of_files
self.port = port
print("start Transfer on port", port)
self.query = Transfer(transfer_type, signal_host, use_log=None)
self.query.start([target_host, port])
self.run()
def __init__(self, signal_host):
self.log = self.__get_logger()
self.timeout = 2000
self.read_pointer = 0
targets = [socket.getfqdn(), "50101", 1]
# create HiDRA Transfer instance which wants data by request only
self.query = Transfer("QUERY_NEXT", signal_host)
self.query.initiate(targets)
self.query.start()
self.metadata = None
self.data = None
def main():
"""Connect to hidra and get the streamed metadata.
"""
parser = argparse.ArgumentParser()
parser.add_argument("--signal_host",
type=str,
help="Host where HiDRA is running",
default=socket.getfqdn())
parser.add_argument("--target_host",
type=str,
help="Host where the data should be send to",
default=socket.getfqdn())
parser.add_argument("--detector_id",
type=str,
help="Which detector to get data from",
default=socket.getfqdn())
arguments = parser.parse_args()
# targets = [arguments.target_host, "50101", 0]
targets = [[arguments.target_host, "50101", 0, ".*(tif|cbf)$"]]
# targets = [[arguments.target_host, "50101", 0, [".tif", ".cbf"]]]
print("\n==== TEST: Stream all files ====\n")
query = Transfer("STREAM_METADATA",
signal_host=arguments.signal_host,
detector_id=arguments.detector_id)
query.initiate(targets)
query.start()
while True:
try:
print("waiting to get metadata")
[metadata, _] = query.get()
except Exception:
break
print()
print("metadata", metadata["filename"])
# print ("data", str(data)[:10])
print()
query.stop()
print("\n==== TEST END: Stream all files ====\n")
def __init__(self, path=None, filetype=None, interval=None, parent=None):
QThread.__init__(self, parent)
if path is not None:
self.path = path
if filetype is not None:
self.filetype = filetype
if interval is not None:
self.interval = interval
# suffix = [".cbf", ".tif", ".hdf5"]
self.query = Transfer("QUERY_NEXT_METADATA", self.transfer_type)
self.query.initiate([socket.getfqdn(), self.data_port, "1"])
self.query.start(self.data_port)
# self.query.initiate(["zitpcx22614w", self.data_port, "1"])
# self.query.start(["zitpcx22614w", self.data_port])
self.mutex = QMutex()
def main():
"""Connects to hidra and stores the streamed data to disk.
"""
parser = argparse.ArgumentParser()
parser.add_argument("--signal_host",
type=str,
help="Host where HiDRA is running",
default=socket.getfqdn())
parser.add_argument("--target_host",
type=str,
help="Host where the data should be send to",
default=socket.getfqdn())
arguments = parser.parse_args()
# enable logging
logfile_path = os.path.join(BASE_DIR, "logs")
logfile = os.path.join(logfile_path, "testAPI.log")
utils.init_logging(logfile, True, "DEBUG")
targets = [arguments.target_host, "50100", 0]
print("\n==== TEST: Stream all files and store them ====\n")
query = Transfer("STREAM", arguments.signal_host, use_log=True)
query.initiate(targets)
query.start()
target_dir = os.path.join(BASE_DIR, "data", "zmq_target")
target_file = os.path.join(target_dir, "test_store")
try:
query.store(target_file)
except Exception as excp:
print("Storing data failed.")
print("Error was:", excp)
query.stop()
print("\n==== TEST END: Stream all files and store them ====\n")
class Worker(multiprocessing.Process):
"""Start a connection to hidra an get a data block.
"""
def __init__(self, identifier, transfer_type, signal_host, target_host,
port): # noqa F811
super().__init__()
self.identifier = identifier
self.port = port
self.query = Transfer(transfer_type, signal_host, use_log=False)
# Set up ZeroMQ for this worker
print("start Transfer on port {0}".format(port))
self.query.start([target_host, port])
self.run()
def run(self):
while True:
try:
# Get new data
print("Worker-{0}: waiting".format(self.identifier))
[metadata, data] = self.query.get()
except Exception:
break
print("metadata", metadata)
print("data", str(data)[:100])
def stop(self):
"""Clean up:
"""
self.query.stop()
def __exit__(self, exception_type, exception_value, traceback):
self.stop()
def __del__(self):
self.stop()
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--signal_host",
type=str,
help="Host where HiDRA is running",
default=socket.getfqdn())
parser.add_argument("--target_host",
type=str,
help="Host where the data should be send to",
default=socket.getfqdn())
parser.add_argument("--procname",
type=str,
help="Name with which the service should be running",
default="test_query_speed")
parser.add_argument("--workers",
type=int,
help="How many worker processes should be launched",
default=1)
arguments = parser.parse_args()
setproctitle.setproctitle(arguments.procname)
workers = []
number_of_files = multiprocessing.Value('i', 0)
targets = []
transfer_type = "QUERY_NEXT"
for n in range(arguments.workers):
p = str(50100 + n)
w = multiprocessing.Process(
target=Worker,
args=(n, transfer_type, arguments.signal_host,
arguments.target_host, p, number_of_files))
workers.append(w)
targets.append([arguments.target_host, p, 1, [".cbf"]])
query = Transfer(transfer_type, arguments.signal_host, use_log=None)
query.initiate(targets)
for w in workers:
w.start()
try:
while all(w.is_alive() for w in workers):
time.sleep(0.5)
print("number_of_files={0}".format(number_of_files.value))
except KeyboardInterrupt:
pass
finally:
print("number_of_files={0}".format(number_of_files.value))
for w in workers:
w.terminate()
query.stop()
def main():
"""Register and start workers.
"""
parser = argparse.ArgumentParser()
parser.add_argument("--signal_host",
type=str,
help="Host where HiDRA is running",
default=socket.getfqdn())
parser.add_argument("--target_host",
type=str,
help="Host where the data should be send to",
default=socket.getfqdn())
parser.add_argument("--procname",
type=str,
help="Name with which the service should be running",
default="example_onda")
arguments = parser.parse_args()
transfer_type = "QUERY_NEXT"
number_of_worker = 3
workers = []
targets = []
# Create <number_of_worker> workers to receive and process data
for i in range(number_of_worker):
port = str(50100 + i)
targets.append([arguments.target_host, port, 1, [".cbf"]])
proc = multiprocessing.Process(target=Worker,
args=(i, transfer_type,
arguments.signal_host,
arguments.target_host, port))
workers.append(proc)
# register these workers on the sending side
# this is done from the master to enforce that the data received from the
# workers is disjoint
query = Transfer(transfer_type, arguments.signal_host, use_log=False)
query.initiate(targets)
for i in workers:
i.start()
try:
while True:
pass
except Exception:
pass
finally:
for i in workers:
i.terminate()
query.stop()
def __init__(self, identifier, transfer_type, basepath, signal_host,
target_host, port):
super().__init__()
self.identifier = identifier
self.port = port
self.transfer_type = transfer_type
self.log = logging.getLogger("Worker-{}".format(self.identifier))
self.query = Transfer(self.transfer_type, signal_host, use_log=True)
self.basepath = basepath
self.log.debug("start Transfer on port %s", port)
# targets are locally
self.query.start([target_host, port])
# self.query.start(port)
self.run()
def main():
"""Connect to hidra and get streamed data.
"""
parser = argparse.ArgumentParser()
parser.add_argument("--signal_host",
type=str,
help="Host where HiDRA is running",
default=socket.getfqdn())
parser.add_argument("--target_host",
type=str,
help="Host where the data should be send to",
default=socket.getfqdn())
arguments = parser.parse_args()
# targets = [arguments.target_host, "50101", 0]
targets = [[arguments.target_host, "50101", 0, ".*(tif|cbf)$"]]
# targets = [[arguments.target_host, "50101", 0, [".tif", ".cbf"]]]
print("\n==== TEST: Stream all files ====\n")
query = Transfer("STREAM", arguments.signal_host)
query.initiate(targets)
query.start()
while True:
try:
[metadata, data] = query.get()
except Exception:
break
print()
print("metadata", metadata["filename"])
print("data", len(data))
print()
query.stop()
print("\n==== TEST END: Stream all files ====\n")
def main():
"""De-registers a connection from a broken down program.
"""
# enable logging
logfile_path = os.path.join(BASE_DIR, "logs")
logfile = os.path.join(logfile_path, "example_force_stop.log")
utils.init_logging(logfile, True, "DEBUG")
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("--signal_host",
type=str,
help="Host where HiDRA is running",
default=socket.getfqdn())
parser.add_argument("--target_host",
type=str,
help="Host where the data should be send to",
default=socket.getfqdn())
arguments = parser.parse_args()
# targets = [[arguments.target_host, "50101", 1, ".*(tif|cbf)$"]]
targets = [[arguments.target_host, "50100", 1, [".cbf"]],
[arguments.target_host, "50101", 1, [".cbf"]],
[arguments.target_host, "50102", 1, [".cbf"]]]
transfer_type = "QUERY_NEXT"
# transfer_type = "STREAM"
# transfer_type = "STREAM_METADATA"
# transfer_type = "QUERY_NEXT_METADATA"
query = Transfer(transfer_type, arguments.signal_host, use_log=True)
query.force_stop(targets)
def nexus_transfer():
obj = Transfer("NEXUS", use_log=True)
obj.start([socket_m.getfqdn(), "50100"])
callback_params = {"run_loop": True}
# number to receive + open signal + close signal
try:
while callback_params["run_loop"]:
try:
obj.read(callback_params, open_callback, read_callback,
close_callback)
except KeyboardInterrupt:
break
except Exception:
logging.error("nexus_transfer break", exc_info=True)
break
finally:
obj.stop()
def main():
sender_thread = SenderAsThread()
sender_thread.start()
obj = Transfer("NEXUS", use_log=True)
obj.start([socket_m.get_fqdn(), "50100"])
callback_params = {"run_loop": True}
try:
while callback_params["run_loop"]:
try:
obj.read(callback_params, open_callback, read_callback,
close_callback)
except KeyboardInterrupt:
break
except Exception:
logging.error("break", exc_info=True)
break
finally:
sender_thread.stop()
obj.stop()
print("\n==== TEST END: nexus transfer ====\n")
def main():
"""
The fixes target is configured on the sender side. Start up receiving side.
"""
# enable logging
logfile_path = os.path.join(BASE_DIR, "logs")
logfile = os.path.join(logfile_path, "test_fixedStream.log")
utils.init_logging(logfile, True, "DEBUG")
data_port = "50100"
print("\n==== TEST: Fixed stream ====\n")
query = Transfer("STREAM", use_log=True)
query.start(data_port)
while True:
try:
[metadata, data] = query.get()
except KeyboardInterrupt:
break
except Exception as excp:
print("Getting data failed.")
print("Error was: {0}".format(excp))
break
print()
try:
print("metadata of file", metadata["filename"])
print("data", str(data)[:10])
except TypeError:
print("metadata", metadata)
print()
query.stop()
print("\n==== TEST END: Fixed Stream ====\n")
def main():
"""Requests data from hidra on a query basis.
"""
arguments = get_arguments()
if arguments.debug:
use_log = "debug"
print("Using debug mode")
else:
use_log = False # pylint: disable=redefined-variable-type
targets = [[arguments.target_host, "50101", 1, ".*(tif|cbf)$"]]
detector_id = socket.getfqdn() # the hidra instance to connect to
print("\n==== TEST: Query for the newest filename ====\n")
query = Transfer("QUERY_NEXT",
arguments.signal_host,
use_log=use_log,
detector_id=detector_id)
query.initiate(targets)
try:
query.start()
except Exception:
query.stop()
return
timeout = None
# timeout = 2000 # in ms
while True:
try:
[metadata, data] = query.get(timeout)
except Exception:
print(sys.exc_info())
break
print()
if metadata and data:
print("metadata", metadata["filename"])
print("data", str(data)[:10])
else:
print("metadata", metadata)
print("data", data)
print()
query.stop()
print("\n==== TEST END: Query for the newest filename ====\n")
def main():
"""Requests data from hidra on a query basis.
"""
parser = argparse.ArgumentParser()
parser.add_argument("--signal_host",
type=str,
help="Host where HiDRA is running",
default=socket.getfqdn())
parser.add_argument("--target_host",
type=str,
help="Host where the data should be send to",
default=socket.getfqdn())
arguments = parser.parse_args()
targets = [[arguments.target_host, "50101", 1]]
print("\n==== TEST: Query for the newest filename ====\n")
query = Transfer("QUERY_NEXT", arguments.signal_host, use_log="debug")
query.initiate(targets)
try:
query.start()
except Exception:
query.stop()
return
timeout = None
# timeout = 2000 # in ms
try:
while True:
[metadata, data] = query.get(timeout)
print()
if metadata and data:
print("metadata", metadata["filename"])
print("data", data)
else:
print("metadata", metadata)
print("data", data)
print()
finally:
query.stop()
print("\n==== TEST END: Query for the newest filename ====\n")
class DataReceiver(object):
"""Receives data and stores it to disc usign the hidra API.
"""
def __init__(self):
self.transfer = None
self.checking_thread = None
self.timeout = None
self.config = None
self.log = None
self.dirs_not_to_create = None
self.lock = None
self.target_dir = None
self.data_ip = None
self.data_port = None
self.transfer = None
self.checking_thread = None
self.plugin_handler = None
self.run_loop = True
self.setup()
self.exec_run()
def setup(self):
"""Initializes parameters, logging and transfer object.
"""
global _whitelist
try:
self.config = argument_parsing()
except Exception:
self.log = logging.getLogger("DataReceiver")
raise
config_gen = self.config["general"]
config_recv = self.config["datareceiver"]
# change user
user_info, user_was_changed = utils.change_user(config_gen)
# set up logging
utils.check_writable(config_gen["log_file"])
self._setup_logging()
utils.log_user_change(self.log, user_was_changed, user_info)
# set process name
# pylint: disable=no-member
setproctitle.setproctitle(config_gen["procname"])
self.log.info("Version: %s", __version__)
self.dirs_not_to_create = config_gen["dirs_not_to_create"]
# for proper clean up if kill is called
signal.signal(signal.SIGTERM, self.signal_term_handler)
self.timeout = 2000
self.lock = threading.Lock()
try:
ldap_retry_time = config_gen["ldap_retry_time"]
except KeyError:
ldap_retry_time = 10
try:
check_time = config_gen["netgroup_check_time"]
except KeyError:
check_time = 2
if config_gen["whitelist"] is not None:
self.log.debug("config_gen['whitelist']=%s",
config_gen["whitelist"])
with self.lock:
_whitelist = utils.extend_whitelist(config_gen["whitelist"],
config_gen["ldapuri"],
self.log)
self.log.info("Configured whitelist: %s", _whitelist)
else:
_whitelist = None
# only start the thread if a netgroup was configured
if (config_gen["whitelist"] is not None
and isinstance(config_gen["whitelist"], str)):
self.log.debug("Starting checking thread")
try:
self.checking_thread = CheckNetgroup(config_gen["whitelist"],
self.lock,
config_gen["ldapuri"],
ldap_retry_time,
check_time)
self.checking_thread.start()
except Exception:
self.log.error("Could not start checking thread",
exc_info=True)
else:
self.log.debug("Checking thread not started: %s",
config_gen["whitelist"])
self.target_dir = os.path.normpath(config_recv["target_dir"])
self.data_ip = config_recv["data_stream_ip"]
self.data_port = config_recv["data_stream_port"]
self.log.info("Writing to directory '%s'", self.target_dir)
self.transfer = Transfer(connection_type="STREAM",
use_log=True,
dirs_not_to_create=self.dirs_not_to_create)
self._load_plugin()
def _setup_logging(self):
config_gen = self.config["general"]
# enable logging
root = logging.getLogger()
root.setLevel(logging.DEBUG)
handlers = utils.get_log_handlers(config_gen["log_file"],
config_gen["log_size"],
config_gen["verbose"],
config_gen["onscreen"])
if isinstance(handlers, tuple):
for hdl in handlers:
root.addHandler(hdl)
else:
root.addHandler(handlers)
self.log = logging.getLogger("DataReceiver")
def _load_plugin(self):
try:
plugin_name = self.config["datareceiver"]["plugin"]
plugin_config = self.config[plugin_name]
except KeyError:
self.log.debug("No plugin specified")
return
self.plugin_handler = PluginHandler(plugin_name, plugin_config,
self.target_dir, self.log)
def exec_run(self):
"""Wrapper around run to react to exceptions.
"""
try:
self.run()
except KeyboardInterrupt:
pass
except Exception:
self.log.error("Stopping due to unknown error condition",
exc_info=True)
raise
finally:
self.stop()
def run(self):
"""Start the transfer and store the data.
"""
global _whitelist # pylint: disable=global-variable-not-assigned
global _changed_netgroup
if self.plugin_handler is not None:
plugin_type = self.plugin_handler.get_data_type()
self.plugin_handler.start()
else:
plugin_type = None
try:
self.transfer.start([self.data_ip, self.data_port], _whitelist)
except Exception:
self.log.error("Could not initiate stream", exc_info=True)
self.stop(store=False)
raise
# enable status check requests from any sender
self.transfer.setopt("status_check")
# enable confirmation reply if this is requested in a received data
# packet
self.transfer.setopt("confirmation")
self.log.debug("Waiting for new messages...")
self.run_loop = True
# run loop, and wait for incoming messages
while self.run_loop:
if _changed_netgroup:
self.log.debug("Re-registering whitelist")
self.transfer.register(_whitelist)
# reset flag
with self.lock:
_changed_netgroup = False
try:
ret_val = self.transfer.store(target_base_path=self.target_dir,
timeout=self.timeout,
return_type=plugin_type)
except KeyboardInterrupt:
break
except Exception:
self.log.error("Storing data...failed.", exc_info=True)
raise
if self.plugin_handler is None or ret_val is None:
continue
try:
self.plugin_handler.put(ret_val)
# ret_val might have been mutated by the plugin and therefore
# should only be reused if this is acceptable
except Exception:
self.log.error("Cannot submit message to plugin")
def stop(self, store=True):
"""Stop threads, close sockets and cleans up.
Args:
store (optional, bool): Run a little longer to store remaining
data.
"""
self.run_loop = False
if self.transfer is not None:
self.transfer.status = [b"ERROR", "receiver is shutting down"]
if store:
stop_timeout = 0.5
start_time = time.time()
diff_time = (time.time() - start_time) * 1000
self.log.debug("Storing remaining data.")
while diff_time < stop_timeout:
try:
self.log.debug("Storing remaining data...")
self.transfer.store(self.target_dir, self.timeout)
except Exception:
self.log.error("Storing data...failed.", exc_info=True)
diff_time = (time.time() - start_time) * 1000
self.log.info("Shutting down receiver...")
self.transfer.stop()
self.transfer = None
if self.plugin_handler is not None:
self.plugin_handler.stop()
if self.checking_thread is not None:
self.checking_thread.stop()
self.checking_thread.join()
self.log.debug("checking_thread stopped")
self.checking_thread = None
# pylint: disable=unused-argument
def signal_term_handler(self, signal_to_react, frame):
"""React on external SIGTERM signal.
"""
self.log.debug('got SIGTERM')
self.stop()
def __exit__(self, exception_type, exception_value, traceback):
self.stop()
def __del__(self):
self.stop()
class DataFetcher(DataFetcherBase):
"""
Implementation of the data fetcher reacting on data sent by another
hidra instance.
"""
def __init__(self, datafetcher_base_config):
"""Initial setup
Checks if all required parameters are set in the configuration
"""
self.f_descriptors = dict()
self.transfer = None
DataFetcherBase.__init__(self, datafetcher_base_config, name=__name__)
# base class sets
# self.config_all - all configurations
# self.config_df - the config of the datafetcher
# self.config - the module specific config
# self.df_type - the name of the datafetcher module
# self.log_queue
# self.log
self.metadata_r = None
self.data_r = None
self.set_required_params()
# check that the required_params are set inside of module specific
# config
self.check_config()
self._setup()
def set_required_params(self):
"""
Defines the parameters to be in configuration to run this data fetcher.
Depending if on Linux or Windows other parameters are required.
"""
self.required_params = {
"network": ["ext_ip"],
}
df_params = [
"status_check_resp_port", "confirmation_resp_port", "context"
]
if utils.is_windows():
df_params += ["datafetcher_port"]
else:
self.required_params["network"] += ["ipc_dir", "main_pid"]
self.required_params["datafetcher"] = [
"store_data", {
self.df_type: df_params
}
]
def _setup(self):
"""Sets up and configures the transfer.
"""
self.transfer = Transfer("STREAM", use_log=self.log_queue)
config_net = self.config_all["network"]
endpoint = "{}_{}".format(config_net["main_pid"], "out")
self.transfer.start([config_net["ipc_dir"], endpoint],
protocol="ipc",
data_con_style="connect")
# enable status check requests from any sender
self.transfer.setopt(option="status_check",
value=[
config_net["ext_ip"],
self.config["status_check_resp_port"]
])
# enable confirmation reply if this is requested in a received data
# packet
self.transfer.setopt(option="confirmation",
value=[
config_net["ext_ip"],
self.config["confirmation_resp_port"]
])
def get_metadata(self, targets, metadata):
"""Implementation of the abstract method get_metadata.
Args:
targets (list): The target list this file is supposed to go.
metadata (dict): The dictionary with the metadata to extend.
"""
timeout = 10000
# Get new data
self.metadata_r, self.data_r = self.transfer.get(timeout)
if (metadata["relative_path"] != self.metadata_r["relative_path"]
or metadata["source_path"] != self.metadata_r["source_path"]
or metadata["filename"] != self.metadata_r["filename"]):
self.log.error("Received metadata do not match data")
# Use received data to prevent mismatch of metadata and data
# TODO handle case if file type requested by target does not match
# pylint: disable=attribute-defined-outside-init
# Build source file
self.source_file = generate_filepath(self.metadata_r["source_path"],
self.metadata_r)
# Build target file
# if local_target is not set (== None) generate_filepath returns None
self.target_file = generate_filepath(self.config_df["local_target"],
self.metadata_r)
# Extends metadata
if targets:
if "filesize" not in self.metadata_r:
self.log.error("Received metadata do not contain 'filesize'")
if "file_mod_time" not in self.metadata_r:
self.log.error("Received metadata do not contain "
"'file_mod_time'. Setting it to current time")
self.metadata_r["file_mod_time"] = time.time()
if "file_create_time" not in self.metadata_r:
self.log.error("Received metadata do not contain "
"'file_create_time'. Setting it to current "
"time")
self.metadata_r["file_create_time"] = time.time()
if "chunksize" not in self.metadata_r:
self.log.error("Received metadata do not contain 'chunksize'. "
"Setting it to locally configured one")
self.metadata_r["chunksize"] = self.config_df["chunksize"]
def send_data(self, targets, metadata, open_connections):
"""Implementation of the abstract method send_data.
Args:
targets (list): The target list this file is supposed to go.
metadata (dict): The dictionary with the metadata of the file
open_connections (dict): The dictionary containing all open zmq
connections.
"""
# pylint: disable=unused-argument
if not targets:
return
# targets are of the form [[<host:port>, <prio>, <metadata|data>], ...]
targets_data = [i for i in targets if i[2] == "data"]
if not targets_data:
return
self.log.debug("Received data for file %s (chunknumber %s)",
self.source_file, self.metadata_r["chunk_number"])
self.log.debug("Passing multipart-message for file '%s'...",
self.source_file)
try:
chunk_payload = [
json.dumps(self.metadata_r).encode("utf-8"), self.data_r
]
except Exception:
self.log.error("Unable to pack multipart-message for file "
"'%s'",
self.source_file,
exc_info=True)
return
# send message to data targets
try:
self.send_to_targets(targets=targets_data,
open_connections=open_connections,
metadata=None,
payload=chunk_payload,
chunk_number=self.metadata_r["chunk_number"])
except DataError:
self.log.error(
"Unable to send multipart-message for file '%s' (chunk %s)",
self.source_file,
self.metadata_r["chunk_number"],
exc_info=True)
except Exception:
self.log.error(
"Unable to send multipart-message for file '%s' (chunk %s)",
self.source_file,
self.metadata_r["chunk_number"],
exc_info=True)
def finish(self, targets, metadata, open_connections):
"""Implementation of the abstract method finish.
Args:
targets (list): The target list this file is supposed to go.
metadata (dict): The dictionary with the metadata of the file
open_connections (dict): The dictionary containing all open zmq
connections.
"""
# targets are of the form [[<host:port>, <prio>, <metadata|data>], ...]
targets_metadata = [i for i in targets if i[2] == "metadata"]
# send message to metadata targets
if targets_metadata:
try:
self.send_to_targets(targets=targets_metadata,
open_connections=open_connections,
metadata=metadata,
payload=None,
chunk_number=None,
timeout=self.config["send_timeout"])
self.log.debug(
"Passing metadata multipart-message for file "
"%s...done.", self.source_file)
except Exception:
self.log.error(
"Unable to send metadata multipart-message for file"
"'%s' to '%s'",
self.source_file,
targets_metadata,
exc_info=True)
# store data
if self.config_df["store_data"]:
try:
# TODO: save message to file using a thread (avoids blocking)
self.transfer.store_chunk(
descriptors=self.f_descriptors,
filepath=self.target_file,
payload=self.data_r,
base_path=self.config_df["local_target"],
metadata=self.metadata_r)
except Exception:
self.log.error(
"Storing multipart message for file '%s' failed",
self.source_file,
exc_info=True)
def stop(self):
"""Implementation of the abstract method stop.
"""
# Close base class zmq sockets
self.close_socket()
# Close open file handler to prevent file corruption
for target_file in list(self.f_descriptors):
self.f_descriptors[target_file].close()
del self.f_descriptors[target_file]
# Close zmq sockets
if self.transfer is not None:
self.transfer.stop()
def setup(self):
"""Initializes parameters, logging and transfer object.
"""
global _whitelist
try:
self.config = argument_parsing()
except Exception:
self.log = logging.getLogger("DataReceiver")
raise
config_gen = self.config["general"]
config_recv = self.config["datareceiver"]
# change user
user_info, user_was_changed = utils.change_user(config_gen)
# set up logging
utils.check_writable(config_gen["log_file"])
self._setup_logging()
utils.log_user_change(self.log, user_was_changed, user_info)
# set process name
# pylint: disable=no-member
setproctitle.setproctitle(config_gen["procname"])
self.log.info("Version: %s", __version__)
self.dirs_not_to_create = config_gen["dirs_not_to_create"]
# for proper clean up if kill is called
signal.signal(signal.SIGTERM, self.signal_term_handler)
self.timeout = 2000
self.lock = threading.Lock()
try:
ldap_retry_time = config_gen["ldap_retry_time"]
except KeyError:
ldap_retry_time = 10
try:
check_time = config_gen["netgroup_check_time"]
except KeyError:
check_time = 2
if config_gen["whitelist"] is not None:
self.log.debug("config_gen['whitelist']=%s",
config_gen["whitelist"])
with self.lock:
_whitelist = utils.extend_whitelist(config_gen["whitelist"],
config_gen["ldapuri"],
self.log)
self.log.info("Configured whitelist: %s", _whitelist)
else:
_whitelist = None
# only start the thread if a netgroup was configured
if (config_gen["whitelist"] is not None
and isinstance(config_gen["whitelist"], str)):
self.log.debug("Starting checking thread")
try:
self.checking_thread = CheckNetgroup(config_gen["whitelist"],
self.lock,
config_gen["ldapuri"],
ldap_retry_time,
check_time)
self.checking_thread.start()
except Exception:
self.log.error("Could not start checking thread",
exc_info=True)
else:
self.log.debug("Checking thread not started: %s",
config_gen["whitelist"])
self.target_dir = os.path.normpath(config_recv["target_dir"])
self.data_ip = config_recv["data_stream_ip"]
self.data_port = config_recv["data_stream_port"]
self.log.info("Writing to directory '%s'", self.target_dir)
self.transfer = Transfer(connection_type="STREAM",
use_log=True,
dirs_not_to_create=self.dirs_not_to_create)
self._load_plugin()
def main():
"""Connects to hidra and request metadata.
"""
parser = argparse.ArgumentParser()
parser.add_argument("--signal_host",
type=str,
help="Host where HiDRA is running",
default=socket.getfqdn())
parser.add_argument("--target_host",
type=str,
help="Host where the data should be send to",
default=socket.getfqdn())
arguments = parser.parse_args()
targets = [[arguments.target_host, "50101", 0]]
base_target_path = os.path.join(BASE_DIR, "data", "target")
print("\n==== TEST: Query for the newest filename ====\n")
query = Transfer("QUERY_NEXT_METADATA", arguments.signal_host)
query.initiate(targets)
query.start()
try:
while True:
try:
[metadata, _] = query.get()
except Exception:
query.stop()
raise
print()
print(generate_filepath(base_target_path, metadata))
print()
finally:
query.stop()
print("\n==== TEST END: Query for the newest filename ====\n")
class LiveView(QThread):
FILETYPE_CBF = 0
FILETYPE_TIF = 1
FILETYPE_HDF5 = 2
alive = False
path = ""
filetype = 0
interval = 0.5 # s
stoptimer = -1.0
viewer = None
subframe = None
mutex = None
query = None
transfer_type = socket.getfqdn()
data_port = "50122"
basepath = os.path.join(BASE_DIR, "data", "target")
# basepath = "/gpfs"
def __init__(self, path=None, filetype=None, interval=None, parent=None):
QThread.__init__(self, parent)
if path is not None:
self.path = path
if filetype is not None:
self.filetype = filetype
if interval is not None:
self.interval = interval
# suffix = [".cbf", ".tif", ".hdf5"]
self.query = Transfer("QUERY_NEXT_METADATA", self.transfer_type)
self.query.initiate([socket.getfqdn(), self.data_port, "1"])
self.query.start(self.data_port)
# self.query.initiate(["zitpcx22614w", self.data_port, "1"])
# self.query.start(["zitpcx22614w", self.data_port])
self.mutex = QMutex()
def start(self, path=None, filetype=None, interval=None):
if path is not None:
self.path = path
if filetype is not None:
self.filetype = filetype
if interval is not None:
self.interval = interval
QThread.start(self)
def stop(self, interval=0.0):
if self.stoptimer < 0.0 and interval > 0.0:
print("Live view thread: Stopping in %d seconds" % interval)
self.stoptimer = interval
return
print("Live view thread: Stopping thread")
self.alive = False
self.wait() # waits until run stops on his own
def run(self):
self.alive = True
print("Live view thread: started")
if self.filetype in [LiveView.FILETYPE_CBF,
LiveView.FILETYPE_TIF]: # noqa F821
# open viewer
while self.alive:
# find latest image
self.mutex.lock()
# get latest file from reveiver
[metadata, _] = self.query.get(2000)
receivedfile = (self.query.generate_target_filepath(
self.basepath, metadata))
print("Next file: ", receivedfile)
if receivedfile is None:
self.mutex.unlock()
continue
# time.sleep(0.2)
# display image
# try:
# self.subframe.loadFile(receivedfile)
# # viewer or subframe has been closed by the user
# except:
# self.mutex.unlock()
# time.sleep(0.1)
# try:
# self.subframe = self.viewer.openSubFrame()
# except:
# self.viewer = albula.openMainFrame()
# self.subframe = self.viewer.openSubFrame()
# continue
self.mutex.unlock()
# wait interval
interval = 0.0
while interval < self.interval and self.alive:
if self.stoptimer > 0.0:
self.stoptimer -= 0.05
if self.stoptimer < 0.0:
self.stoptimer = -1.0
self.alive = False
time.sleep(0.05)
interval += 0.05
elif self.filetype == LiveView.FILETYPE_HDF5: # noqa F821
print("Live view thread: HDF5 not supported yet")
print("Live view thread: Thread for Live view died")
self.alive = False
def setPath(self, path=None): # noqa N802
self.mutex.lock()
if path is not None:
self.path = path
self.mutex.unlock()
def setFiletype(self, filetype=None): # noqa N802
restart = False
if self.alive:
restart = True
self.stop()
if filetype is not None:
self.filetype = filetype
if restart:
self.start()
def setInterval(self, interval=None): # noqa N802
if interval is not None:
self.interval = interval
def __exit__(self):
self.query.stop()
def __del__(self):
self.query.stop()
class Worker(multiprocessing.Process):
"""Start a connection to hidra an get a data block.
"""
def __init__(self, identifier, transfer_type, basepath, signal_host,
target_host, port):
super().__init__()
self.identifier = identifier
self.port = port
self.transfer_type = transfer_type
self.log = logging.getLogger("Worker-{}".format(self.identifier))
self.query = Transfer(self.transfer_type, signal_host, use_log=True)
self.basepath = basepath
self.log.debug("start Transfer on port %s", port)
# targets are locally
self.query.start([target_host, port])
# self.query.start(port)
self.run()
def run(self):
while True:
try:
self.log.debug("Worker-%s: waiting", self.identifier)
[metadata, data] = self.query.get()
time.sleep(0.1)
except Exception:
break
if self.transfer_type in [
"QUERY_NEXT_METADATA", "STREAM_METADATA"
]:
self.log.debug("Worker-%s: metadata %s", self.identifier,
metadata["filename"])
filepath = generate_filepath(self.basepath, metadata)
self.log.debug("Worker-%s: filepath %s", self.identifier,
filepath)
with open(filepath, "r") as file_descriptor:
file_descriptor.read()
self.log.debug("Worker-%s: file %s read", self.identifier,
filepath)
else:
print("filepath", generate_filepath(self.basepath, metadata))
print("metadata", metadata)
print("data", str(data)[:100])
def stop(self):
"""Clean up.
"""
self.query.stop()
def __exit__(self, exception_type, exception_value, traceback):
self.stop()
def __del__(self):
self.stop()
def main():
"""Register and start workers.
"""
# enable logging
logfile_path = os.path.join(BASE_DIR, "logs")
logfile = os.path.join(logfile_path, "test_onda.log")
utils.init_logging(logfile, True, "DEBUG")
parser = argparse.ArgumentParser()
parser.add_argument("--signal_host",
type=str,
help="Host where HiDRA is running",
default=socket.getfqdn())
parser.add_argument("--target_host",
type=str,
help="Host where the data should be send to",
default=socket.getfqdn())
parser.add_argument("--procname",
type=str,
help="Name with which the service should be running",
default="example_onda")
arguments = parser.parse_args()
setproctitle.setproctitle(arguments.procname) # pylint: disable=no-member
transfer_type = "QUERY_NEXT"
# transfer_type = "STREAM"
# transfer_type = "STREAM_METADATA"
# transfer_type = "QUERY_NEXT_METADATA"
basepath = os.path.join(BASE_DIR, "data", "target")
number_of_worker = 3
workers = []
targets = []
for i in range(number_of_worker):
port = str(50200 + i)
targets.append([arguments.target_host, port, 1, [".cbf"]])
proc = multiprocessing.Process(target=Worker,
args=(i, transfer_type, basepath,
arguments.signal_host,
arguments.target_host, port))
workers.append(proc)
query = Transfer(transfer_type, arguments.signal_host, use_log=True)
query.initiate(targets)
for i in workers:
i.start()
try:
while True:
pass
except Exception:
pass
finally:
for i in workers:
i.terminate()
query.stop()
def main():
"""Requests data from hidra on a query basis.
"""
parser = argparse.ArgumentParser()
parser.add_argument("--signal_host",
type=str,
help="Host where HiDRA is running",
default=socket.getfqdn())
parser.add_argument("--target_host",
type=str,
help="Host where the data should be send to",
default=socket.getfqdn())
arguments = parser.parse_args()
# targets = [[arguments.target_host, "50101", 1]]
targets = [[arguments.target_host, "50101", 1, ".*(tif|cbf)$"]]
# targets = [[arguments.target_host, "50101", 1, [".tif", ".cbf"]]]
print("\n==== TEST: Query for the newest filename ====\n")
query = Transfer("QUERY_NEXT", arguments.signal_host)
query.initiate(targets)
try:
query.start()
except Exception:
query.stop()
return
use_md5sum = False
timeout = None
# timeout = 2000 # in ms
while True:
try:
[metadata, data] = query.get(timeout)
except Exception:
print(sys.exc_info())
break
print()
if metadata and data:
print("metadata", metadata["filename"])
print("data", str(data)[:10])
# generate md5sum
if use_md5sum:
md5sum = hashlib.md5()
md5sum.update(data)
print("md5sum", md5sum.hexdigest())
else:
print("metadata", metadata)
print("data", data)
print()
query.stop()
print("\n==== TEST END: Query for the newest filename ====\n")
class Passthrough(Operations):
def __init__(self, signal_host):
self.log = self.__get_logger()
self.timeout = 2000
self.read_pointer = 0
targets = [socket.getfqdn(), "50101", 1]
# create HiDRA Transfer instance which wants data by request only
self.query = Transfer("QUERY_NEXT", signal_host)
self.query.initiate(targets)
self.query.start()
self.metadata = None
self.data = None
# utils
# ==================
def __get_logger(self):
# create the default logger used by the logging mixin
log = logging.getLogger("fuse.log-mixin")
log.setLevel(logging.DEBUG)
# create console handler with a higher log level
handler = logging.StreamHandler()
handler.setLevel(logging.DEBUG)
# add the handlers to the logger
log.addHandler(handler)
return log
# Filesystem methods
# ==================
# def access(self, path, mode):
# pass
# def chmod(self, path, mode):
# pass
# def chown(self, path, uid, gid):
# pass
def getattr(self, path, fh=None):
self.log.debug("path=%s", path)
if path == "/" or path.startswith("/.Trash"):
st = os.lstat(path) # pylint: disable=invalid-name
return {
"st_mode": getattr(st, "st_mode"),
"st_nlink": getattr(st, "st_nlink"),
"st_uid": getattr(st, "st_uid"),
"st_gid": getattr(st, "st_gid"),
"st_ctime": getattr(st, "st_ctime"),
"st_mtime": getattr(st, "st_mtime"),
"st_size": getattr(st, "st_size")
}
else:
if self.metadata is None and self.data is None:
self.log.debug("get")
[self.metadata, self.data] = self.query.get(self.timeout)
return {
"st_mode": (stat.S_IFREG | 0644),
"st_nlink": 1,
"st_uid": 1000,
"st_gid": 1000,
"st_ctime": self.metadata["file_create_time"],
"st_mtime": self.metadata["file_mod_time"],
"st_size": self.metadata["filesize"]
}
def readdir(self, path, fh):
# if self.metadata is None and self.data is None:
[self.metadata, self.data] = self.query.get(self.timeout)
if self.metadata is None:
return [".", ".."]
else:
return [".", "..", self.metadata["filename"]]
# The method readlink() returns a string representing the path to which the
# symbolic link points. It may return an absolute or relative pathname.
# def readlink(self, path):
# pass
# The method mknod() creates a filesystem node (file, device special file
# or named pipe) named filename.
# def mknod(self, path, mode, dev):
# pass
# def rmdir(self, path):
# pass
# def mkdir(self, path, mode):
# pass
# The method statvfs() perform a statvfs system call on the given path.
# def statfs(self, path):
# pass
# The method unlink() removes (deletes) the file path. If the path is a
# directory, OSError is raised.
# def unlink(self, path):
# pass
# The method symlink() creates a symbolic link dst pointing to src.
# def symlink(self, name, target):
# pass
# def rename(self, old, new):
# pass
# def link(self, target, name):
# signal_host = "zitpcx19282.desy.de"
# targets = ["zitpcx19282.desy.de", "50101", 1]
# pass
# The method utime() sets the access and modified times of the file
# specified by path.
# def utimens(self, path, times=None):
# pass
# File methods
# ============
# The method open() opens the file file and set various flags according to
# flags and possibly its mode according to mode.The default mode is 0777
# (octal), and the current umask value is first masked out.
def open(self, path, flags):
# self.log.debug("open")
if self.metadata is None and self.data is None:
self.log.debug("get")
[self.metadata, self.data] = self.query.get(self.timeout)
# for reading
self.read_pointer = 0
return 0
# def create(self, path, mode, fi=None):
# pass
def read(self, path, length, offset, fh):
# self.log.debug("read")
self.read_pointer += length
return self.data[self.read_pointer - length:self.read_pointer]
# def write(self, path, buf, offset, fh):
# pass
# The method truncate() truncates the file's size. The file is truncated to
# (at most) that size of the argument length
def truncate(self, path, length, fh=None):
self.log.debug("truncate")
# The method fsync() forces write of file with file descriptor fd to disk.
# def flush(self, path, fh):
# self.release(path, fh)
def release(self, path, fh):
# self.log.debug("release")
self.metadata = None
self.data = None