def main(): """Get the hidra version on the sending side. """ # enable logging logfile_path = os.path.join(BASE_DIR, "logs") logfile = os.path.join(logfile_path, "example_get_remote_version.log") utils.init_logging(logfile, True, "DEBUG") parser = argparse.ArgumentParser() parser.add_argument("--signal_host", type=str, help="Host where HiDRA is running", default=socket.getfqdn()) parser.add_argument("--target_host", type=str, help="Host where the data should be send to", default=socket.getfqdn()) arguments = parser.parse_args() transfer_type = "QUERY_NEXT" query = Transfer(transfer_type, arguments.signal_host, use_log=None) version = query.get_remote_version() print("Remote version: {0}".format(version))
def main(): """Connects to hidra and stores the streamed data to disk. """ parser = argparse.ArgumentParser() parser.add_argument("--signal_host", type=str, help="Host where HiDRA is running", default=socket.getfqdn()) parser.add_argument("--target_host", type=str, help="Host where the data should be send to", default=socket.getfqdn()) arguments = parser.parse_args() # enable logging logfile_path = os.path.join(BASE_DIR, "logs") logfile = os.path.join(logfile_path, "testAPI.log") utils.init_logging(logfile, True, "DEBUG") targets = [arguments.target_host, "50100", 0] print("\n==== TEST: Stream all files and store them ====\n") query = Transfer("STREAM", arguments.signal_host, use_log=True) query.initiate(targets) query.start() target_dir = os.path.join(BASE_DIR, "data", "zmq_target") target_file = os.path.join(target_dir, "test_store") try: query.store(target_file) except Exception as excp: print("Storing data failed.") print("Error was:", excp) query.stop() print("\n==== TEST END: Stream all files and store them ====\n")
def main(): """ The fixes target is configured on the sender side. Start up receiving side. """ # enable logging logfile_path = os.path.join(BASE_DIR, "logs") logfile = os.path.join(logfile_path, "test_fixedStream.log") utils.init_logging(logfile, True, "DEBUG") data_port = "50100" print("\n==== TEST: Fixed stream ====\n") query = Transfer("STREAM", use_log=True) query.start(data_port) while True: try: [metadata, data] = query.get() except KeyboardInterrupt: break except Exception as excp: print("Getting data failed.") print("Error was: {0}".format(excp)) break print() try: print("metadata of file", metadata["filename"]) print("data", str(data)[:10]) except TypeError: print("metadata", metadata) print() query.stop() print("\n==== TEST END: Fixed Stream ====\n")
def main(): """De-registers a connection from a broken down program. """ # enable logging logfile_path = os.path.join(BASE_DIR, "logs") logfile = os.path.join(logfile_path, "example_force_stop.log") utils.init_logging(logfile, True, "DEBUG") if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("--signal_host", type=str, help="Host where HiDRA is running", default=socket.getfqdn()) parser.add_argument("--target_host", type=str, help="Host where the data should be send to", default=socket.getfqdn()) arguments = parser.parse_args() # targets = [[arguments.target_host, "50101", 1, ".*(tif|cbf)$"]] targets = [[arguments.target_host, "50100", 1, [".cbf"]], [arguments.target_host, "50101", 1, [".cbf"]], [arguments.target_host, "50102", 1, [".cbf"]]] transfer_type = "QUERY_NEXT" # transfer_type = "STREAM" # transfer_type = "STREAM_METADATA" # transfer_type = "QUERY_NEXT_METADATA" query = Transfer(transfer_type, arguments.signal_host, use_log=True) query.force_stop(targets)
def main(): """Register and start workers. """ # enable logging logfile_path = os.path.join(BASE_DIR, "logs") logfile = os.path.join(logfile_path, "test_onda.log") utils.init_logging(logfile, True, "DEBUG") parser = argparse.ArgumentParser() parser.add_argument("--signal_host", type=str, help="Host where HiDRA is running", default=socket.getfqdn()) parser.add_argument("--target_host", type=str, help="Host where the data should be send to", default=socket.getfqdn()) parser.add_argument("--procname", type=str, help="Name with which the service should be running", default="example_onda") arguments = parser.parse_args() setproctitle.setproctitle(arguments.procname) # pylint: disable=no-member transfer_type = "QUERY_NEXT" # transfer_type = "STREAM" # transfer_type = "STREAM_METADATA" # transfer_type = "QUERY_NEXT_METADATA" basepath = os.path.join(BASE_DIR, "data", "target") number_of_worker = 3 workers = [] targets = [] for i in range(number_of_worker): port = str(50200 + i) targets.append([arguments.target_host, port, 1, [".cbf"]]) proc = multiprocessing.Process(target=Worker, args=(i, transfer_type, basepath, arguments.signal_host, arguments.target_host, port)) workers.append(proc) query = Transfer(transfer_type, arguments.signal_host, use_log=True) query.initiate(targets) for i in workers: i.start() try: while True: pass except Exception: pass finally: for i in workers: i.terminate() query.stop()
# from __future__ import unicode_literals import os import zmq import logging import socket as socket_m import json from _environment import BASE_DIR from hidra import utils # enable logging logfile_path = os.path.join(BASE_DIR, "logs") logfile = os.path.join(logfile_path, "test_nexus_transfer.log") utils.init_logging(logfile, True, "DEBUG") class Sender (object): def __init__(self): self.ext_host = "0.0.0.0" self.localhost = socket_m.getfqdn() self.signal_port = "50050" self.data_port = "50100" self.context = zmq.Context() self.file_op_socket = self.context.socket(zmq.REQ) connection_str = ("tcp://{0}:{1}" .format(self.localhost, self.signal_port)) self.file_op_socket.connect(connection_str)