def main():
"""Get the hidra version on the sending side.
"""
# enable logging
logfile_path = os.path.join(BASE_DIR, "logs")
logfile = os.path.join(logfile_path, "example_get_remote_version.log")
utils.init_logging(logfile, True, "DEBUG")
parser = argparse.ArgumentParser()
parser.add_argument("--signal_host",
type=str,
help="Host where HiDRA is running",
default=socket.getfqdn())
parser.add_argument("--target_host",
type=str,
help="Host where the data should be send to",
default=socket.getfqdn())
arguments = parser.parse_args()
transfer_type = "QUERY_NEXT"
query = Transfer(transfer_type, arguments.signal_host, use_log=None)
version = query.get_remote_version()
print("Remote version: {0}".format(version))
def main():
"""Connects to hidra and stores the streamed data to disk.
"""
parser = argparse.ArgumentParser()
parser.add_argument("--signal_host",
type=str,
help="Host where HiDRA is running",
default=socket.getfqdn())
parser.add_argument("--target_host",
type=str,
help="Host where the data should be send to",
default=socket.getfqdn())
arguments = parser.parse_args()
# enable logging
logfile_path = os.path.join(BASE_DIR, "logs")
logfile = os.path.join(logfile_path, "testAPI.log")
utils.init_logging(logfile, True, "DEBUG")
targets = [arguments.target_host, "50100", 0]
print("\n==== TEST: Stream all files and store them ====\n")
query = Transfer("STREAM", arguments.signal_host, use_log=True)
query.initiate(targets)
query.start()
target_dir = os.path.join(BASE_DIR, "data", "zmq_target")
target_file = os.path.join(target_dir, "test_store")
try:
query.store(target_file)
except Exception as excp:
print("Storing data failed.")
print("Error was:", excp)
query.stop()
print("\n==== TEST END: Stream all files and store them ====\n")
def main():
"""
The fixes target is configured on the sender side. Start up receiving side.
"""
# enable logging
logfile_path = os.path.join(BASE_DIR, "logs")
logfile = os.path.join(logfile_path, "test_fixedStream.log")
utils.init_logging(logfile, True, "DEBUG")
data_port = "50100"
print("\n==== TEST: Fixed stream ====\n")
query = Transfer("STREAM", use_log=True)
query.start(data_port)
while True:
try:
[metadata, data] = query.get()
except KeyboardInterrupt:
break
except Exception as excp:
print("Getting data failed.")
print("Error was: {0}".format(excp))
break
print()
try:
print("metadata of file", metadata["filename"])
print("data", str(data)[:10])
except TypeError:
print("metadata", metadata)
print()
query.stop()
print("\n==== TEST END: Fixed Stream ====\n")
def main():
"""De-registers a connection from a broken down program.
"""
# enable logging
logfile_path = os.path.join(BASE_DIR, "logs")
logfile = os.path.join(logfile_path, "example_force_stop.log")
utils.init_logging(logfile, True, "DEBUG")
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("--signal_host",
type=str,
help="Host where HiDRA is running",
default=socket.getfqdn())
parser.add_argument("--target_host",
type=str,
help="Host where the data should be send to",
default=socket.getfqdn())
arguments = parser.parse_args()
# targets = [[arguments.target_host, "50101", 1, ".*(tif|cbf)$"]]
targets = [[arguments.target_host, "50100", 1, [".cbf"]],
[arguments.target_host, "50101", 1, [".cbf"]],
[arguments.target_host, "50102", 1, [".cbf"]]]
transfer_type = "QUERY_NEXT"
# transfer_type = "STREAM"
# transfer_type = "STREAM_METADATA"
# transfer_type = "QUERY_NEXT_METADATA"
query = Transfer(transfer_type, arguments.signal_host, use_log=True)
query.force_stop(targets)
def main():
"""Register and start workers.
"""
# enable logging
logfile_path = os.path.join(BASE_DIR, "logs")
logfile = os.path.join(logfile_path, "test_onda.log")
utils.init_logging(logfile, True, "DEBUG")
parser = argparse.ArgumentParser()
parser.add_argument("--signal_host",
type=str,
help="Host where HiDRA is running",
default=socket.getfqdn())
parser.add_argument("--target_host",
type=str,
help="Host where the data should be send to",
default=socket.getfqdn())
parser.add_argument("--procname",
type=str,
help="Name with which the service should be running",
default="example_onda")
arguments = parser.parse_args()
setproctitle.setproctitle(arguments.procname) # pylint: disable=no-member
transfer_type = "QUERY_NEXT"
# transfer_type = "STREAM"
# transfer_type = "STREAM_METADATA"
# transfer_type = "QUERY_NEXT_METADATA"
basepath = os.path.join(BASE_DIR, "data", "target")
number_of_worker = 3
workers = []
targets = []
for i in range(number_of_worker):
port = str(50200 + i)
targets.append([arguments.target_host, port, 1, [".cbf"]])
proc = multiprocessing.Process(target=Worker,
args=(i, transfer_type, basepath,
arguments.signal_host,
arguments.target_host, port))
workers.append(proc)
query = Transfer(transfer_type, arguments.signal_host, use_log=True)
query.initiate(targets)
for i in workers:
i.start()
try:
while True:
pass
except Exception:
pass
finally:
for i in workers:
i.terminate()
query.stop()
# from __future__ import unicode_literals
import os
import zmq
import logging
import socket as socket_m
import json
from _environment import BASE_DIR
from hidra import utils
# enable logging
logfile_path = os.path.join(BASE_DIR, "logs")
logfile = os.path.join(logfile_path, "test_nexus_transfer.log")
utils.init_logging(logfile, True, "DEBUG")
class Sender (object):
def __init__(self):
self.ext_host = "0.0.0.0"
self.localhost = socket_m.getfqdn()
self.signal_port = "50050"
self.data_port = "50100"
self.context = zmq.Context()
self.file_op_socket = self.context.socket(zmq.REQ)
connection_str = ("tcp://{0}:{1}"
.format(self.localhost, self.signal_port))
self.file_op_socket.connect(connection_str)