def main():
parser = argparse.ArgumentParser()
parser.add_argument("--signal_host",
type=str,
help="Host where HiDRA is running",
default=socket.getfqdn())
parser.add_argument("--target_host",
type=str,
help="Host where the data should be send to",
default=socket.getfqdn())
parser.add_argument("--procname",
type=str,
help="Name with which the service should be running",
default="test_query_speed")
parser.add_argument("--workers",
type=int,
help="How many worker processes should be launched",
default=1)
arguments = parser.parse_args()
setproctitle.setproctitle(arguments.procname)
workers = []
number_of_files = multiprocessing.Value('i', 0)
targets = []
transfer_type = "QUERY_NEXT"
for n in range(arguments.workers):
p = str(50100 + n)
w = multiprocessing.Process(
target=Worker,
args=(n, transfer_type, arguments.signal_host,
arguments.target_host, p, number_of_files))
workers.append(w)
targets.append([arguments.target_host, p, 1, [".cbf"]])
query = Transfer(transfer_type, arguments.signal_host, use_log=None)
query.initiate(targets)
for w in workers:
w.start()
try:
while all(w.is_alive() for w in workers):
time.sleep(0.5)
print("number_of_files={0}".format(number_of_files.value))
except KeyboardInterrupt:
pass
finally:
print("number_of_files={0}".format(number_of_files.value))
for w in workers:
w.terminate()
query.stop()
def main():
"""Register and start workers.
"""
parser = argparse.ArgumentParser()
parser.add_argument("--signal_host",
type=str,
help="Host where HiDRA is running",
default=socket.getfqdn())
parser.add_argument("--target_host",
type=str,
help="Host where the data should be send to",
default=socket.getfqdn())
parser.add_argument("--procname",
type=str,
help="Name with which the service should be running",
default="example_onda")
arguments = parser.parse_args()
transfer_type = "QUERY_NEXT"
number_of_worker = 3
workers = []
targets = []
# Create <number_of_worker> workers to receive and process data
for i in range(number_of_worker):
port = str(50100 + i)
targets.append([arguments.target_host, port, 1, [".cbf"]])
proc = multiprocessing.Process(target=Worker,
args=(i, transfer_type,
arguments.signal_host,
arguments.target_host, port))
workers.append(proc)
# register these workers on the sending side
# this is done from the master to enforce that the data received from the
# workers is disjoint
query = Transfer(transfer_type, arguments.signal_host, use_log=False)
query.initiate(targets)
for i in workers:
i.start()
try:
while True:
pass
except Exception:
pass
finally:
for i in workers:
i.terminate()
query.stop()
def main():
"""Requests data from hidra on a query basis.
"""
arguments = get_arguments()
if arguments.debug:
use_log = "debug"
print("Using debug mode")
else:
use_log = False # pylint: disable=redefined-variable-type
targets = [[arguments.target_host, "50101", 1, ".*(tif|cbf)$"]]
detector_id = socket.getfqdn() # the hidra instance to connect to
print("\n==== TEST: Query for the newest filename ====\n")
query = Transfer("QUERY_NEXT",
arguments.signal_host,
use_log=use_log,
detector_id=detector_id)
query.initiate(targets)
try:
query.start()
except Exception:
query.stop()
return
timeout = None
# timeout = 2000 # in ms
while True:
try:
[metadata, data] = query.get(timeout)
except Exception:
print(sys.exc_info())
break
print()
if metadata and data:
print("metadata", metadata["filename"])
print("data", str(data)[:10])
else:
print("metadata", metadata)
print("data", data)
print()
query.stop()
print("\n==== TEST END: Query for the newest filename ====\n")
def main():
"""Requests data from hidra on a query basis.
"""
parser = argparse.ArgumentParser()
parser.add_argument("--signal_host",
type=str,
help="Host where HiDRA is running",
default=socket.getfqdn())
parser.add_argument("--target_host",
type=str,
help="Host where the data should be send to",
default=socket.getfqdn())
arguments = parser.parse_args()
targets = [[arguments.target_host, "50101", 1]]
print("\n==== TEST: Query for the newest filename ====\n")
query = Transfer("QUERY_NEXT", arguments.signal_host, use_log="debug")
query.initiate(targets)
try:
query.start()
except Exception:
query.stop()
return
timeout = None
# timeout = 2000 # in ms
try:
while True:
[metadata, data] = query.get(timeout)
print()
if metadata and data:
print("metadata", metadata["filename"])
print("data", data)
else:
print("metadata", metadata)
print("data", data)
print()
finally:
query.stop()
print("\n==== TEST END: Query for the newest filename ====\n")
def main():
"""Connect to hidra and get the streamed metadata.
"""
parser = argparse.ArgumentParser()
parser.add_argument("--signal_host",
type=str,
help="Host where HiDRA is running",
default=socket.getfqdn())
parser.add_argument("--target_host",
type=str,
help="Host where the data should be send to",
default=socket.getfqdn())
parser.add_argument("--detector_id",
type=str,
help="Which detector to get data from",
default=socket.getfqdn())
arguments = parser.parse_args()
# targets = [arguments.target_host, "50101", 0]
targets = [[arguments.target_host, "50101", 0, ".*(tif|cbf)$"]]
# targets = [[arguments.target_host, "50101", 0, [".tif", ".cbf"]]]
print("\n==== TEST: Stream all files ====\n")
query = Transfer("STREAM_METADATA",
signal_host=arguments.signal_host,
detector_id=arguments.detector_id)
query.initiate(targets)
query.start()
while True:
try:
print("waiting to get metadata")
[metadata, _] = query.get()
except Exception:
break
print()
print("metadata", metadata["filename"])
# print ("data", str(data)[:10])
print()
query.stop()
print("\n==== TEST END: Stream all files ====\n")
def main():
"""Connect to hidra and get streamed data.
"""
parser = argparse.ArgumentParser()
parser.add_argument("--signal_host",
type=str,
help="Host where HiDRA is running",
default=socket.getfqdn())
parser.add_argument("--target_host",
type=str,
help="Host where the data should be send to",
default=socket.getfqdn())
arguments = parser.parse_args()
# targets = [arguments.target_host, "50101", 0]
targets = [[arguments.target_host, "50101", 0, ".*(tif|cbf)$"]]
# targets = [[arguments.target_host, "50101", 0, [".tif", ".cbf"]]]
print("\n==== TEST: Stream all files ====\n")
query = Transfer("STREAM", arguments.signal_host)
query.initiate(targets)
query.start()
while True:
try:
[metadata, data] = query.get()
except Exception:
break
print()
print("metadata", metadata["filename"])
print("data", len(data))
print()
query.stop()
print("\n==== TEST END: Stream all files ====\n")
def main():
"""Connects to hidra and stores the streamed data to disk.
"""
parser = argparse.ArgumentParser()
parser.add_argument("--signal_host",
type=str,
help="Host where HiDRA is running",
default=socket.getfqdn())
parser.add_argument("--target_host",
type=str,
help="Host where the data should be send to",
default=socket.getfqdn())
arguments = parser.parse_args()
# enable logging
logfile_path = os.path.join(BASE_DIR, "logs")
logfile = os.path.join(logfile_path, "testAPI.log")
utils.init_logging(logfile, True, "DEBUG")
targets = [arguments.target_host, "50100", 0]
print("\n==== TEST: Stream all files and store them ====\n")
query = Transfer("STREAM", arguments.signal_host, use_log=True)
query.initiate(targets)
query.start()
target_dir = os.path.join(BASE_DIR, "data", "zmq_target")
target_file = os.path.join(target_dir, "test_store")
try:
query.store(target_file)
except Exception as excp:
print("Storing data failed.")
print("Error was:", excp)
query.stop()
print("\n==== TEST END: Stream all files and store them ====\n")
def main():
"""Connects to hidra and request metadata.
"""
parser = argparse.ArgumentParser()
parser.add_argument("--signal_host",
type=str,
help="Host where HiDRA is running",
default=socket.getfqdn())
parser.add_argument("--target_host",
type=str,
help="Host where the data should be send to",
default=socket.getfqdn())
arguments = parser.parse_args()
targets = [[arguments.target_host, "50101", 0]]
base_target_path = os.path.join(BASE_DIR, "data", "target")
print("\n==== TEST: Query for the newest filename ====\n")
query = Transfer("QUERY_NEXT_METADATA", arguments.signal_host)
query.initiate(targets)
query.start()
try:
while True:
try:
[metadata, _] = query.get()
except Exception:
query.stop()
raise
print()
print(generate_filepath(base_target_path, metadata))
print()
finally:
query.stop()
print("\n==== TEST END: Query for the newest filename ====\n")
def main():
"""Register and start workers.
"""
# enable logging
logfile_path = os.path.join(BASE_DIR, "logs")
logfile = os.path.join(logfile_path, "test_onda.log")
utils.init_logging(logfile, True, "DEBUG")
parser = argparse.ArgumentParser()
parser.add_argument("--signal_host",
type=str,
help="Host where HiDRA is running",
default=socket.getfqdn())
parser.add_argument("--target_host",
type=str,
help="Host where the data should be send to",
default=socket.getfqdn())
parser.add_argument("--procname",
type=str,
help="Name with which the service should be running",
default="example_onda")
arguments = parser.parse_args()
setproctitle.setproctitle(arguments.procname) # pylint: disable=no-member
transfer_type = "QUERY_NEXT"
# transfer_type = "STREAM"
# transfer_type = "STREAM_METADATA"
# transfer_type = "QUERY_NEXT_METADATA"
basepath = os.path.join(BASE_DIR, "data", "target")
number_of_worker = 3
workers = []
targets = []
for i in range(number_of_worker):
port = str(50200 + i)
targets.append([arguments.target_host, port, 1, [".cbf"]])
proc = multiprocessing.Process(target=Worker,
args=(i, transfer_type, basepath,
arguments.signal_host,
arguments.target_host, port))
workers.append(proc)
query = Transfer(transfer_type, arguments.signal_host, use_log=True)
query.initiate(targets)
for i in workers:
i.start()
try:
while True:
pass
except Exception:
pass
finally:
for i in workers:
i.terminate()
query.stop()
class Passthrough(Operations):
def __init__(self, signal_host):
self.log = self.__get_logger()
self.timeout = 2000
self.read_pointer = 0
targets = [socket.getfqdn(), "50101", 1]
# create HiDRA Transfer instance which wants data by request only
self.query = Transfer("QUERY_NEXT", signal_host)
self.query.initiate(targets)
self.query.start()
self.metadata = None
self.data = None
# utils
# ==================
def __get_logger(self):
# create the default logger used by the logging mixin
log = logging.getLogger("fuse.log-mixin")
log.setLevel(logging.DEBUG)
# create console handler with a higher log level
handler = logging.StreamHandler()
handler.setLevel(logging.DEBUG)
# add the handlers to the logger
log.addHandler(handler)
return log
# Filesystem methods
# ==================
# def access(self, path, mode):
# pass
# def chmod(self, path, mode):
# pass
# def chown(self, path, uid, gid):
# pass
def getattr(self, path, fh=None):
self.log.debug("path=%s", path)
if path == "/" or path.startswith("/.Trash"):
st = os.lstat(path) # pylint: disable=invalid-name
return {
"st_mode": getattr(st, "st_mode"),
"st_nlink": getattr(st, "st_nlink"),
"st_uid": getattr(st, "st_uid"),
"st_gid": getattr(st, "st_gid"),
"st_ctime": getattr(st, "st_ctime"),
"st_mtime": getattr(st, "st_mtime"),
"st_size": getattr(st, "st_size")
}
else:
if self.metadata is None and self.data is None:
self.log.debug("get")
[self.metadata, self.data] = self.query.get(self.timeout)
return {
"st_mode": (stat.S_IFREG | 0644),
"st_nlink": 1,
"st_uid": 1000,
"st_gid": 1000,
"st_ctime": self.metadata["file_create_time"],
"st_mtime": self.metadata["file_mod_time"],
"st_size": self.metadata["filesize"]
}
def readdir(self, path, fh):
# if self.metadata is None and self.data is None:
[self.metadata, self.data] = self.query.get(self.timeout)
if self.metadata is None:
return [".", ".."]
else:
return [".", "..", self.metadata["filename"]]
# The method readlink() returns a string representing the path to which the
# symbolic link points. It may return an absolute or relative pathname.
# def readlink(self, path):
# pass
# The method mknod() creates a filesystem node (file, device special file
# or named pipe) named filename.
# def mknod(self, path, mode, dev):
# pass
# def rmdir(self, path):
# pass
# def mkdir(self, path, mode):
# pass
# The method statvfs() perform a statvfs system call on the given path.
# def statfs(self, path):
# pass
# The method unlink() removes (deletes) the file path. If the path is a
# directory, OSError is raised.
# def unlink(self, path):
# pass
# The method symlink() creates a symbolic link dst pointing to src.
# def symlink(self, name, target):
# pass
# def rename(self, old, new):
# pass
# def link(self, target, name):
# signal_host = "zitpcx19282.desy.de"
# targets = ["zitpcx19282.desy.de", "50101", 1]
# pass
# The method utime() sets the access and modified times of the file
# specified by path.
# def utimens(self, path, times=None):
# pass
# File methods
# ============
# The method open() opens the file file and set various flags according to
# flags and possibly its mode according to mode.The default mode is 0777
# (octal), and the current umask value is first masked out.
def open(self, path, flags):
# self.log.debug("open")
if self.metadata is None and self.data is None:
self.log.debug("get")
[self.metadata, self.data] = self.query.get(self.timeout)
# for reading
self.read_pointer = 0
return 0
# def create(self, path, mode, fi=None):
# pass
def read(self, path, length, offset, fh):
# self.log.debug("read")
self.read_pointer += length
return self.data[self.read_pointer - length:self.read_pointer]
# def write(self, path, buf, offset, fh):
# pass
# The method truncate() truncates the file's size. The file is truncated to
# (at most) that size of the argument length
def truncate(self, path, length, fh=None):
self.log.debug("truncate")
# The method fsync() forces write of file with file descriptor fd to disk.
# def flush(self, path, fh):
# self.release(path, fh)
def release(self, path, fh):
# self.log.debug("release")
self.metadata = None
self.data = None
def main():
"""Requests data from hidra on a query basis.
"""
parser = argparse.ArgumentParser()
parser.add_argument("--signal_host",
type=str,
help="Host where HiDRA is running",
default=socket.getfqdn())
parser.add_argument("--target_host",
type=str,
help="Host where the data should be send to",
default=socket.getfqdn())
arguments = parser.parse_args()
# targets = [[arguments.target_host, "50101", 1]]
targets = [[arguments.target_host, "50101", 1, ".*(tif|cbf)$"]]
# targets = [[arguments.target_host, "50101", 1, [".tif", ".cbf"]]]
print("\n==== TEST: Query for the newest filename ====\n")
query = Transfer("QUERY_NEXT", arguments.signal_host)
query.initiate(targets)
try:
query.start()
except Exception:
query.stop()
return
use_md5sum = False
timeout = None
# timeout = 2000 # in ms
while True:
try:
[metadata, data] = query.get(timeout)
except Exception:
print(sys.exc_info())
break
print()
if metadata and data:
print("metadata", metadata["filename"])
print("data", str(data)[:10])
# generate md5sum
if use_md5sum:
md5sum = hashlib.md5()
md5sum.update(data)
print("md5sum", md5sum.hexdigest())
else:
print("metadata", metadata)
print("data", data)
print()
query.stop()
print("\n==== TEST END: Query for the newest filename ====\n")
class LiveView(QThread):
FILETYPE_CBF = 0
FILETYPE_TIF = 1
FILETYPE_HDF5 = 2
alive = False
path = ""
filetype = 0
interval = 0.5 # s
stoptimer = -1.0
viewer = None
subframe = None
mutex = None
query = None
transfer_type = socket.getfqdn()
data_port = "50122"
basepath = os.path.join(BASE_DIR, "data", "target")
# basepath = "/gpfs"
def __init__(self, path=None, filetype=None, interval=None, parent=None):
QThread.__init__(self, parent)
if path is not None:
self.path = path
if filetype is not None:
self.filetype = filetype
if interval is not None:
self.interval = interval
# suffix = [".cbf", ".tif", ".hdf5"]
self.query = Transfer("QUERY_NEXT_METADATA", self.transfer_type)
self.query.initiate([socket.getfqdn(), self.data_port, "1"])
self.query.start(self.data_port)
# self.query.initiate(["zitpcx22614w", self.data_port, "1"])
# self.query.start(["zitpcx22614w", self.data_port])
self.mutex = QMutex()
def start(self, path=None, filetype=None, interval=None):
if path is not None:
self.path = path
if filetype is not None:
self.filetype = filetype
if interval is not None:
self.interval = interval
QThread.start(self)
def stop(self, interval=0.0):
if self.stoptimer < 0.0 and interval > 0.0:
print("Live view thread: Stopping in %d seconds" % interval)
self.stoptimer = interval
return
print("Live view thread: Stopping thread")
self.alive = False
self.wait() # waits until run stops on his own
def run(self):
self.alive = True
print("Live view thread: started")
if self.filetype in [LiveView.FILETYPE_CBF,
LiveView.FILETYPE_TIF]: # noqa F821
# open viewer
while self.alive:
# find latest image
self.mutex.lock()
# get latest file from reveiver
[metadata, _] = self.query.get(2000)
receivedfile = (self.query.generate_target_filepath(
self.basepath, metadata))
print("Next file: ", receivedfile)
if receivedfile is None:
self.mutex.unlock()
continue
# time.sleep(0.2)
# display image
# try:
# self.subframe.loadFile(receivedfile)
# # viewer or subframe has been closed by the user
# except:
# self.mutex.unlock()
# time.sleep(0.1)
# try:
# self.subframe = self.viewer.openSubFrame()
# except:
# self.viewer = albula.openMainFrame()
# self.subframe = self.viewer.openSubFrame()
# continue
self.mutex.unlock()
# wait interval
interval = 0.0
while interval < self.interval and self.alive:
if self.stoptimer > 0.0:
self.stoptimer -= 0.05
if self.stoptimer < 0.0:
self.stoptimer = -1.0
self.alive = False
time.sleep(0.05)
interval += 0.05
elif self.filetype == LiveView.FILETYPE_HDF5: # noqa F821
print("Live view thread: HDF5 not supported yet")
print("Live view thread: Thread for Live view died")
self.alive = False
def setPath(self, path=None): # noqa N802
self.mutex.lock()
if path is not None:
self.path = path
self.mutex.unlock()
def setFiletype(self, filetype=None): # noqa N802
restart = False
if self.alive:
restart = True
self.stop()
if filetype is not None:
self.filetype = filetype
if restart:
self.start()
def setInterval(self, interval=None): # noqa N802
if interval is not None:
self.interval = interval
def __exit__(self):
self.query.stop()
def __del__(self):
self.query.stop()
help="Name with which the service should be running",
default="example_onda")
arguments = parser.parse_args()
transfer_type = "QUERY_NEXT"
number_of_worker = 3
workers = []
targets = []
# Create <number_of_worker> workers to receive and process data
for n in range(number_of_worker):
p = str(50100 + n)
targets.append([arguments.target_host, p, 1, [".cbf"]])
# register these workers on the sending side
# this is done from the master to enforce that the data received from the
# workers is disjoint
query = Transfer(transfer_type, arguments.signal_host, use_log=False)
query.initiate(targets)
query.stop = lambda: None
try:
while True:
pass
except Exception:
pass
def main():
"""Requests data from hidra on a query basis.
"""
parser = argparse.ArgumentParser()
parser.add_argument("--signal_host",
type=str,
help="Host where HiDRA is running",
default=socket.getfqdn())
parser.add_argument("--target_host",
type=str,
help="Host where the data should be send to",
default=socket.getfqdn())
parser.add_argument("--detector_id",
type=str,
help="Used for connection with control server",
default=None)
parser.add_argument("--query_type",
type=str,
help="Query type like QUERY_NEXT",
default=True)
parser.add_argument("--max_queries",
type=int,
help="Maximum number of queries",
default=None)
parser.add_argument("--timeout",
type=int,
help="Timeout for get call",
default=2000)
args = parser.parse_args()
targets = [[args.target_host, "50101", 1, ".*(tif|cbf)$"]]
timeout = args.timeout
query = Transfer(args.query_type,
signal_host=args.signal_host,
detector_id=args.detector_id,
use_log="DEBUG") # should use the logging module
query.initiate(targets)
try:
query.start()
except Exception:
query.stop()
return
print("Begin query...", file=sys.stderr)
count = 0
while args.max_queries is None or count < args.max_queries:
count += 1
try:
[metadata, data] = query.get(timeout)
except Exception:
print(sys.exc_info(), file=sys.stderr)
break
if metadata and data != "null":
md5sum = hashlib.md5()
md5sum.update(data)
print("{}: {}".format(get_filename(metadata), md5sum.hexdigest()))
elif metadata:
print("{}: null".format(get_filename(metadata)))
break
else:
break
query.stop()