class Worker(multiprocessing.Process):
def __init__(self, worker_id, transfer_type, signal_host, target_host,
port, number_of_files):
super().__init__()
self.id = worker_id
self.basepath = os.path.join(BASE_DIR, "data", "target")
self.number_of_files = number_of_files
self.port = port
print("start Transfer on port", port)
self.query = Transfer(transfer_type, signal_host, use_log=None)
self.query.start([target_host, port])
self.run()
def run(self):
try:
while True:
# print("Worker-{0}: waiting".format(self.id))
[metadata, data] = self.query.get()
if metadata and data:
self.number_of_files.value += 1
finally:
self.stop()
def stop(self):
self.query.stop()
def __exit__(self):
self.stop()
def __del__(self):
self.stop()
def main():
"""Requests data from hidra on a query basis.
"""
arguments = get_arguments()
if arguments.debug:
use_log = "debug"
print("Using debug mode")
else:
use_log = False # pylint: disable=redefined-variable-type
targets = [[arguments.target_host, "50101", 1, ".*(tif|cbf)$"]]
detector_id = socket.getfqdn() # the hidra instance to connect to
print("\n==== TEST: Query for the newest filename ====\n")
query = Transfer("QUERY_NEXT",
arguments.signal_host,
use_log=use_log,
detector_id=detector_id)
query.initiate(targets)
try:
query.start()
except Exception:
query.stop()
return
timeout = None
# timeout = 2000 # in ms
while True:
try:
[metadata, data] = query.get(timeout)
except Exception:
print(sys.exc_info())
break
print()
if metadata and data:
print("metadata", metadata["filename"])
print("data", str(data)[:10])
else:
print("metadata", metadata)
print("data", data)
print()
query.stop()
print("\n==== TEST END: Query for the newest filename ====\n")
def main():
"""Requests data from hidra on a query basis.
"""
parser = argparse.ArgumentParser()
parser.add_argument("--signal_host",
type=str,
help="Host where HiDRA is running",
default=socket.getfqdn())
parser.add_argument("--target_host",
type=str,
help="Host where the data should be send to",
default=socket.getfqdn())
arguments = parser.parse_args()
targets = [[arguments.target_host, "50101", 1]]
print("\n==== TEST: Query for the newest filename ====\n")
query = Transfer("QUERY_NEXT", arguments.signal_host, use_log="debug")
query.initiate(targets)
try:
query.start()
except Exception:
query.stop()
return
timeout = None
# timeout = 2000 # in ms
try:
while True:
[metadata, data] = query.get(timeout)
print()
if metadata and data:
print("metadata", metadata["filename"])
print("data", data)
else:
print("metadata", metadata)
print("data", data)
print()
finally:
query.stop()
print("\n==== TEST END: Query for the newest filename ====\n")
def main():
"""Connect to hidra and get the streamed metadata.
"""
parser = argparse.ArgumentParser()
parser.add_argument("--signal_host",
type=str,
help="Host where HiDRA is running",
default=socket.getfqdn())
parser.add_argument("--target_host",
type=str,
help="Host where the data should be send to",
default=socket.getfqdn())
parser.add_argument("--detector_id",
type=str,
help="Which detector to get data from",
default=socket.getfqdn())
arguments = parser.parse_args()
# targets = [arguments.target_host, "50101", 0]
targets = [[arguments.target_host, "50101", 0, ".*(tif|cbf)$"]]
# targets = [[arguments.target_host, "50101", 0, [".tif", ".cbf"]]]
print("\n==== TEST: Stream all files ====\n")
query = Transfer("STREAM_METADATA",
signal_host=arguments.signal_host,
detector_id=arguments.detector_id)
query.initiate(targets)
query.start()
while True:
try:
print("waiting to get metadata")
[metadata, _] = query.get()
except Exception:
break
print()
print("metadata", metadata["filename"])
# print ("data", str(data)[:10])
print()
query.stop()
print("\n==== TEST END: Stream all files ====\n")
def main():
"""Connect to hidra and get streamed data.
"""
parser = argparse.ArgumentParser()
parser.add_argument("--signal_host",
type=str,
help="Host where HiDRA is running",
default=socket.getfqdn())
parser.add_argument("--target_host",
type=str,
help="Host where the data should be send to",
default=socket.getfqdn())
arguments = parser.parse_args()
# targets = [arguments.target_host, "50101", 0]
targets = [[arguments.target_host, "50101", 0, ".*(tif|cbf)$"]]
# targets = [[arguments.target_host, "50101", 0, [".tif", ".cbf"]]]
print("\n==== TEST: Stream all files ====\n")
query = Transfer("STREAM", arguments.signal_host)
query.initiate(targets)
query.start()
while True:
try:
[metadata, data] = query.get()
except Exception:
break
print()
print("metadata", metadata["filename"])
print("data", len(data))
print()
query.stop()
print("\n==== TEST END: Stream all files ====\n")
def main():
"""Connects to hidra and request metadata.
"""
parser = argparse.ArgumentParser()
parser.add_argument("--signal_host",
type=str,
help="Host where HiDRA is running",
default=socket.getfqdn())
parser.add_argument("--target_host",
type=str,
help="Host where the data should be send to",
default=socket.getfqdn())
arguments = parser.parse_args()
targets = [[arguments.target_host, "50101", 0]]
base_target_path = os.path.join(BASE_DIR, "data", "target")
print("\n==== TEST: Query for the newest filename ====\n")
query = Transfer("QUERY_NEXT_METADATA", arguments.signal_host)
query.initiate(targets)
query.start()
try:
while True:
try:
[metadata, _] = query.get()
except Exception:
query.stop()
raise
print()
print(generate_filepath(base_target_path, metadata))
print()
finally:
query.stop()
print("\n==== TEST END: Query for the newest filename ====\n")
class Worker(multiprocessing.Process):
"""Start a connection to hidra an get a data block.
"""
def __init__(self, identifier, transfer_type, signal_host, target_host,
port): # noqa F811
super().__init__()
self.identifier = identifier
self.port = port
self.query = Transfer(transfer_type, signal_host, use_log=False)
# Set up ZeroMQ for this worker
print("start Transfer on port {0}".format(port))
self.query.start([target_host, port])
self.run()
def run(self):
while True:
try:
# Get new data
print("Worker-{0}: waiting".format(self.identifier))
[metadata, data] = self.query.get()
except Exception:
break
print("metadata", metadata)
print("data", str(data)[:100])
def stop(self):
"""Clean up:
"""
self.query.stop()
def __exit__(self, exception_type, exception_value, traceback):
self.stop()
def __del__(self):
self.stop()
def main():
"""
The fixes target is configured on the sender side. Start up receiving side.
"""
# enable logging
logfile_path = os.path.join(BASE_DIR, "logs")
logfile = os.path.join(logfile_path, "test_fixedStream.log")
utils.init_logging(logfile, True, "DEBUG")
data_port = "50100"
print("\n==== TEST: Fixed stream ====\n")
query = Transfer("STREAM", use_log=True)
query.start(data_port)
while True:
try:
[metadata, data] = query.get()
except KeyboardInterrupt:
break
except Exception as excp:
print("Getting data failed.")
print("Error was: {0}".format(excp))
break
print()
try:
print("metadata of file", metadata["filename"])
print("data", str(data)[:10])
except TypeError:
print("metadata", metadata)
print()
query.stop()
print("\n==== TEST END: Fixed Stream ====\n")
class DataFetcher(DataFetcherBase):
"""
Implementation of the data fetcher reacting on data sent by another
hidra instance.
"""
def __init__(self, datafetcher_base_config):
"""Initial setup
Checks if all required parameters are set in the configuration
"""
self.f_descriptors = dict()
self.transfer = None
DataFetcherBase.__init__(self, datafetcher_base_config, name=__name__)
# base class sets
# self.config_all - all configurations
# self.config_df - the config of the datafetcher
# self.config - the module specific config
# self.df_type - the name of the datafetcher module
# self.log_queue
# self.log
self.metadata_r = None
self.data_r = None
self.set_required_params()
# check that the required_params are set inside of module specific
# config
self.check_config()
self._setup()
def set_required_params(self):
"""
Defines the parameters to be in configuration to run this data fetcher.
Depending if on Linux or Windows other parameters are required.
"""
self.required_params = {
"network": ["ext_ip"],
}
df_params = [
"status_check_resp_port", "confirmation_resp_port", "context"
]
if utils.is_windows():
df_params += ["datafetcher_port"]
else:
self.required_params["network"] += ["ipc_dir", "main_pid"]
self.required_params["datafetcher"] = [
"store_data", {
self.df_type: df_params
}
]
def _setup(self):
"""Sets up and configures the transfer.
"""
self.transfer = Transfer("STREAM", use_log=self.log_queue)
config_net = self.config_all["network"]
endpoint = "{}_{}".format(config_net["main_pid"], "out")
self.transfer.start([config_net["ipc_dir"], endpoint],
protocol="ipc",
data_con_style="connect")
# enable status check requests from any sender
self.transfer.setopt(option="status_check",
value=[
config_net["ext_ip"],
self.config["status_check_resp_port"]
])
# enable confirmation reply if this is requested in a received data
# packet
self.transfer.setopt(option="confirmation",
value=[
config_net["ext_ip"],
self.config["confirmation_resp_port"]
])
def get_metadata(self, targets, metadata):
"""Implementation of the abstract method get_metadata.
Args:
targets (list): The target list this file is supposed to go.
metadata (dict): The dictionary with the metadata to extend.
"""
timeout = 10000
# Get new data
self.metadata_r, self.data_r = self.transfer.get(timeout)
if (metadata["relative_path"] != self.metadata_r["relative_path"]
or metadata["source_path"] != self.metadata_r["source_path"]
or metadata["filename"] != self.metadata_r["filename"]):
self.log.error("Received metadata do not match data")
# Use received data to prevent mismatch of metadata and data
# TODO handle case if file type requested by target does not match
# pylint: disable=attribute-defined-outside-init
# Build source file
self.source_file = generate_filepath(self.metadata_r["source_path"],
self.metadata_r)
# Build target file
# if local_target is not set (== None) generate_filepath returns None
self.target_file = generate_filepath(self.config_df["local_target"],
self.metadata_r)
# Extends metadata
if targets:
if "filesize" not in self.metadata_r:
self.log.error("Received metadata do not contain 'filesize'")
if "file_mod_time" not in self.metadata_r:
self.log.error("Received metadata do not contain "
"'file_mod_time'. Setting it to current time")
self.metadata_r["file_mod_time"] = time.time()
if "file_create_time" not in self.metadata_r:
self.log.error("Received metadata do not contain "
"'file_create_time'. Setting it to current "
"time")
self.metadata_r["file_create_time"] = time.time()
if "chunksize" not in self.metadata_r:
self.log.error("Received metadata do not contain 'chunksize'. "
"Setting it to locally configured one")
self.metadata_r["chunksize"] = self.config_df["chunksize"]
def send_data(self, targets, metadata, open_connections):
"""Implementation of the abstract method send_data.
Args:
targets (list): The target list this file is supposed to go.
metadata (dict): The dictionary with the metadata of the file
open_connections (dict): The dictionary containing all open zmq
connections.
"""
# pylint: disable=unused-argument
if not targets:
return
# targets are of the form [[<host:port>, <prio>, <metadata|data>], ...]
targets_data = [i for i in targets if i[2] == "data"]
if not targets_data:
return
self.log.debug("Received data for file %s (chunknumber %s)",
self.source_file, self.metadata_r["chunk_number"])
self.log.debug("Passing multipart-message for file '%s'...",
self.source_file)
try:
chunk_payload = [
json.dumps(self.metadata_r).encode("utf-8"), self.data_r
]
except Exception:
self.log.error("Unable to pack multipart-message for file "
"'%s'",
self.source_file,
exc_info=True)
return
# send message to data targets
try:
self.send_to_targets(targets=targets_data,
open_connections=open_connections,
metadata=None,
payload=chunk_payload,
chunk_number=self.metadata_r["chunk_number"])
except DataError:
self.log.error(
"Unable to send multipart-message for file '%s' (chunk %s)",
self.source_file,
self.metadata_r["chunk_number"],
exc_info=True)
except Exception:
self.log.error(
"Unable to send multipart-message for file '%s' (chunk %s)",
self.source_file,
self.metadata_r["chunk_number"],
exc_info=True)
def finish(self, targets, metadata, open_connections):
"""Implementation of the abstract method finish.
Args:
targets (list): The target list this file is supposed to go.
metadata (dict): The dictionary with the metadata of the file
open_connections (dict): The dictionary containing all open zmq
connections.
"""
# targets are of the form [[<host:port>, <prio>, <metadata|data>], ...]
targets_metadata = [i for i in targets if i[2] == "metadata"]
# send message to metadata targets
if targets_metadata:
try:
self.send_to_targets(targets=targets_metadata,
open_connections=open_connections,
metadata=metadata,
payload=None,
chunk_number=None,
timeout=self.config["send_timeout"])
self.log.debug(
"Passing metadata multipart-message for file "
"%s...done.", self.source_file)
except Exception:
self.log.error(
"Unable to send metadata multipart-message for file"
"'%s' to '%s'",
self.source_file,
targets_metadata,
exc_info=True)
# store data
if self.config_df["store_data"]:
try:
# TODO: save message to file using a thread (avoids blocking)
self.transfer.store_chunk(
descriptors=self.f_descriptors,
filepath=self.target_file,
payload=self.data_r,
base_path=self.config_df["local_target"],
metadata=self.metadata_r)
except Exception:
self.log.error(
"Storing multipart message for file '%s' failed",
self.source_file,
exc_info=True)
def stop(self):
"""Implementation of the abstract method stop.
"""
# Close base class zmq sockets
self.close_socket()
# Close open file handler to prevent file corruption
for target_file in list(self.f_descriptors):
self.f_descriptors[target_file].close()
del self.f_descriptors[target_file]
# Close zmq sockets
if self.transfer is not None:
self.transfer.stop()
class Worker(multiprocessing.Process):
"""Start a connection to hidra an get a data block.
"""
def __init__(self, identifier, transfer_type, basepath, signal_host,
target_host, port):
super().__init__()
self.identifier = identifier
self.port = port
self.transfer_type = transfer_type
self.log = logging.getLogger("Worker-{}".format(self.identifier))
self.query = Transfer(self.transfer_type, signal_host, use_log=True)
self.basepath = basepath
self.log.debug("start Transfer on port %s", port)
# targets are locally
self.query.start([target_host, port])
# self.query.start(port)
self.run()
def run(self):
while True:
try:
self.log.debug("Worker-%s: waiting", self.identifier)
[metadata, data] = self.query.get()
time.sleep(0.1)
except Exception:
break
if self.transfer_type in [
"QUERY_NEXT_METADATA", "STREAM_METADATA"
]:
self.log.debug("Worker-%s: metadata %s", self.identifier,
metadata["filename"])
filepath = generate_filepath(self.basepath, metadata)
self.log.debug("Worker-%s: filepath %s", self.identifier,
filepath)
with open(filepath, "r") as file_descriptor:
file_descriptor.read()
self.log.debug("Worker-%s: file %s read", self.identifier,
filepath)
else:
print("filepath", generate_filepath(self.basepath, metadata))
print("metadata", metadata)
print("data", str(data)[:100])
def stop(self):
"""Clean up.
"""
self.query.stop()
def __exit__(self, exception_type, exception_value, traceback):
self.stop()
def __del__(self):
self.stop()
class Passthrough(Operations):
def __init__(self, signal_host):
self.log = self.__get_logger()
self.timeout = 2000
self.read_pointer = 0
targets = [socket.getfqdn(), "50101", 1]
# create HiDRA Transfer instance which wants data by request only
self.query = Transfer("QUERY_NEXT", signal_host)
self.query.initiate(targets)
self.query.start()
self.metadata = None
self.data = None
# utils
# ==================
def __get_logger(self):
# create the default logger used by the logging mixin
log = logging.getLogger("fuse.log-mixin")
log.setLevel(logging.DEBUG)
# create console handler with a higher log level
handler = logging.StreamHandler()
handler.setLevel(logging.DEBUG)
# add the handlers to the logger
log.addHandler(handler)
return log
# Filesystem methods
# ==================
# def access(self, path, mode):
# pass
# def chmod(self, path, mode):
# pass
# def chown(self, path, uid, gid):
# pass
def getattr(self, path, fh=None):
self.log.debug("path=%s", path)
if path == "/" or path.startswith("/.Trash"):
st = os.lstat(path) # pylint: disable=invalid-name
return {
"st_mode": getattr(st, "st_mode"),
"st_nlink": getattr(st, "st_nlink"),
"st_uid": getattr(st, "st_uid"),
"st_gid": getattr(st, "st_gid"),
"st_ctime": getattr(st, "st_ctime"),
"st_mtime": getattr(st, "st_mtime"),
"st_size": getattr(st, "st_size")
}
else:
if self.metadata is None and self.data is None:
self.log.debug("get")
[self.metadata, self.data] = self.query.get(self.timeout)
return {
"st_mode": (stat.S_IFREG | 0644),
"st_nlink": 1,
"st_uid": 1000,
"st_gid": 1000,
"st_ctime": self.metadata["file_create_time"],
"st_mtime": self.metadata["file_mod_time"],
"st_size": self.metadata["filesize"]
}
def readdir(self, path, fh):
# if self.metadata is None and self.data is None:
[self.metadata, self.data] = self.query.get(self.timeout)
if self.metadata is None:
return [".", ".."]
else:
return [".", "..", self.metadata["filename"]]
# The method readlink() returns a string representing the path to which the
# symbolic link points. It may return an absolute or relative pathname.
# def readlink(self, path):
# pass
# The method mknod() creates a filesystem node (file, device special file
# or named pipe) named filename.
# def mknod(self, path, mode, dev):
# pass
# def rmdir(self, path):
# pass
# def mkdir(self, path, mode):
# pass
# The method statvfs() perform a statvfs system call on the given path.
# def statfs(self, path):
# pass
# The method unlink() removes (deletes) the file path. If the path is a
# directory, OSError is raised.
# def unlink(self, path):
# pass
# The method symlink() creates a symbolic link dst pointing to src.
# def symlink(self, name, target):
# pass
# def rename(self, old, new):
# pass
# def link(self, target, name):
# signal_host = "zitpcx19282.desy.de"
# targets = ["zitpcx19282.desy.de", "50101", 1]
# pass
# The method utime() sets the access and modified times of the file
# specified by path.
# def utimens(self, path, times=None):
# pass
# File methods
# ============
# The method open() opens the file file and set various flags according to
# flags and possibly its mode according to mode.The default mode is 0777
# (octal), and the current umask value is first masked out.
def open(self, path, flags):
# self.log.debug("open")
if self.metadata is None and self.data is None:
self.log.debug("get")
[self.metadata, self.data] = self.query.get(self.timeout)
# for reading
self.read_pointer = 0
return 0
# def create(self, path, mode, fi=None):
# pass
def read(self, path, length, offset, fh):
# self.log.debug("read")
self.read_pointer += length
return self.data[self.read_pointer - length:self.read_pointer]
# def write(self, path, buf, offset, fh):
# pass
# The method truncate() truncates the file's size. The file is truncated to
# (at most) that size of the argument length
def truncate(self, path, length, fh=None):
self.log.debug("truncate")
# The method fsync() forces write of file with file descriptor fd to disk.
# def flush(self, path, fh):
# self.release(path, fh)
def release(self, path, fh):
# self.log.debug("release")
self.metadata = None
self.data = None
def main():
"""Requests data from hidra on a query basis.
"""
parser = argparse.ArgumentParser()
parser.add_argument("--signal_host",
type=str,
help="Host where HiDRA is running",
default=socket.getfqdn())
parser.add_argument("--target_host",
type=str,
help="Host where the data should be send to",
default=socket.getfqdn())
arguments = parser.parse_args()
# targets = [[arguments.target_host, "50101", 1]]
targets = [[arguments.target_host, "50101", 1, ".*(tif|cbf)$"]]
# targets = [[arguments.target_host, "50101", 1, [".tif", ".cbf"]]]
print("\n==== TEST: Query for the newest filename ====\n")
query = Transfer("QUERY_NEXT", arguments.signal_host)
query.initiate(targets)
try:
query.start()
except Exception:
query.stop()
return
use_md5sum = False
timeout = None
# timeout = 2000 # in ms
while True:
try:
[metadata, data] = query.get(timeout)
except Exception:
print(sys.exc_info())
break
print()
if metadata and data:
print("metadata", metadata["filename"])
print("data", str(data)[:10])
# generate md5sum
if use_md5sum:
md5sum = hashlib.md5()
md5sum.update(data)
print("md5sum", md5sum.hexdigest())
else:
print("metadata", metadata)
print("data", data)
print()
query.stop()
print("\n==== TEST END: Query for the newest filename ====\n")
class LiveView(QThread):
FILETYPE_CBF = 0
FILETYPE_TIF = 1
FILETYPE_HDF5 = 2
alive = False
path = ""
filetype = 0
interval = 0.5 # s
stoptimer = -1.0
viewer = None
subframe = None
mutex = None
query = None
transfer_type = socket.getfqdn()
data_port = "50122"
basepath = os.path.join(BASE_DIR, "data", "target")
# basepath = "/gpfs"
def __init__(self, path=None, filetype=None, interval=None, parent=None):
QThread.__init__(self, parent)
if path is not None:
self.path = path
if filetype is not None:
self.filetype = filetype
if interval is not None:
self.interval = interval
# suffix = [".cbf", ".tif", ".hdf5"]
self.query = Transfer("QUERY_NEXT_METADATA", self.transfer_type)
self.query.initiate([socket.getfqdn(), self.data_port, "1"])
self.query.start(self.data_port)
# self.query.initiate(["zitpcx22614w", self.data_port, "1"])
# self.query.start(["zitpcx22614w", self.data_port])
self.mutex = QMutex()
def start(self, path=None, filetype=None, interval=None):
if path is not None:
self.path = path
if filetype is not None:
self.filetype = filetype
if interval is not None:
self.interval = interval
QThread.start(self)
def stop(self, interval=0.0):
if self.stoptimer < 0.0 and interval > 0.0:
print("Live view thread: Stopping in %d seconds" % interval)
self.stoptimer = interval
return
print("Live view thread: Stopping thread")
self.alive = False
self.wait() # waits until run stops on his own
def run(self):
self.alive = True
print("Live view thread: started")
if self.filetype in [LiveView.FILETYPE_CBF,
LiveView.FILETYPE_TIF]: # noqa F821
# open viewer
while self.alive:
# find latest image
self.mutex.lock()
# get latest file from reveiver
[metadata, _] = self.query.get(2000)
receivedfile = (self.query.generate_target_filepath(
self.basepath, metadata))
print("Next file: ", receivedfile)
if receivedfile is None:
self.mutex.unlock()
continue
# time.sleep(0.2)
# display image
# try:
# self.subframe.loadFile(receivedfile)
# # viewer or subframe has been closed by the user
# except:
# self.mutex.unlock()
# time.sleep(0.1)
# try:
# self.subframe = self.viewer.openSubFrame()
# except:
# self.viewer = albula.openMainFrame()
# self.subframe = self.viewer.openSubFrame()
# continue
self.mutex.unlock()
# wait interval
interval = 0.0
while interval < self.interval and self.alive:
if self.stoptimer > 0.0:
self.stoptimer -= 0.05
if self.stoptimer < 0.0:
self.stoptimer = -1.0
self.alive = False
time.sleep(0.05)
interval += 0.05
elif self.filetype == LiveView.FILETYPE_HDF5: # noqa F821
print("Live view thread: HDF5 not supported yet")
print("Live view thread: Thread for Live view died")
self.alive = False
def setPath(self, path=None): # noqa N802
self.mutex.lock()
if path is not None:
self.path = path
self.mutex.unlock()
def setFiletype(self, filetype=None): # noqa N802
restart = False
if self.alive:
restart = True
self.stop()
if filetype is not None:
self.filetype = filetype
if restart:
self.start()
def setInterval(self, interval=None): # noqa N802
if interval is not None:
self.interval = interval
def __exit__(self):
self.query.stop()
def __del__(self):
self.query.stop()
def main():
"""Requests data from hidra on a query basis.
"""
parser = argparse.ArgumentParser()
parser.add_argument("--signal_host",
type=str,
help="Host where HiDRA is running",
default=socket.getfqdn())
parser.add_argument("--target_host",
type=str,
help="Host where the data should be send to",
default=socket.getfqdn())
parser.add_argument("--detector_id",
type=str,
help="Used for connection with control server",
default=None)
parser.add_argument("--query_type",
type=str,
help="Query type like QUERY_NEXT",
default=True)
parser.add_argument("--max_queries",
type=int,
help="Maximum number of queries",
default=None)
parser.add_argument("--timeout",
type=int,
help="Timeout for get call",
default=2000)
args = parser.parse_args()
targets = [[args.target_host, "50101", 1, ".*(tif|cbf)$"]]
timeout = args.timeout
query = Transfer(args.query_type,
signal_host=args.signal_host,
detector_id=args.detector_id,
use_log="DEBUG") # should use the logging module
query.initiate(targets)
try:
query.start()
except Exception:
query.stop()
return
print("Begin query...", file=sys.stderr)
count = 0
while args.max_queries is None or count < args.max_queries:
count += 1
try:
[metadata, data] = query.get(timeout)
except Exception:
print(sys.exc_info(), file=sys.stderr)
break
if metadata and data != "null":
md5sum = hashlib.md5()
md5sum.update(data)
print("{}: {}".format(get_filename(metadata), md5sum.hexdigest()))
elif metadata:
print("{}: null".format(get_filename(metadata)))
break
else:
break
query.stop()