def get_event(self):
"""
Retrieves and returns an event from the queues.
First it checks the high priority queue, if fails then checks the regular event queue.
If there are no more events, then it returns the no_event_event, which is defined in the configuration.
"""
try:
try:
return self.event_queue_hi.get_nowait()
except:
ev = self.event_queue.get(True, self.config.event_timeout)
self.wait_for_event = False
return ev
except Exception as e:
if self.wait_for_event:
ev = Event("no_event", None)
else:
ev = self.config.no_event_event
if ev is None:
return None
time.sleep(self.config.no_event_wait_time)
ev.args = Arguments(name=ev.name,
time=datetime.datetime.ctime(
datetime.datetime.now()))
return ev
def append_event(self, event_name, args):
"""
Appends low priority event to the end of the queue
"""
if args is None:
args = self.store_arguments
self.event_queue.put(Event(event_name, args))
def push_hi_event(self, event_name, args):
"""
Creates a new event and appends it to the high priority event queue.
The high priority queue is local to the current process.
The new event is not recurrent.
"""
self.event_queue_hi.put(Event(event_name, args, recurrent=False))
def append_item(self, filename):
"""
Appends item, if not already exists.
"""
if filename is None:
self.logger.warning(f"filename is defined")
return
if not os.path.isfile(filename):
self.logger.warning(f"{filename} is not a file")
return
if filename in self.data_table.index:
self.logger.warning(f"{filename} is already in the table")
return
row = self.digest_new_item(filename)
if not row is None:
short = os.path.basename(filename)
self.logger.debug(f"Appending {short} to the data set")
self.data_table = self.data_table.append(row)
try:
self.event_queue.put(
Event(self.config.default_ingestion_event,
Arguments(name=filename)))
except:
self.logger.warning(
"There is no default ingestion event in the configuration file"
)
def test_shared_queue_producer():
# Append items to the shared queue
queue = get_event_queue(QueueHost, QueuePortNr, QueueAuthCode)
assert queue is not None, "Failed to connect to queue manager"
for cnt in range(NItems):
queue.put(Event(f"Item {cnt}", None))
def test_simple_queue():
eq = SimpleEventQueue()
for i in range(5):
eq.put(Event("test event", Arguments(name="test argument", i=i)))
assert eq.qsize() == 5, "Size mismatch"
e1 = eq.get()
e2 = eq.get()
assert e2.args.i == 1, "Wrong event argument"
def _push_event(self, event_name, args):
"""
Pushes high priority events
Normal events go to the lower priority queue
This method is only used in execute.
"""
self.logger.info(f"Push event {event_name}, {args.name}")
self.event_queue_hi.put(Event(event_name, args))
def ingest_data(self, path=None, files=None):
"""
Adds files to the data_set.
The data_set resides in the framework context.
"""
ds = self.context.data_set
if ds is None:
# Data_set will scan and import the content of the directory
ds = Data_set(path, self.logger, self.config)
if files is not None:
for f in files:
ds.append_item(f)
for ditem in ds.data_table.index:
self.event_queue.put(Event("next_file", Arguments(name=ditem)))
self.context.data_set = ds
def start(self,
qm_only=False,
ingest_data_only=False,
wait_for_event=False,
continuous=False):
if qm_only:
self.logger.info("Queue manager only mode, no processing")
self.waitForEver()
else:
if ingest_data_only:
# Release the queue
self.end()
else:
if continuous:
self.config.no_event_event = Event("no_event", None)
self.wait_for_event = wait_for_event
self._start()
self.logger.info("Framework main loop started")
self.waitForEver()
def append_event(self, event_name, args, recurrent=False):
"""
Creates a new event and appends it to the low priority event queue.
The low priority queue is local to the current process.
"""
self.event_queue.put(Event(event_name, args, recurrent=recurrent))
def push_event(self, name, args):
self.event_queue_hi.put(Event(name, args))
type=str,
help="Port number")
parser.add_argument(dest="event_name", type=str, help="Event name")
parser.add_argument(dest="event_argument", type=str, help="Event argument")
try:
return parser.parse_args(in_args[1:])
except:
# parser.print_help()
sys.exit(0)
if __name__ == "__main__":
args = _parseArguments(sys.argv)
cfg = ConfigClass(args.config_file)
cfg.properties["want_multiprocessing"] = True
hostname = cfg.queue_manager_hostname if args.hostname is None else args.hostname
portnr = cfg.queue_manager_portnr if args.portnr is None else args.portnr
auth_code = cfg.queue_manager_auth_code
queue = queues.get_event_queue(hostname, portnr, auth_code)
print("Event queue:")
print(f"Hostname = {hostname}\nPort nr = {portnr}\n")
if queue is None:
print("Failed to connect to Queue Manager")
else:
event = Event(args.event_name, Arguments(name=args.event_argument))
queue.put(event)
print("Event added")