def dirchecker_poll(self):
tid = 0
found_files = set()
while self.keep_running:
json_msgs = self.dc_read_output()
if len(json_msgs) == 0:
time.sleep(self.poll_time)
continue
logger.debug("Read %d records from dircheckers" % len(json_msgs))
inserted = 0
for json_msg in json_msgs:
fname = json_msg.get('filename', None)
if fname is None or fname in found_files:
continue
found_files.add(fname)
tid_msg = "%s-%s" % (tid, fname)
tid += 1
if 'tid' not in json_msg:
json_msg['tid'] = tid_msg
if os.path.isfile(fname):
self.jfr_add_json_msg(json_msg)
inserted += 1
logger.debug("Submitted %d records from parsing" % inserted)
def jsonupdate_poll(self):
while self.keep_running:
json_msgs = self.jsu_read_output()
m = "jsonupdate_poll: Recieved %d messages for processing"
logger.debug(m % len(json_msgs))
if len(json_msgs) == 0:
time.sleep(self.poll_time)
continue
for json_msg in json_msgs:
self.esj_add_json_msg(json_msg)
self.pgsj_add_json_msg(json_msg)
def rmfiles_poll(self):
lm = "Remove file completed for %s tid: %s removed: %s error: %s"
while self.keep_running:
data = self.rmf_read_output()
if len(data) == 0:
time.sleep(self.poll_time)
continue
for d in data:
tid = d.get('tid', None)
removed = d.get('removed', 'unknown')
error = d.get('error', None)
filename = d.get('filename', 'unknown')
logger.debug(lm % (filename, tid, removed, error))
def pgsubmitjsonnc_poll(self):
while self.keep_running:
json_msgs = self.pgsj_read_output()
m = "pgsubmitjsonnc_poll: Recieved %d messages for processing"
logger.debug(m % len(json_msgs))
if len(json_msgs) == 0:
time.sleep(self.poll_time)
continue
for d in json_msgs:
tid = d.get('tid', None)
status = d.get('status', None)
logger.debug("Postgres Submit completed tid: %s status: %s" %
(tid, status))
def rmf_add_json_msg(self, json_msg):
if self.rmfiles is None:
return False
fname = json_msg.get('filename', None)
allowed_to_manip = json_msg.get('allowed_to_manip', False)
if fname is not None:
logger.debug("preparing to remove: %s" % fname)
if allowed_to_manip:
return self.generic_msg_add(json_msg, [
self.rmfiles,
], 'rmfiles')
return False
def jsonfilereaders_poll(self):
while self.keep_running:
json_msgs = self.jfr_read_output()
m = "jsonfilereaders_poll: Recieved %d messages for processing"
logger.debug(m % len(json_msgs))
if len(json_msgs) == 0:
time.sleep(self.poll_time)
continue
for json_msg in json_msgs:
completed = json_msg.get('completed', False)
fname = json_msg.get('filename', None)
if completed:
m = "%s is ready to be removed, sending it to rm task"
logger.debug(m % fname)
self.rmf_add_json_msg(json_msg)
if self.jsonupdates is not None:
# print("Adding message tp jsu: %s" % json_msg)
self.jsu_add_json_msg(json_msg)
else:
self.esj_add_json_msg(json_msg)
self.pgsj_add_json_msg(json_msg)
def generic_msg_add(self, json_msg, objs, name):
bm = "Adding msg to %s queues" % name
fm = "Failed to add msgs to %s queues" % name
em = "Successfully to added msgs to %s queues" % name
logger.debug(bm)
added = False
# dont try to add a message if objs are None
if all([o is None for o in objs]):
return added
for obj in objs:
if objs is not None:
obj.add_json_msg(json_msg)
added = True
if not added:
logger.debug(fm)
else:
logger.debug(em)
return added