def perform(self, parent):
logger = parent.logger
msg = parent.msg
import calendar
from sarra.sr_util import timestr2flt
then = timestr2flt(msg.pubtime)
now = time.time()
lag = now - then
if msg.onfly_checksum != msg.checksum:
msg = "checksum differ - %s - %s msg %s" % (
msg.new_file, parent.onfly_checksum, msg.checksum)
if lag > parent.part_check_lag_threshold:
logger.warning(
"part_check might just be referring to an older version of file, but "
+ msg)
return True
else:
logger.error("part_check rejecting " + msg)
return False
logger.info("part_check Checksum matched for : %s" % msg.new_file)
return True
def on_message(self, parent):
logger = parent.logger
msg = parent.msg
if msg.isRetry: return True
import calendar
import humanize
import datetime
import sys
if (parent.msg_total_msgcount == 0):
logger.info(
"msg_total: 0 messages received: 0 msg/s, 0.0 bytes/s, lag: 0.0 s (RESET)"
)
msgtime = timestr2flt(msg.pubtime)
now = nowflt()
parent.msg_total_msgcount = parent.msg_total_msgcount + 1
lag = now - msgtime
parent.msg_total_lag = parent.msg_total_lag + lag
# guess the size of the message payload, ignoring overheads.
parent.msg_total_bytecount += (len(parent.msg.exchange) +
len(parent.msg.topic) +
len(parent.msg.notice) +
len(parent.msg.hdrstr))
#not time to report yet.
if parent.msg_total_interval > now - parent.msg_total_last:
return True
logger.info(
"msg_total: %3d messages received: %5.2g msg/s, %s bytes/s, lag: %4.2g s"
%
(parent.msg_total_msgcount, parent.msg_total_msgcount /
(now - parent.msg_total_start),
humanize.naturalsize(
parent.msg_total_bytecount / (now - parent.msg_total_start),
binary=True,
gnu=True), parent.msg_total_lag / parent.msg_total_msgcount))
# Set the maximum age, in seconds, of a message to retrieve.
if lag > parent.msg_total_maxlag:
logger.warn("total: Excessive lag! Messages posted %s " %
humanize.naturaltime(datetime.timedelta(seconds=lag)))
parent.msg_total_last = now
if (parent.msg_total_count > 0) and (parent.msg_total_msgcount >=
parent.msg_total_count):
os._exit(0)
return True
def perform(self, parent):
import time
if not 'mtime' in parent.msg.headers.keys():
return True
now = nowflt()
mtime = timestr2flt(parent.msg.headers['mtime'])
age = now - mtime
parent.logger.info("file_age %g seconds for %s" %
(age, parent.msg.new_file))
return True
def on_message(self, parent):
logger = parent.logger
msg = parent.msg
import calendar
then = timestr2flt(msg.pubtime)
now = nowflt()
logger.info("print_lag, posted: %s, lag: %g sec. to deliver: %s, " %
(msg.pubtime, (now - then), msg.new_file))
return True
def on_file(self, parent):
logger = parent.logger
msg = parent.msg
import calendar
import humanize
import datetime
from sarra.sr_util import timestr2flt
if (parent.file_total_bytecount == 0):
logger.info(
"file_total: 0 files received: 0 msg/s, 0.0 bytes/s, lag: 0.0 s (RESET)"
)
msgtime = timestr2flt(msg.pubtime)
now = time.time()
parent.file_total_msgcount = parent.file_total_msgcount + 1
lag = now - msgtime
parent.file_total_lag = parent.file_total_lag + lag
(method, psize, ptot, prem, pno) = msg.partstr.split(',')
parent.file_total_bytecount = parent.file_total_bytecount + int(psize)
#not time to report yet.
if parent.file_total_interval > now - parent.file_total_last:
return True
logger.info(
"file_total: %3d files received: %5.2g msg/s, %s bytes/s, lag: %4.2g s"
% (parent.file_total_msgcount, parent.file_total_msgcount /
(now - parent.file_total_start),
humanize.naturalsize(parent.file_total_bytecount /
(now - parent.file_total_start),
binary=True,
gnu=True),
parent.file_total_lag / parent.file_total_msgcount))
# Set the maximum age, in seconds, of a message to retrieve.
if lag > parent.file_total_maxlag:
logger.warn(
"total: Excessive lag! downloading too slowly/late %s behind" %
humanize.naturaltime(datetime.timedelta(seconds=lag)))
parent.file_total_last = now
return True
def on_message(self,parent):
logger = parent.logger
msg = parent.msg
if msg.isRetry : return True
import calendar
import humanize
import datetime
from sarra.sr_util import timestr2flt
if (parent.msg_total_msgcount == 0):
logger.info("msg_total: 0 messages received: 0 msg/s, 0.0 bytes/s, lag: 0.0 s (RESET)" )
msgtime=timestr2flt(msg.pubtime)
now=nowflt()
parent.msg_total_msgcount = parent.msg_total_msgcount + 1
lag=now-msgtime
parent.msg_total_lag = parent.msg_total_lag + lag
# message with sum 'R' and 'L' have no partstr
if parent.msg.partstr :
(method,psize,ptot,prem,pno) = msg.partstr.split(',')
parent.msg_total_bytecount = parent.msg_total_bytecount + int(psize)
#not time to report yet.
if parent.msg_total_interval > now-parent.msg_total_last :
return True
logger.info("msg_total: %3d messages received: %5.2g msg/s, %s bytes/s, lag: %4.2g s" % (
parent.msg_total_msgcount,
parent.msg_total_msgcount/(now-parent.msg_total_start),
humanize.naturalsize(parent.msg_total_bytecount/(now-parent.msg_total_start),binary=True,gnu=True),
parent.msg_total_lag/parent.msg_total_msgcount ))
# Set the maximum age, in seconds, of a message to retrieve.
if lag > parent.msg_total_maxlag :
logger.warn("total: Excessive lag! Messages posted %s " %
humanize.naturaltime(datetime.timedelta(seconds=lag)))
parent.msg_total_last = now
return True
def on_message(self,parent):
logger = parent.logger
msg = parent.msg
import calendar
then=timestr2flt(msg.pubtime)
now=nowflt()
# Set the maximum age, in seconds, of a message to retrieve.
lag=now-then
if lag > int(parent.msg_skip_threshold) :
logger.info("msg_skip_old, Excessive lag: %g sec. Skipping download of: %s, " % (lag, msg.new_file))
return False
return True
def on_message(self, parent):
logger = parent.logger
msg = parent.msg
import calendar
import humanize
import datetime
msgtime = timestr2flt(msg.pubtime)
now = nowflt()
parent.msg_speedo_msgcount = parent.msg_speedo_msgcount + 1
(method, psize, ptot, prem, pno) = msg.partstr.split(',')
parent.msg_speedo_bytecount = parent.msg_speedo_bytecount + int(psize)
#not time to report yet.
if parent.msg_speedo_interval > now - parent.msg_speedo_last:
return True
lag = now - msgtime
logger.info(
"speedo: %3d messages received: %5.2g msg/s, %s bytes/s, lag: %4.2g s"
% (parent.msg_speedo_msgcount, parent.msg_speedo_msgcount /
(now - parent.msg_speedo_last),
humanize.naturalsize(parent.msg_speedo_bytecount /
(now - parent.msg_speedo_last),
binary=True,
gnu=True), lag))
# Set the maximum age, in seconds, of a message to retrieve.
if lag > parent.msg_speedo_maxlag:
logger.warn("speedo: Excessive lag! Messages posted %s " %
humanize.naturaltime(datetime.timedelta(seconds=lag)))
parent.msg_speedo_last = now
parent.msg_speedo_msgcount = 0
parent.msg_speedo_bytecount = 0
return True
def perform(self, parent):
logger = parent.logger
msg = parent.msg
import calendar
import humanize
import datetime
from sarra.sr_util import timestr2flt
if parent.post_total_msgcount == 0:
logger.info(
"post_total: 0 messages posted: 0 msg/s, 0.0 bytes/s, lag: 0.0 s (RESET)"
)
msgtime = timestr2flt(msg.pubtime)
now = time.time()
parent.post_total_msgcount = parent.post_total_msgcount + 1
lag = now - msgtime
parent.post_total_lag = parent.post_total_lag + lag
#(method,psize,ptot,prem,pno) = msg.partstr.split(',')
#parent.post_total_bytecount = parent.post_total_bytecount + int(psize)
#not time to report yet.
if parent.post_total_interval > now - parent.post_total_last:
return True
logger.info(
"post_total: %3d messages posted: %5.2g msg/s, lag: %4.2g s" %
(parent.post_total_msgcount, parent.post_total_msgcount /
(now - parent.post_total_start),
parent.post_total_lag / parent.post_total_msgcount))
# Set the maximum age, in seconds, of a message to retrieve.
if lag > parent.post_total_maxlag:
logger.warn("total: Excessive lag! Messages posted %s " %
humanize.naturaltime(datetime.timedelta(seconds=lag)))
parent.post_total_last = now
return True
def on_message(self,parent):
if not 'delay' in parent.msg.headers:
parent.msg.headers['delay'] = nowstr()
# Test msg delay
elapsedtime = nowflt() - timestr2flt(parent.msg.headers['pubTime'])
if 0 < elapsedtime < 1:
parent.logger.debug("msg_delay received msg")
else:
parent.logger.info("trying msg with {:.3f}s elapsed".format(elapsedtime))
if elapsedtime < parent.msg_delay:
dbg_msg = "message not old enough, sleeping for {:.3f} seconds"
parent.logger.debug(dbg_msg.format(elapsedtime, parent.msg_delay - elapsedtime))
parent.consumer.sleep_now = parent.consumer.sleep_min
parent.consumer.msg_to_retry()
parent.msg.isRetry = False
return False
return True
def on_message(self, parent):
import os
import stat
# Prepare msg delay test
if parent.msg.sumflg == 'R':
# 'R' msg will be removed by itself
return False
# Test msg delay
elapsedtime = nowflt() - timestr2flt(parent.msg.headers['pubTime'])
if elapsedtime < parent.msg_fdelay:
dbg_msg = "message not old enough, sleeping for {:.3f} seconds"
parent.logger.debug(
dbg_msg.format(elapsedtime, parent.msg_fdelay - elapsedtime))
parent.consumer.sleep_now = parent.consumer.sleep_min
parent.consumer.msg_to_retry()
parent.msg.isRetry = False
return False
# Prepare file delay test
if '/cfr/' in parent.msg.new_dir:
f = os.path.join(parent.msg.new_dir, parent.msg.new_file)
else:
f = parent.msg.relpath
if not os.path.exists(f):
parent.logger.error("did not find file {}".format(f))
return False
# Test file delay
filetime = os.stat(f)[stat.ST_MTIME]
elapsedtime = nowflt() - filetime
if elapsedtime < parent.msg_fdelay:
dbg_msg = "file not old enough, sleeping for {:.3f} seconds"
parent.logger.debug(
dbg_msg.format(elapsedtime, parent.msg_fdelay - elapsedtime))
parent.consumer.sleep_now = parent.consumer.sleep_min
parent.consumer.msg_to_retry()
parent.msg.isRetry = False
return False
return True
def on_message(self, parent):
import calendar
from sarra.sr_util import timestr2flt
msg = parent.msg
msgtime = timestr2flt(msg.pubtime)
now = time.time()
lag = now - msgtime
parent.logger.info("msg_delay received: %s %s%s topic=%s lag=%g %s" % \
( msg.pubtime, msg.baseurl, msg.relpath, msg.topic, msg.get_elapse(), msg.hdrstr ) )
if lag < parent.msg_delay:
parent.logger.info(
"msg_delay message not old enough, sleeping for %d seconds" %
(parent.msg_delay - lag))
time.sleep(parent.msg_delay - lag)
return True
def on_message(self, parent):
import filecmp
import os
import random
from difflib import Differ
parent.logger.info("msg_pclean_f90.py on_message")
result = True
msg_relpath = parent.msg.relpath
f20_path = msg_relpath.replace("{}/".format(self.all_fxx_dirs[1]),
self.all_fxx_dirs[0])
path_dict = self.build_path_dict(self.all_fxx_dirs[2:], msg_relpath)
ext = self.get_extension(msg_relpath)
for fxx_dir, path in path_dict.items():
# f90 test
if not os.path.exists(path):
# propagation check to all path except f20 which is the origin
err_msg = "file not in folder {} with {:.3f}s elapsed"
lag = nowflt() - timestr2flt(parent.msg.headers['pubTime'])
parent.logger.error(err_msg.format(fxx_dir, lag))
parent.logger.debug("file missing={}".format(path))
result = False
break
elif ext not in self.test_extension_list and not filecmp.cmp(
f20_path, path):
# file differ check: f20 against others
parent.logger.warning(
"skipping, file differs from f20 file: {}".format(path))
with open(f20_path, 'r', encoding='iso-8859-1') as f:
f20_lines = f.readlines()
with open(path, 'r', encoding='iso-8859-1') as f:
f_lines = f.readlines()
diff = Differ().compare(f20_lines, f_lines)
diff = [d for d in diff
if d[0] != ' '] # Diffs without context
parent.logger.debug("diffs found:\n{}".format("".join(diff)))
if ext not in self.test_extension_list:
# prepare next f90 test
test_extension = random.choice(
self.test_extension_list
) # pick one test identified by file extension
src = msg_relpath # src file is in f30 dir
dest = "{}{}".format(
src, test_extension
) # format input file for extension test (next f90)
try:
if test_extension == '.slink':
os.symlink(src, dest)
elif test_extension == '.hlink':
os.link(src, dest)
elif test_extension == '.moved':
os.rename(src, dest)
else:
parent.logger.error(
"test '{}' is not supported".format(test_extension))
except FileNotFoundError as err:
# src is not there
parent.logger.error("test failed: {}".format(err))
parent.logger.debug("Exception details:", exc_info=True)
result = False
except FileExistsError as err:
# dest is already there
parent.logger.error(
'skipping, found a moving target {}'.format(err))
parent.logger.debug("Exception details:", exc_info=True)
result = False
if 'toolong' in parent.msg.headers:
# cleanup
del parent.msg.headers['toolong']
return result
