def perform(self, parent): logger = parent.logger msg = parent.msg import calendar from sarra.sr_util import timestr2flt then = timestr2flt(msg.pubtime) now = time.time() lag = now - then if msg.onfly_checksum != msg.checksum: msg = "checksum differ - %s - %s msg %s" % ( msg.new_file, parent.onfly_checksum, msg.checksum) if lag > parent.part_check_lag_threshold: logger.warning( "part_check might just be referring to an older version of file, but " + msg) return True else: logger.error("part_check rejecting " + msg) return False logger.info("part_check Checksum matched for : %s" % msg.new_file) return True
def on_message(self, parent): logger = parent.logger msg = parent.msg if msg.isRetry: return True import calendar import humanize import datetime import sys if (parent.msg_total_msgcount == 0): logger.info( "msg_total: 0 messages received: 0 msg/s, 0.0 bytes/s, lag: 0.0 s (RESET)" ) msgtime = timestr2flt(msg.pubtime) now = nowflt() parent.msg_total_msgcount = parent.msg_total_msgcount + 1 lag = now - msgtime parent.msg_total_lag = parent.msg_total_lag + lag # guess the size of the message payload, ignoring overheads. parent.msg_total_bytecount += (len(parent.msg.exchange) + len(parent.msg.topic) + len(parent.msg.notice) + len(parent.msg.hdrstr)) #not time to report yet. if parent.msg_total_interval > now - parent.msg_total_last: return True logger.info( "msg_total: %3d messages received: %5.2g msg/s, %s bytes/s, lag: %4.2g s" % (parent.msg_total_msgcount, parent.msg_total_msgcount / (now - parent.msg_total_start), humanize.naturalsize( parent.msg_total_bytecount / (now - parent.msg_total_start), binary=True, gnu=True), parent.msg_total_lag / parent.msg_total_msgcount)) # Set the maximum age, in seconds, of a message to retrieve. if lag > parent.msg_total_maxlag: logger.warn("total: Excessive lag! Messages posted %s " % humanize.naturaltime(datetime.timedelta(seconds=lag))) parent.msg_total_last = now if (parent.msg_total_count > 0) and (parent.msg_total_msgcount >= parent.msg_total_count): os._exit(0) return True
def perform(self, parent): import time if not 'mtime' in parent.msg.headers.keys(): return True now = nowflt() mtime = timestr2flt(parent.msg.headers['mtime']) age = now - mtime parent.logger.info("file_age %g seconds for %s" % (age, parent.msg.new_file)) return True
def on_message(self, parent): logger = parent.logger msg = parent.msg import calendar then = timestr2flt(msg.pubtime) now = nowflt() logger.info("print_lag, posted: %s, lag: %g sec. to deliver: %s, " % (msg.pubtime, (now - then), msg.new_file)) return True
def on_file(self, parent): logger = parent.logger msg = parent.msg import calendar import humanize import datetime from sarra.sr_util import timestr2flt if (parent.file_total_bytecount == 0): logger.info( "file_total: 0 files received: 0 msg/s, 0.0 bytes/s, lag: 0.0 s (RESET)" ) msgtime = timestr2flt(msg.pubtime) now = time.time() parent.file_total_msgcount = parent.file_total_msgcount + 1 lag = now - msgtime parent.file_total_lag = parent.file_total_lag + lag (method, psize, ptot, prem, pno) = msg.partstr.split(',') parent.file_total_bytecount = parent.file_total_bytecount + int(psize) #not time to report yet. if parent.file_total_interval > now - parent.file_total_last: return True logger.info( "file_total: %3d files received: %5.2g msg/s, %s bytes/s, lag: %4.2g s" % (parent.file_total_msgcount, parent.file_total_msgcount / (now - parent.file_total_start), humanize.naturalsize(parent.file_total_bytecount / (now - parent.file_total_start), binary=True, gnu=True), parent.file_total_lag / parent.file_total_msgcount)) # Set the maximum age, in seconds, of a message to retrieve. if lag > parent.file_total_maxlag: logger.warn( "total: Excessive lag! downloading too slowly/late %s behind" % humanize.naturaltime(datetime.timedelta(seconds=lag))) parent.file_total_last = now return True
def on_message(self,parent): logger = parent.logger msg = parent.msg if msg.isRetry : return True import calendar import humanize import datetime from sarra.sr_util import timestr2flt if (parent.msg_total_msgcount == 0): logger.info("msg_total: 0 messages received: 0 msg/s, 0.0 bytes/s, lag: 0.0 s (RESET)" ) msgtime=timestr2flt(msg.pubtime) now=nowflt() parent.msg_total_msgcount = parent.msg_total_msgcount + 1 lag=now-msgtime parent.msg_total_lag = parent.msg_total_lag + lag # message with sum 'R' and 'L' have no partstr if parent.msg.partstr : (method,psize,ptot,prem,pno) = msg.partstr.split(',') parent.msg_total_bytecount = parent.msg_total_bytecount + int(psize) #not time to report yet. if parent.msg_total_interval > now-parent.msg_total_last : return True logger.info("msg_total: %3d messages received: %5.2g msg/s, %s bytes/s, lag: %4.2g s" % ( parent.msg_total_msgcount, parent.msg_total_msgcount/(now-parent.msg_total_start), humanize.naturalsize(parent.msg_total_bytecount/(now-parent.msg_total_start),binary=True,gnu=True), parent.msg_total_lag/parent.msg_total_msgcount )) # Set the maximum age, in seconds, of a message to retrieve. if lag > parent.msg_total_maxlag : logger.warn("total: Excessive lag! Messages posted %s " % humanize.naturaltime(datetime.timedelta(seconds=lag))) parent.msg_total_last = now return True
def on_message(self,parent): logger = parent.logger msg = parent.msg import calendar then=timestr2flt(msg.pubtime) now=nowflt() # Set the maximum age, in seconds, of a message to retrieve. lag=now-then if lag > int(parent.msg_skip_threshold) : logger.info("msg_skip_old, Excessive lag: %g sec. Skipping download of: %s, " % (lag, msg.new_file)) return False return True
def on_message(self, parent): logger = parent.logger msg = parent.msg import calendar import humanize import datetime msgtime = timestr2flt(msg.pubtime) now = nowflt() parent.msg_speedo_msgcount = parent.msg_speedo_msgcount + 1 (method, psize, ptot, prem, pno) = msg.partstr.split(',') parent.msg_speedo_bytecount = parent.msg_speedo_bytecount + int(psize) #not time to report yet. if parent.msg_speedo_interval > now - parent.msg_speedo_last: return True lag = now - msgtime logger.info( "speedo: %3d messages received: %5.2g msg/s, %s bytes/s, lag: %4.2g s" % (parent.msg_speedo_msgcount, parent.msg_speedo_msgcount / (now - parent.msg_speedo_last), humanize.naturalsize(parent.msg_speedo_bytecount / (now - parent.msg_speedo_last), binary=True, gnu=True), lag)) # Set the maximum age, in seconds, of a message to retrieve. if lag > parent.msg_speedo_maxlag: logger.warn("speedo: Excessive lag! Messages posted %s " % humanize.naturaltime(datetime.timedelta(seconds=lag))) parent.msg_speedo_last = now parent.msg_speedo_msgcount = 0 parent.msg_speedo_bytecount = 0 return True
def perform(self, parent): logger = parent.logger msg = parent.msg import calendar import humanize import datetime from sarra.sr_util import timestr2flt if parent.post_total_msgcount == 0: logger.info( "post_total: 0 messages posted: 0 msg/s, 0.0 bytes/s, lag: 0.0 s (RESET)" ) msgtime = timestr2flt(msg.pubtime) now = time.time() parent.post_total_msgcount = parent.post_total_msgcount + 1 lag = now - msgtime parent.post_total_lag = parent.post_total_lag + lag #(method,psize,ptot,prem,pno) = msg.partstr.split(',') #parent.post_total_bytecount = parent.post_total_bytecount + int(psize) #not time to report yet. if parent.post_total_interval > now - parent.post_total_last: return True logger.info( "post_total: %3d messages posted: %5.2g msg/s, lag: %4.2g s" % (parent.post_total_msgcount, parent.post_total_msgcount / (now - parent.post_total_start), parent.post_total_lag / parent.post_total_msgcount)) # Set the maximum age, in seconds, of a message to retrieve. if lag > parent.post_total_maxlag: logger.warn("total: Excessive lag! Messages posted %s " % humanize.naturaltime(datetime.timedelta(seconds=lag))) parent.post_total_last = now return True
def on_message(self,parent): if not 'delay' in parent.msg.headers: parent.msg.headers['delay'] = nowstr() # Test msg delay elapsedtime = nowflt() - timestr2flt(parent.msg.headers['pubTime']) if 0 < elapsedtime < 1: parent.logger.debug("msg_delay received msg") else: parent.logger.info("trying msg with {:.3f}s elapsed".format(elapsedtime)) if elapsedtime < parent.msg_delay: dbg_msg = "message not old enough, sleeping for {:.3f} seconds" parent.logger.debug(dbg_msg.format(elapsedtime, parent.msg_delay - elapsedtime)) parent.consumer.sleep_now = parent.consumer.sleep_min parent.consumer.msg_to_retry() parent.msg.isRetry = False return False return True
def on_message(self, parent): import os import stat # Prepare msg delay test if parent.msg.sumflg == 'R': # 'R' msg will be removed by itself return False # Test msg delay elapsedtime = nowflt() - timestr2flt(parent.msg.headers['pubTime']) if elapsedtime < parent.msg_fdelay: dbg_msg = "message not old enough, sleeping for {:.3f} seconds" parent.logger.debug( dbg_msg.format(elapsedtime, parent.msg_fdelay - elapsedtime)) parent.consumer.sleep_now = parent.consumer.sleep_min parent.consumer.msg_to_retry() parent.msg.isRetry = False return False # Prepare file delay test if '/cfr/' in parent.msg.new_dir: f = os.path.join(parent.msg.new_dir, parent.msg.new_file) else: f = parent.msg.relpath if not os.path.exists(f): parent.logger.error("did not find file {}".format(f)) return False # Test file delay filetime = os.stat(f)[stat.ST_MTIME] elapsedtime = nowflt() - filetime if elapsedtime < parent.msg_fdelay: dbg_msg = "file not old enough, sleeping for {:.3f} seconds" parent.logger.debug( dbg_msg.format(elapsedtime, parent.msg_fdelay - elapsedtime)) parent.consumer.sleep_now = parent.consumer.sleep_min parent.consumer.msg_to_retry() parent.msg.isRetry = False return False return True
def on_message(self, parent): import calendar from sarra.sr_util import timestr2flt msg = parent.msg msgtime = timestr2flt(msg.pubtime) now = time.time() lag = now - msgtime parent.logger.info("msg_delay received: %s %s%s topic=%s lag=%g %s" % \ ( msg.pubtime, msg.baseurl, msg.relpath, msg.topic, msg.get_elapse(), msg.hdrstr ) ) if lag < parent.msg_delay: parent.logger.info( "msg_delay message not old enough, sleeping for %d seconds" % (parent.msg_delay - lag)) time.sleep(parent.msg_delay - lag) return True
def on_message(self, parent): import filecmp import os import random from difflib import Differ parent.logger.info("msg_pclean_f90.py on_message") result = True msg_relpath = parent.msg.relpath f20_path = msg_relpath.replace("{}/".format(self.all_fxx_dirs[1]), self.all_fxx_dirs[0]) path_dict = self.build_path_dict(self.all_fxx_dirs[2:], msg_relpath) ext = self.get_extension(msg_relpath) for fxx_dir, path in path_dict.items(): # f90 test if not os.path.exists(path): # propagation check to all path except f20 which is the origin err_msg = "file not in folder {} with {:.3f}s elapsed" lag = nowflt() - timestr2flt(parent.msg.headers['pubTime']) parent.logger.error(err_msg.format(fxx_dir, lag)) parent.logger.debug("file missing={}".format(path)) result = False break elif ext not in self.test_extension_list and not filecmp.cmp( f20_path, path): # file differ check: f20 against others parent.logger.warning( "skipping, file differs from f20 file: {}".format(path)) with open(f20_path, 'r', encoding='iso-8859-1') as f: f20_lines = f.readlines() with open(path, 'r', encoding='iso-8859-1') as f: f_lines = f.readlines() diff = Differ().compare(f20_lines, f_lines) diff = [d for d in diff if d[0] != ' '] # Diffs without context parent.logger.debug("diffs found:\n{}".format("".join(diff))) if ext not in self.test_extension_list: # prepare next f90 test test_extension = random.choice( self.test_extension_list ) # pick one test identified by file extension src = msg_relpath # src file is in f30 dir dest = "{}{}".format( src, test_extension ) # format input file for extension test (next f90) try: if test_extension == '.slink': os.symlink(src, dest) elif test_extension == '.hlink': os.link(src, dest) elif test_extension == '.moved': os.rename(src, dest) else: parent.logger.error( "test '{}' is not supported".format(test_extension)) except FileNotFoundError as err: # src is not there parent.logger.error("test failed: {}".format(err)) parent.logger.debug("Exception details:", exc_info=True) result = False except FileExistsError as err: # dest is already there parent.logger.error( 'skipping, found a moving target {}'.format(err)) parent.logger.debug("Exception details:", exc_info=True) result = False if 'toolong' in parent.msg.headers: # cleanup del parent.msg.headers['toolong'] return result