def connect(self): # ============= # create message if needed # ============= self.msg = sr_message(self) # ============= # consumer queue_name : let consumer takes care of it # ============= self.consumer = sr_consumer(self) # ============= # publisher... (publish back to consumer) # ============= self.publisher = self.consumer.publish_back() # ============= # setup message publisher # ============= self.msg.publisher = self.consumer.publisher self.msg.post_exchange_split = self.post_exchange_split
def connect(self): # ============= # create message # ============= self.msg = sr_message(self) # ============= # consumer # ============= self.consumer = sr_consumer(self) self.msg.report_publisher = self.consumer.publish_back() self.msg.report_exchange = self.report_exchange self.msg.user = self.details.url.username self.msg.host = self.details.url.scheme + '://' + self.details.url.hostname self.msg.post_exchange_split = self.post_exchange_split # ============= # poster # ============= if self.post_broker : self.poster = sr_poster(self) self.msg.publisher = self.poster.publisher self.msg.pub_exchange = self.post_exchange self.msg.user = self.post_broker.username self.connected = True
def run(self): self.logger.info("sr_report run") parent = self self.consumer = sr_consumer(parent) # # loop on all message # while True : try : # is it sleeping ? if not self.has_vip() : self.logger.debug("sr_report does not have vip=%s, is sleeping", self.vip) time.sleep(5) continue else: self.logger.debug("sr_report is active on vip=%s", self.vip) ok, self.msg = self.consumer.consume() if not ok : continue ok = self.__on_message__() except : (stype, svalue, tb) = sys.exc_info() self.logger.error("Type: %s, Value: %s, ..." % (stype, svalue))
def self_test(): failed = False logger = test_logger() opt1 = 'accept .*bulletins.*' opt2 = 'reject .*' #setup consumer to catch first post cfg = sr_config() cfg.defaults() cfg.load_sums() cfg.logger = logger cfg.debug = True cfg.broker = urllib.parse.urlparse( "amqps://*****:*****@dd.weather.gc.ca/") cfg.prefetch = 10 cfg.bindings = [('xpublic', 'v02.post.#')] cfg.durable = False cfg.expire = 300 * 1000 # 5 mins = 300 secs cfg.message_ttl = 300 * 1000 # 5 mins = 300 secs cfg.user_cache_dir = os.getcwd() cfg.config_name = "test" cfg.queue_name = None cfg.retry_path = '/tmp/retry' cfg.option(opt1.split()) cfg.option(opt2.split()) consumer = sr_consumer(cfg) # loop 10000 times to try to catch a bulletin i = 0 while True: ok, msg = consumer.consume() if ok: break i = i + 1 if i == 10000: msg = None break os.unlink(consumer.queuepath) consumer.cleanup() if msg == None: print("test 01: sr_consumer TEST Failed no message") failed = True elif not 'bulletins' in msg.notice: print("test 02: sr_consumer TEST Failed not a bulletin") failed = True if not failed: print("sr_consumer.py TEST PASSED") else: print("sr_consumer.py TEST FAILED") sys.exit(1)
def connect(self): # ============= # create message if needed # ============= self.msg = sr_message(self) # ============= # consumer queue_name : let consumer takes care of it # ============= self.consumer = sr_consumer(self) # ============= # publisher... (publish back to consumer) # ============= if self.reportback : self.publisher = self.consumer.publish_back() self.msg.report_publisher = self.publisher self.msg.report_exchange = 'xs_' + self.broker.username self.logger.info("reportback to %s@%s, exchange: %s" % ( self.broker.username, self.broker.hostname, self.msg.report_exchange ) ) else: self.logger.warning("reportback suppressed")
def connect(self): # ============= # create message # ============= self.msg = sr_message(self) # ============= # consumer queue_name : let consumer takes care of it # ============= self.consumer = sr_consumer(self) # ============= # publisher... on remote cluster # ============= self.hc = HostConnect(logger=self.logger) self.hc.set_url(self.cluster_broker) self.hc.connect() self.publisher = Publisher(self.hc) self.publisher.build() # ============= # setup message publisher # ============= self.msg.publisher = self.publisher self.msg.post_exchange_split = self.post_exchange_split
def connect(self): # ============= # create message if needed # ============= self.msg = sr_message(self) # ============= # consumer queue_name : let consumer takes care of it # ============= self.consumer = sr_consumer(self) # ============= # publisher... (publish back to consumer) # ============= if self.reportback : self.publisher = self.consumer.publish_back() self.msg.report_publisher = self.publisher self.msg.report_exchange = 'xs_' + self.broker.username self.logger.info("report_back to %s@%s, exchange: %s" % ( self.broker.username, self.broker.hostname, self.msg.report_exchange ) ) else: self.logger.warning("report_back suppressed")
def connect(self): # ============= # create message # ============= self.msg = sr_message(self) # ============= # consumer # ============= self.consumer = sr_consumer(self) self.msg.report_publisher = self.consumer.publish_back() self.msg.report_exchange = self.report_exchange self.msg.user = self.details.url.username self.msg.host = self.details.url.scheme + '://' + self.details.url.hostname self.msg.post_exchange_split = self.post_exchange_split # ============= # poster # ============= if self.post_broker: self.poster = sr_poster(self) self.msg.publisher = self.poster.publisher self.msg.pub_exchange = self.post_exchange self.msg.user = self.post_broker.username self.connected = True
def connect(self): # ============= # create message # ============= self.msg = sr_message(self) # ============= # consumer queue_name : let consumer takes care of it # ============= self.consumer = sr_consumer(self) # ============= # publisher... on remote cluster # ============= self.hc = HostConnect( logger = self.logger ) self.hc.set_url( self.cluster_broker ) self.hc.connect() self.publisher = Publisher(self.hc) self.publisher.build() # ============= # setup message publisher # ============= self.msg.publisher = self.publisher self.msg.post_exchange_split = self.post_exchange_split
def connect(self): # ============= # create message # ============= self.msg = sr_message(self) # ============= # consumer # ============= self.consumer = sr_consumer(self) self.msg.report_publisher = self.consumer.publish_back() self.msg.report_exchange = self.report_exchange self.msg.user = self.broker.username self.logger.info( "reading from to %s@%s, exchange: %s" % (self.broker.username, self.broker.hostname, self.msg.exchange)) self.logger.info("report back to %s@%s, exchange: %s" % (self.broker.username, self.broker.hostname, self.msg.report_exchange)) # ============= # poster if post_broker different from broker # ============= self.poster = None if self.post_broker.geturl() != self.broker.geturl(): self.poster = sr_poster(self) self.msg.publisher = self.poster.publisher # ============= # publisher if post_broker is same as broker # ============= else: self.publisher = Publisher(self.consumer.hc) self.publisher.build() self.msg.publisher = self.publisher self.logger.info("Output AMQP broker(%s) user(%s) vhost(%s)" % \ (self.broker.hostname,self.broker.username,self.broker.path) ) self.msg.pub_exchange = self.post_exchange self.msg.post_exchange_split = self.post_exchange_split self.logger.info("Output AMQP exchange(%s)" % self.msg.pub_exchange)
def connect(self): # ============= # create message # ============= self.msg = sr_message(self) # ============= # consumer # ============= self.consumer = sr_consumer(self) self.msg.report_publisher = self.consumer.publish_back() self.msg.report_exchange = self.report_exchange self.msg.user = self.broker.username self.logger.info("reading from to %s@%s, exchange: %s" % ( self.broker.username, self.broker.hostname, self.msg.exchange ) ) self.logger.info("report back to %s@%s, exchange: %s" % ( self.broker.username, self.broker.hostname, self.msg.report_exchange ) ) # ============= # poster if post_broker different from broker # ============= self.poster = None if self.post_broker.geturl() != self.broker.geturl() : self.poster = sr_poster(self) self.msg.publisher = self.poster.publisher # ============= # publisher if post_broker is same as broker # ============= else : self.publisher = Publisher(self.consumer.hc) self.publisher.build() self.msg.publisher = self.publisher self.logger.info("Output AMQP broker(%s) user(%s) vhost(%s)" % \ (self.broker.hostname,self.broker.username,self.broker.path) ) self.msg.pub_exchange = self.post_exchange self.msg.post_exchange_split = self.post_exchange_split self.logger.info("Output AMQP exchange(%s)" % self.msg.pub_exchange )
def connect(self): # ============= # create message if needed # ============= self.msg = sr_message(self) # ============= # consumer # ============= self.consumer = sr_consumer(self) if self.reportback: self.msg.report_publisher = self.consumer.publish_back() self.msg.report_exchange = self.report_exchange self.logger.info("report_back to %s@%s, exchange: %s" % (self.broker.username, self.broker.hostname, self.msg.report_exchange)) else: self.logger.info("report_back suppressed") # ============= # publisher # ============= # publisher host self.hc_pst = HostConnect(logger=self.logger) self.hc_pst.set_url(self.post_broker) self.hc_pst.connect() # publisher self.publisher = Publisher(self.hc_pst) self.publisher.build() self.msg.publisher = self.publisher self.msg.post_exchange_split = self.post_exchange_split
def connect(self): # ============= # create message if needed # ============= self.msg = sr_message(self) # ============= # consumer # ============= self.consumer = sr_consumer(self) if self.reportback : self.msg.report_publisher = self.consumer.publish_back() self.msg.report_exchange = self.report_exchange self.logger.info("reportback to %s@%s, exchange: %s" % ( self.broker.username, self.broker.hostname, self.msg.report_exchange ) ) else: self.logger.info( "reportback suppressed" ) # ============= # publisher # ============= # publisher host self.hc_pst = HostConnect( logger = self.logger ) self.hc_pst.set_url( self.post_broker ) self.hc_pst.connect() # publisher self.publisher = Publisher(self.hc_pst) self.publisher.build() self.msg.publisher = self.publisher self.msg.post_exchange_split = self.post_exchange_split
def run(self): self.logger.info("sr_report run") parent = self self.consumer = sr_consumer(parent) # # loop on all message # while True : try : ok, self.msg = self.consumer.consume() if not ok : continue ok = self.__on_message__() except : (stype, svalue, tb) = sys.exc_info() self.logger.error("Type: %s, Value: %s, ..." % (stype, svalue))
def self_test(): logger = test_logger() # config setup cfg = sr_config() cfg.defaults() cfg.general() cfg.debug = True cfg.logger = logger cfg.kbytes_ps = 10.0 #setup consumer to catch first post #dd.weather has strange permissions, so queue declare fails. 'unknown method' #ok, cfg.broker = cfg.validate_urlstr("amqp://[email protected]/") ok, cfg.broker = cfg.validate_urlstr("amqp://tsub@localhost/") cfg.bindings = [ ( 'xpublic', 'v02.post.#') ] cfg.user_cache_dir = os.getcwd() cfg.config_name = "test" cfg.queue_name = None cfg.reset = True consumer = sr_consumer(cfg) i = 0 while True : ok, msg = consumer.consume() if ok: break cfg.msg = msg cfg.msg.sumalgo = None cfg.msg.local_file = "toto" cfg.msg.local_offset = 0 try: tr = http_transport() cfg.lock = None tr.download(cfg) logger.debug("checksum = %s" % cfg.msg.onfly_checksum) cfg.lock = '.' tr.download(cfg) logger.debug("checksum = %s" % cfg.msg.onfly_checksum) cfg.lock = '.tmp' tr.download(cfg) logger.debug("checksum = %s" % cfg.msg.onfly_checksum) cfg.msg.sumalgo = cfg.sumalgo tr.download(cfg) logger.debug("checksum = %s" % cfg.msg.onfly_checksum) logger.debug("checksum = %s" % cfg.msg.checksum) except: logger.error("sr_http TEST FAILED") (stype, svalue, tb) = sys.exc_info() logger.error('Unexpected error Type: %s, Value: %s' % (stype, svalue)) sys.exit(2) os.unlink(consumer.queuepath) consumer.close() print("sr_http TEST PASSED") sys.exit(0)
def self_test(): failed = False logger = test_logger() logger.info("sr_http: BEGIN TEST\n") opt1 = 'accept .*' logger.info("SETUP 0: get 1 message from dd.weather") #setup consumer to catch first post cfg = sr_config() cfg.configure() cfg.logger = logger cfg.use_pika = False cfg.broker = urllib.parse.urlparse( "amqps://*****:*****@dd.weather.gc.ca/") cfg.prefetch = 10 cfg.bindings = [('xpublic', 'v02.post.#')] cfg.durable = False cfg.expire = 60 * 1000 # 60 secs cfg.message_ttl = 10 * 1000 # 10 secs cfg.user_cache_dir = os.getcwd() cfg.config_name = "test" cfg.queue_name = None cfg.retry_path = '/tmp/retry' cfg.option(opt1.split()) logger.info = logger.silence consumer = sr_consumer(cfg) i = 0 while True: ok, msg = consumer.consume() if ok: break logger.info = print logger.info("SETUP 0: OK message received\n") cfg.set_sumalgo('d') cfg.msg = msg msg.sumalgo = cfg.sumalgo msg.new_dir = "." msg.new_file = "toto" cfg.msg.local_offset = 0 tr = http_transport() logger.info("TEST 01: download file with exact name") cfg.inflight = None tr.download(cfg) try: os.unlink("./toto") logger.info("TEST 01: OK") except: logger.error("TEST 01: FAILED, file not found") failed = True logger.info("TEST 02: download file lock is .filename") cfg.inflight = '.' tr.download(cfg) try: os.unlink("./toto") logger.info("TEST 02: OK") except: logger.error("TEST 02: FAILED, file not found") failed = True logger.info("TEST 03: download file lock is filename.tmp") cfg.inflight = '.tmp' tr.download(cfg) try: os.unlink("./toto") logger.info("TEST 03: OK") except: logger.error("TEST 03: FAILED, file not found") failed = True logger.info("TEST 04: inserting a part in a local file") tr.download(cfg) fp = open("titi", "wb") fp.write(b"01234567890") fp.close() fp = open("toto", "rb") data = fp.read() fp.close() cfg.msg.partflg = 'i' cfg.msg.offset = 3 cfg.msg.length = 5 cfg.msg.local_offset = 1 cfg.msg.new_file = "titi" tr.download(cfg) fp = open("titi", "rb") data2 = fp.read() fp.close() b = cfg.msg.offset e = cfg.msg.offset + cfg.msg.length - 1 b2 = cfg.msg.local_offset e2 = cfg.msg.local_offset + cfg.msg.length - 1 if data[b:e] == data2[b2:e2]: logger.info("TEST 04: OK") else: logger.info("TEST 04: failed, inserted part incorrect") failed = True try: os.unlink("titi") except: pass try: os.unlink("toto") except: pass try: os.unlink(consumer.queuepath) except: pass logger.info = logger.silence consumer.cleanup() consumer.close() logger.info = print logger.info("") if not failed: logger.info("sr_http: TEST PASSED") else: logger.info("sr_http: TEST FAILED") sys.exit(1)
def self_test(): logger = test_logger() # config setup cfg = sr_config() cfg.defaults() cfg.general() cfg.debug = True cfg.logger = logger cfg.kbytes_ps = 10.0 #setup consumer to catch first post #dd.weather has strange permissions, so queue declare fails. 'unknown method' #ok, cfg.broker = cfg.validate_urlstr("amqp://[email protected]/") ok, cfg.broker = cfg.validate_urlstr("amqp://tsub@localhost/") cfg.bindings = [('xpublic', 'v02.post.#')] cfg.user_cache_dir = os.getcwd() cfg.config_name = "test" cfg.queue_name = None cfg.reset = True consumer = sr_consumer(cfg) i = 0 while True: ok, msg = consumer.consume() if ok: break cfg.msg = msg cfg.msg.sumalgo = None cfg.msg.local_file = "toto" cfg.msg.local_offset = 0 try: tr = http_transport() cfg.lock = None tr.download(cfg) logger.debug("checksum = %s" % cfg.msg.onfly_checksum) cfg.lock = '.' tr.download(cfg) logger.debug("checksum = %s" % cfg.msg.onfly_checksum) cfg.lock = '.tmp' tr.download(cfg) logger.debug("checksum = %s" % cfg.msg.onfly_checksum) cfg.msg.sumalgo = cfg.sumalgo tr.download(cfg) logger.debug("checksum = %s" % cfg.msg.onfly_checksum) logger.debug("checksum = %s" % cfg.msg.checksum) except: logger.error("sr_http TEST FAILED") (stype, svalue, tb) = sys.exc_info() logger.error('Unexpected error Type: %s, Value: %s' % (stype, svalue)) sys.exit(2) os.unlink(consumer.queuepath) consumer.close() print("sr_http TEST PASSED") sys.exit(0)