def connect(self):
# =============
# create message if needed
# =============
self.msg = sr_message(self)
# =============
# consumer queue_name : let consumer takes care of it
# =============
self.consumer = sr_consumer(self)
# =============
# publisher... (publish back to consumer)
# =============
self.publisher = self.consumer.publish_back()
# =============
# setup message publisher
# =============
self.msg.publisher = self.consumer.publisher
self.msg.post_exchange_split = self.post_exchange_split
def connect(self):
# =============
# create message
# =============
self.msg = sr_message(self)
# =============
# consumer
# =============
self.consumer = sr_consumer(self)
self.msg.report_publisher = self.consumer.publish_back()
self.msg.report_exchange = self.report_exchange
self.msg.user = self.details.url.username
self.msg.host = self.details.url.scheme + '://' + self.details.url.hostname
self.msg.post_exchange_split = self.post_exchange_split
# =============
# poster
# =============
if self.post_broker :
self.poster = sr_poster(self)
self.msg.publisher = self.poster.publisher
self.msg.pub_exchange = self.post_exchange
self.msg.user = self.post_broker.username
self.connected = True
def run(self):
self.logger.info("sr_report run")
parent = self
self.consumer = sr_consumer(parent)
#
# loop on all message
#
while True :
try :
# is it sleeping ?
if not self.has_vip() :
self.logger.debug("sr_report does not have vip=%s, is sleeping", self.vip)
time.sleep(5)
continue
else:
self.logger.debug("sr_report is active on vip=%s", self.vip)
ok, self.msg = self.consumer.consume()
if not ok : continue
ok = self.__on_message__()
except :
(stype, svalue, tb) = sys.exc_info()
self.logger.error("Type: %s, Value: %s, ..." % (stype, svalue))
def self_test():
failed = False
logger = test_logger()
opt1 = 'accept .*bulletins.*'
opt2 = 'reject .*'
#setup consumer to catch first post
cfg = sr_config()
cfg.defaults()
cfg.load_sums()
cfg.logger = logger
cfg.debug = True
cfg.broker = urllib.parse.urlparse(
"amqps://*****:*****@dd.weather.gc.ca/")
cfg.prefetch = 10
cfg.bindings = [('xpublic', 'v02.post.#')]
cfg.durable = False
cfg.expire = 300 * 1000 # 5 mins = 300 secs
cfg.message_ttl = 300 * 1000 # 5 mins = 300 secs
cfg.user_cache_dir = os.getcwd()
cfg.config_name = "test"
cfg.queue_name = None
cfg.retry_path = '/tmp/retry'
cfg.option(opt1.split())
cfg.option(opt2.split())
consumer = sr_consumer(cfg)
# loop 10000 times to try to catch a bulletin
i = 0
while True:
ok, msg = consumer.consume()
if ok: break
i = i + 1
if i == 10000:
msg = None
break
os.unlink(consumer.queuepath)
consumer.cleanup()
if msg == None:
print("test 01: sr_consumer TEST Failed no message")
failed = True
elif not 'bulletins' in msg.notice:
print("test 02: sr_consumer TEST Failed not a bulletin")
failed = True
if not failed:
print("sr_consumer.py TEST PASSED")
else:
print("sr_consumer.py TEST FAILED")
sys.exit(1)
def connect(self):
# =============
# create message if needed
# =============
self.msg = sr_message(self)
# =============
# consumer queue_name : let consumer takes care of it
# =============
self.consumer = sr_consumer(self)
# =============
# publisher... (publish back to consumer)
# =============
if self.reportback :
self.publisher = self.consumer.publish_back()
self.msg.report_publisher = self.publisher
self.msg.report_exchange = 'xs_' + self.broker.username
self.logger.info("reportback to %s@%s, exchange: %s" %
( self.broker.username, self.broker.hostname, self.msg.report_exchange ) )
else:
self.logger.warning("reportback suppressed")
def connect(self):
# =============
# create message if needed
# =============
self.msg = sr_message(self)
# =============
# consumer queue_name : let consumer takes care of it
# =============
self.consumer = sr_consumer(self)
# =============
# publisher... (publish back to consumer)
# =============
self.publisher = self.consumer.publish_back()
# =============
# setup message publisher
# =============
self.msg.publisher = self.consumer.publisher
self.msg.post_exchange_split = self.post_exchange_split
def connect(self):
# =============
# create message
# =============
self.msg = sr_message(self)
# =============
# consumer queue_name : let consumer takes care of it
# =============
self.consumer = sr_consumer(self)
# =============
# publisher... on remote cluster
# =============
self.hc = HostConnect(logger=self.logger)
self.hc.set_url(self.cluster_broker)
self.hc.connect()
self.publisher = Publisher(self.hc)
self.publisher.build()
# =============
# setup message publisher
# =============
self.msg.publisher = self.publisher
self.msg.post_exchange_split = self.post_exchange_split
def connect(self):
# =============
# create message if needed
# =============
self.msg = sr_message(self)
# =============
# consumer queue_name : let consumer takes care of it
# =============
self.consumer = sr_consumer(self)
# =============
# publisher... (publish back to consumer)
# =============
if self.reportback :
self.publisher = self.consumer.publish_back()
self.msg.report_publisher = self.publisher
self.msg.report_exchange = 'xs_' + self.broker.username
self.logger.info("report_back to %s@%s, exchange: %s" %
( self.broker.username, self.broker.hostname, self.msg.report_exchange ) )
else:
self.logger.warning("report_back suppressed")
def connect(self):
# =============
# create message
# =============
self.msg = sr_message(self)
# =============
# consumer
# =============
self.consumer = sr_consumer(self)
self.msg.report_publisher = self.consumer.publish_back()
self.msg.report_exchange = self.report_exchange
self.msg.user = self.details.url.username
self.msg.host = self.details.url.scheme + '://' + self.details.url.hostname
self.msg.post_exchange_split = self.post_exchange_split
# =============
# poster
# =============
if self.post_broker:
self.poster = sr_poster(self)
self.msg.publisher = self.poster.publisher
self.msg.pub_exchange = self.post_exchange
self.msg.user = self.post_broker.username
self.connected = True
def connect(self):
# =============
# create message
# =============
self.msg = sr_message(self)
# =============
# consumer queue_name : let consumer takes care of it
# =============
self.consumer = sr_consumer(self)
# =============
# publisher... on remote cluster
# =============
self.hc = HostConnect( logger = self.logger )
self.hc.set_url( self.cluster_broker )
self.hc.connect()
self.publisher = Publisher(self.hc)
self.publisher.build()
# =============
# setup message publisher
# =============
self.msg.publisher = self.publisher
self.msg.post_exchange_split = self.post_exchange_split
def connect(self):
# =============
# create message
# =============
self.msg = sr_message(self)
# =============
# consumer
# =============
self.consumer = sr_consumer(self)
self.msg.report_publisher = self.consumer.publish_back()
self.msg.report_exchange = self.report_exchange
self.msg.user = self.broker.username
self.logger.info(
"reading from to %s@%s, exchange: %s" %
(self.broker.username, self.broker.hostname, self.msg.exchange))
self.logger.info("report back to %s@%s, exchange: %s" %
(self.broker.username, self.broker.hostname,
self.msg.report_exchange))
# =============
# poster if post_broker different from broker
# =============
self.poster = None
if self.post_broker.geturl() != self.broker.geturl():
self.poster = sr_poster(self)
self.msg.publisher = self.poster.publisher
# =============
# publisher if post_broker is same as broker
# =============
else:
self.publisher = Publisher(self.consumer.hc)
self.publisher.build()
self.msg.publisher = self.publisher
self.logger.info("Output AMQP broker(%s) user(%s) vhost(%s)" % \
(self.broker.hostname,self.broker.username,self.broker.path) )
self.msg.pub_exchange = self.post_exchange
self.msg.post_exchange_split = self.post_exchange_split
self.logger.info("Output AMQP exchange(%s)" % self.msg.pub_exchange)
def connect(self):
# =============
# create message
# =============
self.msg = sr_message(self)
# =============
# consumer
# =============
self.consumer = sr_consumer(self)
self.msg.report_publisher = self.consumer.publish_back()
self.msg.report_exchange = self.report_exchange
self.msg.user = self.broker.username
self.logger.info("reading from to %s@%s, exchange: %s" %
( self.broker.username, self.broker.hostname, self.msg.exchange ) )
self.logger.info("report back to %s@%s, exchange: %s" %
( self.broker.username, self.broker.hostname, self.msg.report_exchange ) )
# =============
# poster if post_broker different from broker
# =============
self.poster = None
if self.post_broker.geturl() != self.broker.geturl() :
self.poster = sr_poster(self)
self.msg.publisher = self.poster.publisher
# =============
# publisher if post_broker is same as broker
# =============
else :
self.publisher = Publisher(self.consumer.hc)
self.publisher.build()
self.msg.publisher = self.publisher
self.logger.info("Output AMQP broker(%s) user(%s) vhost(%s)" % \
(self.broker.hostname,self.broker.username,self.broker.path) )
self.msg.pub_exchange = self.post_exchange
self.msg.post_exchange_split = self.post_exchange_split
self.logger.info("Output AMQP exchange(%s)" % self.msg.pub_exchange )
def connect(self):
# =============
# create message if needed
# =============
self.msg = sr_message(self)
# =============
# consumer
# =============
self.consumer = sr_consumer(self)
if self.reportback:
self.msg.report_publisher = self.consumer.publish_back()
self.msg.report_exchange = self.report_exchange
self.logger.info("report_back to %s@%s, exchange: %s" %
(self.broker.username, self.broker.hostname,
self.msg.report_exchange))
else:
self.logger.info("report_back suppressed")
# =============
# publisher
# =============
# publisher host
self.hc_pst = HostConnect(logger=self.logger)
self.hc_pst.set_url(self.post_broker)
self.hc_pst.connect()
# publisher
self.publisher = Publisher(self.hc_pst)
self.publisher.build()
self.msg.publisher = self.publisher
self.msg.post_exchange_split = self.post_exchange_split
def connect(self):
# =============
# create message if needed
# =============
self.msg = sr_message(self)
# =============
# consumer
# =============
self.consumer = sr_consumer(self)
if self.reportback :
self.msg.report_publisher = self.consumer.publish_back()
self.msg.report_exchange = self.report_exchange
self.logger.info("reportback to %s@%s, exchange: %s" %
( self.broker.username, self.broker.hostname, self.msg.report_exchange ) )
else:
self.logger.info( "reportback suppressed" )
# =============
# publisher
# =============
# publisher host
self.hc_pst = HostConnect( logger = self.logger )
self.hc_pst.set_url( self.post_broker )
self.hc_pst.connect()
# publisher
self.publisher = Publisher(self.hc_pst)
self.publisher.build()
self.msg.publisher = self.publisher
self.msg.post_exchange_split = self.post_exchange_split
def run(self):
self.logger.info("sr_report run")
parent = self
self.consumer = sr_consumer(parent)
#
# loop on all message
#
while True :
try :
ok, self.msg = self.consumer.consume()
if not ok : continue
ok = self.__on_message__()
except :
(stype, svalue, tb) = sys.exc_info()
self.logger.error("Type: %s, Value: %s, ..." % (stype, svalue))
def self_test():
logger = test_logger()
# config setup
cfg = sr_config()
cfg.defaults()
cfg.general()
cfg.debug = True
cfg.logger = logger
cfg.kbytes_ps = 10.0
#setup consumer to catch first post
#dd.weather has strange permissions, so queue declare fails. 'unknown method'
#ok, cfg.broker = cfg.validate_urlstr("amqp://[email protected]/")
ok, cfg.broker = cfg.validate_urlstr("amqp://tsub@localhost/")
cfg.bindings = [ ( 'xpublic', 'v02.post.#') ]
cfg.user_cache_dir = os.getcwd()
cfg.config_name = "test"
cfg.queue_name = None
cfg.reset = True
consumer = sr_consumer(cfg)
i = 0
while True :
ok, msg = consumer.consume()
if ok: break
cfg.msg = msg
cfg.msg.sumalgo = None
cfg.msg.local_file = "toto"
cfg.msg.local_offset = 0
try:
tr = http_transport()
cfg.lock = None
tr.download(cfg)
logger.debug("checksum = %s" % cfg.msg.onfly_checksum)
cfg.lock = '.'
tr.download(cfg)
logger.debug("checksum = %s" % cfg.msg.onfly_checksum)
cfg.lock = '.tmp'
tr.download(cfg)
logger.debug("checksum = %s" % cfg.msg.onfly_checksum)
cfg.msg.sumalgo = cfg.sumalgo
tr.download(cfg)
logger.debug("checksum = %s" % cfg.msg.onfly_checksum)
logger.debug("checksum = %s" % cfg.msg.checksum)
except:
logger.error("sr_http TEST FAILED")
(stype, svalue, tb) = sys.exc_info()
logger.error('Unexpected error Type: %s, Value: %s' % (stype, svalue))
sys.exit(2)
os.unlink(consumer.queuepath)
consumer.close()
print("sr_http TEST PASSED")
sys.exit(0)
def self_test():
failed = False
logger = test_logger()
logger.info("sr_http: BEGIN TEST\n")
opt1 = 'accept .*'
logger.info("SETUP 0: get 1 message from dd.weather")
#setup consumer to catch first post
cfg = sr_config()
cfg.configure()
cfg.logger = logger
cfg.use_pika = False
cfg.broker = urllib.parse.urlparse(
"amqps://*****:*****@dd.weather.gc.ca/")
cfg.prefetch = 10
cfg.bindings = [('xpublic', 'v02.post.#')]
cfg.durable = False
cfg.expire = 60 * 1000 # 60 secs
cfg.message_ttl = 10 * 1000 # 10 secs
cfg.user_cache_dir = os.getcwd()
cfg.config_name = "test"
cfg.queue_name = None
cfg.retry_path = '/tmp/retry'
cfg.option(opt1.split())
logger.info = logger.silence
consumer = sr_consumer(cfg)
i = 0
while True:
ok, msg = consumer.consume()
if ok: break
logger.info = print
logger.info("SETUP 0: OK message received\n")
cfg.set_sumalgo('d')
cfg.msg = msg
msg.sumalgo = cfg.sumalgo
msg.new_dir = "."
msg.new_file = "toto"
cfg.msg.local_offset = 0
tr = http_transport()
logger.info("TEST 01: download file with exact name")
cfg.inflight = None
tr.download(cfg)
try:
os.unlink("./toto")
logger.info("TEST 01: OK")
except:
logger.error("TEST 01: FAILED, file not found")
failed = True
logger.info("TEST 02: download file lock is .filename")
cfg.inflight = '.'
tr.download(cfg)
try:
os.unlink("./toto")
logger.info("TEST 02: OK")
except:
logger.error("TEST 02: FAILED, file not found")
failed = True
logger.info("TEST 03: download file lock is filename.tmp")
cfg.inflight = '.tmp'
tr.download(cfg)
try:
os.unlink("./toto")
logger.info("TEST 03: OK")
except:
logger.error("TEST 03: FAILED, file not found")
failed = True
logger.info("TEST 04: inserting a part in a local file")
tr.download(cfg)
fp = open("titi", "wb")
fp.write(b"01234567890")
fp.close()
fp = open("toto", "rb")
data = fp.read()
fp.close()
cfg.msg.partflg = 'i'
cfg.msg.offset = 3
cfg.msg.length = 5
cfg.msg.local_offset = 1
cfg.msg.new_file = "titi"
tr.download(cfg)
fp = open("titi", "rb")
data2 = fp.read()
fp.close()
b = cfg.msg.offset
e = cfg.msg.offset + cfg.msg.length - 1
b2 = cfg.msg.local_offset
e2 = cfg.msg.local_offset + cfg.msg.length - 1
if data[b:e] == data2[b2:e2]:
logger.info("TEST 04: OK")
else:
logger.info("TEST 04: failed, inserted part incorrect")
failed = True
try:
os.unlink("titi")
except:
pass
try:
os.unlink("toto")
except:
pass
try:
os.unlink(consumer.queuepath)
except:
pass
logger.info = logger.silence
consumer.cleanup()
consumer.close()
logger.info = print
logger.info("")
if not failed:
logger.info("sr_http: TEST PASSED")
else:
logger.info("sr_http: TEST FAILED")
sys.exit(1)
def self_test():
logger = test_logger()
# config setup
cfg = sr_config()
cfg.defaults()
cfg.general()
cfg.debug = True
cfg.logger = logger
cfg.kbytes_ps = 10.0
#setup consumer to catch first post
#dd.weather has strange permissions, so queue declare fails. 'unknown method'
#ok, cfg.broker = cfg.validate_urlstr("amqp://[email protected]/")
ok, cfg.broker = cfg.validate_urlstr("amqp://tsub@localhost/")
cfg.bindings = [('xpublic', 'v02.post.#')]
cfg.user_cache_dir = os.getcwd()
cfg.config_name = "test"
cfg.queue_name = None
cfg.reset = True
consumer = sr_consumer(cfg)
i = 0
while True:
ok, msg = consumer.consume()
if ok: break
cfg.msg = msg
cfg.msg.sumalgo = None
cfg.msg.local_file = "toto"
cfg.msg.local_offset = 0
try:
tr = http_transport()
cfg.lock = None
tr.download(cfg)
logger.debug("checksum = %s" % cfg.msg.onfly_checksum)
cfg.lock = '.'
tr.download(cfg)
logger.debug("checksum = %s" % cfg.msg.onfly_checksum)
cfg.lock = '.tmp'
tr.download(cfg)
logger.debug("checksum = %s" % cfg.msg.onfly_checksum)
cfg.msg.sumalgo = cfg.sumalgo
tr.download(cfg)
logger.debug("checksum = %s" % cfg.msg.onfly_checksum)
logger.debug("checksum = %s" % cfg.msg.checksum)
except:
logger.error("sr_http TEST FAILED")
(stype, svalue, tb) = sys.exc_info()
logger.error('Unexpected error Type: %s, Value: %s' % (stype, svalue))
sys.exit(2)
os.unlink(consumer.queuepath)
consumer.close()
print("sr_http TEST PASSED")
sys.exit(0)
