def self_test():
logger = test_logger()
yyyy = time.strftime("%Y", time.gmtime())
opt1 = 'accept .*' + yyyy + '.*'
opt2 = 'reject .*'
#setup consumer to catch first post
cfg = sr_config()
cfg.defaults()
cfg.logger = logger
cfg.debug = True
#cfg.broker = urllib.parse.urlparse("amqp://*****:*****@ddi.cmc.ec.gc.ca/")
cfg.broker = urllib.parse.urlparse(
"amqp://*****:*****@dd.weather.gc.ca/")
cfg.prefetch = 10
cfg.bindings = [('xpublic', 'v02.post.#')]
cfg.durable = True
cfg.expire = 30
cfg.message_ttl = 30
cfg.user_cache_dir = os.getcwd()
cfg.config_name = "test"
cfg.queue_name = None
cfg.option(opt1.split())
cfg.option(opt2.split())
consumer = sr_consumer(cfg)
#FIXME setup another consumer
# from message... log ... catch log messsage
i = 0
while True:
ok, msg = consumer.consume()
if ok: break
i = i + 1
if i == 100:
msg = None
break
os.unlink(consumer.queuepath)
consumer.close()
if msg != None:
if yyyy in msg.notice:
print("sr_consumer TEST PASSED")
sys.exit(0)
print("sr_consumer TEST Failed 1 wrong message")
sys.exit(1)
print("sr_consumer TEST Failed 2 no message")
sys.exit(2)
def self_test():
logger = test_logger()
yyyy = time.strftime("%Y",time.gmtime())
opt1 = 'accept .*' + yyyy + '.*'
opt2 = 'reject .*'
#setup consumer to catch first post
cfg = sr_config()
cfg.defaults()
cfg.logger = logger
cfg.debug = True
#cfg.broker = urllib.parse.urlparse("amqp://*****:*****@ddi.cmc.ec.gc.ca/")
cfg.broker = urllib.parse.urlparse("amqp://*****:*****@dd.weather.gc.ca/")
cfg.prefetch = 10
cfg.bindings = [ ( 'xpublic', 'v02.post.#') ]
cfg.durable = True
cfg.expire = 30
cfg.message_ttl = 30
cfg.user_cache_dir = os.getcwd()
cfg.config_name = "test"
cfg.queue_name = None
cfg.option( opt1.split() )
cfg.option( opt2.split() )
consumer = sr_consumer(cfg)
#FIXME setup another consumer
# from message... log ... catch log messsage
i = 0
while True :
ok, msg = consumer.consume()
if ok: break
i = i + 1
if i == 100 :
msg = None
break
os.unlink(consumer.queuepath)
consumer.close()
if msg != None :
if yyyy in msg.notice :
print("sr_consumer TEST PASSED")
sys.exit(0)
print("sr_consumer TEST Failed 1 wrong message")
sys.exit(1)
print("sr_consumer TEST Failed 2 no message")
sys.exit(2)
def self_test():
failed = False
cfg = sr_config()
# ===================================
# TESTING
# ===================================
# Testing for 2 weeks - today in YYYYMMDD format
print('Testing todays date - 2 weeks')
test = cfg.set_dir_pattern('${YYYYMMDD-2w}')
answer = time.mktime(time.gmtime()) - 2 * 7 * 24 * 60 * 60
answer = time.strftime("%Y%m%d", time.localtime(answer))
print('test: ' + test + ' answer: ' + answer)
if (not test) or (str(test) != str(answer)):
failed = True
else:
print("Pass\n")
# Testing for 12 days - today in JJJ format
print('Testing todays date - 12 days')
test = cfg.set_dir_pattern('${JJJ-12D}')
answer = time.mktime(time.gmtime()) - 12 * 24 * 60 * 60
answer = time.strftime("%j", time.localtime(answer))
print('test: ' + test + ' answer: ' + answer)
if (not test) or (str(test) != str(answer)):
failed = True
else:
print("Pass\n")
# Testing for 60 hours - today in MM format
print('Testing todays date - 1400 hours')
test = cfg.set_dir_pattern('${MM-1400h}')
answer = time.mktime(time.gmtime()) - 1400 * 60 * 60
answer = time.strftime("%m", time.localtime(answer))
print('test: ' + test + ' answer: ' + answer)
if (not test) or (str(test) != str(answer)):
failed = True
else:
print("Pass\n")
if not failed:
print("sr_pattern_test.py TEST PASSED")
else:
print("sr_pattern_test.py TEST FAILED")
sys.exit(1)
def setUpClass(cls) -> None:
""" Setup basic config file with basic include
:return: None
"""
f = open("./bbb.inc", "w")
f.write("randomize True\n")
f.close()
f = open("./aaa.conf", "w")
f.write("include bbb.inc\n")
f.close()
# instantiation, test include and overwrite logs
cls.cfg = sr_config(config="aaa")
cls.cfg.configure()
def self_test():
logger = test_logger()
# config setup
cfg = sr_config()
cfg.defaults()
cfg.general()
#cfg.debug = True
opt1 = "destination sftp://localhost"
cfg.option(opt1.split())
cfg.logger = logger
cfg.timeout = 5.0
# 1 bytes par 5 secs
#cfg.kbytes_ps = 0.0001
cfg.kbytes_ps = 0.01
try:
sftp = sr_sftp(cfg)
sftp.connect()
sftp.mkdir("tztz")
sftp.chmod(775, "tztz")
sftp.cd("tztz")
f = open("aaa", "wb")
f.write(b"1\n")
f.write(b"2\n")
f.write(b"3\n")
f.close()
sftp.put("aaa", "bbb")
ls = sftp.ls()
logger.info("ls = %s" % ls)
sftp.chmod(775, "bbb")
ls = sftp.ls()
logger.info("ls = %s" % ls)
sftp.rename("bbb", "ccc")
ls = sftp.ls()
logger.info("ls = %s" % ls)
sftp.get("ccc", "bbb", 0, 0, 6)
f = open("bbb", "rb")
data = f.read()
f.close()
if data != b"1\n2\n3\n":
logger.error("sr_sftp1 TEST FAILED")
sys.exit(1)
os.unlink("bbb")
msg = sr_message(cfg)
msg.start_timer()
msg.topic = "v02.post.test"
msg.notice = "notice"
msg.urlcred = "sftp://localhost/"
msg.urlstr = "sftp://localhost/tztz/ccc"
msg.url = urllib.parse.urlparse(msg.urlcred + "tztz/ccc")
msg.partflg = '1'
msg.offset = 0
msg.length = 0
msg.local_file = "bbb"
msg.local_offset = 0
msg.sumalgo = None
cfg.msg = msg
cfg.batch = 5
cfg.lock = None
dldr = sftp_transport()
dldr.download(cfg)
logger.debug("checksum = %s" % msg.onfly_checksum)
dldr.download(cfg)
dldr.download(cfg)
cfg.logger.info("lock .")
cfg.lock = '.'
dldr.download(cfg)
dldr.download(cfg)
msg.sumalgo = cfg.sumalgo
dldr.download(cfg)
logger.debug("checksum = %s" % msg.onfly_checksum)
cfg.logger.info("lock .tmp")
cfg.lock = '.tmp'
dldr.download(cfg)
dldr.download(cfg)
dldr.close()
dldr.close()
dldr.close()
dldr = sftp_transport()
cfg.local_file = "bbb"
cfg.local_path = "./bbb"
cfg.remote_file = "ddd"
cfg.remote_path = "tztz/ddd"
cfg.remote_urlstr = "sftp://localhost/tztz/ddd"
cfg.remote_dir = "tztz"
cfg.chmod = 775
cfg.lock = None
dldr.send(cfg)
dldr.sftp.delete("ddd")
cfg.lock = '.'
dldr.send(cfg)
dldr.sftp.delete("ddd")
cfg.lock = '.tmp'
dldr.send(cfg)
dldr.send(cfg)
dldr.send(cfg)
dldr.send(cfg)
dldr.send(cfg)
dldr.send(cfg)
dldr.close()
dldr.close()
dldr.close()
sftp = sr_sftp(cfg)
sftp.connect()
sftp.cd("tztz")
sftp.delete("ccc")
sftp.delete("ddd")
logger.info("%s" % sftp.originalDir)
sftp.cd("")
logger.info("%s" % sftp.sftp.getcwd())
sftp.rmdir("tztz")
sftp.put("aaa", "bbb", 0, 0, 2)
sftp.put("aaa", "bbb", 2, 4, 2)
sftp.put("aaa", "bbb", 4, 2, 2)
sftp.get("bbb", "bbb", 2, 2, 2)
sftp.delete("bbb")
f = open("bbb", "rb")
data = f.read()
f.close()
if data != b"1\n3\n3\n":
logger.error("sr_sftp TEST FAILED ")
sys.exit(1)
sftp.close()
except:
(stype, svalue, tb) = sys.exc_info()
logger.error("(Type: %s, Value: %s)" % (stype, svalue))
logger.error("sr_sftp TEST FAILED")
sys.exit(2)
os.unlink('aaa')
os.unlink('bbb')
print("sr_sftp TEST PASSED")
sys.exit(0)
def self_test():
failed = False
logger = test_logger()
logger.info("sr_http: BEGIN TEST\n")
opt1 = 'accept .*'
logger.info("SETUP 0: get 1 message from dd.weather")
#setup consumer to catch first post
cfg = sr_config()
cfg.configure()
cfg.logger = logger
cfg.use_pika = False
cfg.broker = urllib.parse.urlparse(
"amqps://*****:*****@dd.weather.gc.ca/")
cfg.prefetch = 10
cfg.bindings = [('xpublic', 'v02.post.#')]
cfg.durable = False
cfg.expire = 60 * 1000 # 60 secs
cfg.message_ttl = 10 * 1000 # 10 secs
cfg.user_cache_dir = os.getcwd()
cfg.config_name = "test"
cfg.queue_name = None
cfg.retry_path = '/tmp/retry'
cfg.option(opt1.split())
logger.info = logger.silence
consumer = sr_consumer(cfg)
i = 0
while True:
ok, msg = consumer.consume()
if ok: break
logger.info = print
logger.info("SETUP 0: OK message received\n")
cfg.set_sumalgo('d')
cfg.msg = msg
msg.sumalgo = cfg.sumalgo
msg.new_dir = "."
msg.new_file = "toto"
cfg.msg.local_offset = 0
tr = http_transport()
logger.info("TEST 01: download file with exact name")
cfg.inflight = None
tr.download(cfg)
try:
os.unlink("./toto")
logger.info("TEST 01: OK")
except:
logger.error("TEST 01: FAILED, file not found")
failed = True
logger.info("TEST 02: download file lock is .filename")
cfg.inflight = '.'
tr.download(cfg)
try:
os.unlink("./toto")
logger.info("TEST 02: OK")
except:
logger.error("TEST 02: FAILED, file not found")
failed = True
logger.info("TEST 03: download file lock is filename.tmp")
cfg.inflight = '.tmp'
tr.download(cfg)
try:
os.unlink("./toto")
logger.info("TEST 03: OK")
except:
logger.error("TEST 03: FAILED, file not found")
failed = True
logger.info("TEST 04: inserting a part in a local file")
tr.download(cfg)
fp = open("titi", "wb")
fp.write(b"01234567890")
fp.close()
fp = open("toto", "rb")
data = fp.read()
fp.close()
cfg.msg.partflg = 'i'
cfg.msg.offset = 3
cfg.msg.length = 5
cfg.msg.local_offset = 1
cfg.msg.new_file = "titi"
tr.download(cfg)
fp = open("titi", "rb")
data2 = fp.read()
fp.close()
b = cfg.msg.offset
e = cfg.msg.offset + cfg.msg.length - 1
b2 = cfg.msg.local_offset
e2 = cfg.msg.local_offset + cfg.msg.length - 1
if data[b:e] == data2[b2:e2]:
logger.info("TEST 04: OK")
else:
logger.info("TEST 04: failed, inserted part incorrect")
failed = True
try:
os.unlink("titi")
except:
pass
try:
os.unlink("toto")
except:
pass
try:
os.unlink(consumer.queuepath)
except:
pass
logger.info = logger.silence
consumer.cleanup()
consumer.close()
logger.info = print
logger.info("")
if not failed:
logger.info("sr_http: TEST PASSED")
else:
logger.info("sr_http: TEST FAILED")
sys.exit(1)
from sr_subscribe import *
from sr_watch import *
from sr_winnow import *
except:
from sarra.sr_config import *
from sarra.sr_poll import *
from sarra.sr_post import *
from sarra.sr_report import *
from sarra.sr_sarra import *
from sarra.sr_sender import *
from sarra.sr_shovel import *
from sarra.sr_subscribe import *
from sarra.sr_watch import *
from sarra.sr_winnow import *
cfg = sr_config()
action = sys.argv[-1]
#cfg.configure()
#cfg.loglevel = logging.DEBUG
#cfg.setlog()
# instantiate each program with its configuration file
# and invoke action if one of cleanup,declare,setup
def instantiate(dirconf, pgm, confname, action):
# c stuff always requiere to spawn a call
if pgm in ['audit', 'cpost', 'cpump']:
def self_test():
failed = False
logger = test_logger()
logger.info("sr_ftp : BEGIN TEST\n")
# configuration
cfg = sr_config()
cfg.logger = logger
cfg.configure()
cfg.set_sumalgo('d')
msg = sr_message(cfg)
msg.filesize = None
msg.onfly_checksum = False
# 1 bytes par 5 secs
#cfg.kbytes_ps = 0.0001
cfg.timeout = 5.0
cfg.kbytes_ps = 0.01
cfg.chmod_dir = 0o775
cfg.msg = msg
opt1 = "destination ftp://aspymjg@localhost"
cfg.option(opt1.split())
# make sure test directory is removed before test startup
ftp_url = cfg.destination
ftpuser = ftp_url.split('/')[2].split('@')[0]
testdir = os.path.expanduser('~' + ftpuser) + '/tztz'
try:
shutil.rmtree(testdir)
except:
pass
logger.info("TEST 01: instantiation and connection")
ftp = sr_ftp(cfg)
ftp.connect()
if ftp.check_is_connected():
logger.info("TEST 01: OK")
else:
logger.info("TEST 01: failed ... not connected")
failed = True
logger.info("TEST 02: mkdir,chmod,cd")
ftp.mkdir("tztz")
ls = ftp.ls()
if "tztz" in ls:
ftp.chmod(0o775, "tztz")
ftp.cd("tztz")
logger.info("TEST 02: OK")
else:
logger.info("TEST 02: failed")
failed = True
logger.info(
"TEST 03: put local aaa to remote bbb, chmod bbb, rename bbb ccc")
f = open("aaa", "wb")
f.write(b"1\n")
f.write(b"2\n")
f.write(b"3\n")
f.close()
ftp.put("aaa", "bbb")
ls = ftp.ls()
if "bbb" in ls:
ftp.chmod(0o775, "bbb")
ftp.rename("bbb", "ccc")
ls = ftp.ls()
if "ccc" in ls:
logger.info("TEST 03: OK")
else:
logger.info("TEST 03: failed")
failed = True
# build a message
msg.start_timer()
msg.topic = "v02.post.test"
msg.notice = "notice"
msg.baseurl = ftp_url
msg.relpath = "tztz/ccc"
msg.partflg = '1'
msg.offset = 0
msg.length = 6
msg.local_file = "bbb"
msg.local_offset = 0
cfg.set_sumalgo('d')
msg.sumalgo = cfg.sumalgo
msg.logger = logger
msg.new_file = "bbb"
msg.new_dir = "."
cfg.msg = msg
cfg.batch = 5
cfg.inflight = None
logger.info(
"TEST 04: transport download with exact name and onfly_checksum")
try:
os.unlink("bbb")
except:
pass
msg.onfly_checksum = None
tr = ftp_transport()
tr.download(cfg)
if os.path.exists("bbb") and cfg.msg.onfly_checksum:
os.unlink("./bbb")
logger.info("TEST 04: OK")
else:
logger.info("TEST 04: failed")
failed = True
logger.info("TEST 05: download file lock is .filename")
cfg.inflight = '.'
tr.download(cfg)
try:
os.unlink("./bbb")
logger.info("TEST 05: OK")
except:
logger.info("TEST 05: FAILED, file not found")
failed = True
logger.info("TEST 06: download file lock is filename.tmp")
cfg.inflight = '.tmp'
tr.download(cfg)
try:
os.unlink("./bbb")
logger.info("TEST 06: OK")
except:
logger.info("TEST 06: FAILED, file not found")
failed = True
# download the file... it is sent below
tr.download(cfg)
# closing way too much
tr.close()
tr.close()
tr.close()
tr = None
# configure sending
tr = ftp_transport()
cfg.local_file = "bbb"
msg.relpath = "./bbb"
msg.new_dir = "tztz"
msg.new_file = "ddd"
cfg.remote_file = "ddd"
cfg.remote_path = "tztz/ddd"
cfg.remote_urlstr = ftp_url + "/tztz/ddd"
cfg.remote_dir = "tztz"
cfg.chmod = 0o775
logger.info("TEST 07: send file with exact name")
cfg.inflight = None
tr.send(cfg)
if os.path.exists(testdir + os.sep + 'ddd'):
logger.info("TEST 07: OK")
else:
logger.info("TEST 07: FAILED, file not found")
failed = True
logger.debug = logger.silence
logger.info("TEST 08: delete remote file")
ftp = tr.proto
ftp.delete("ddd")
if not os.path.exists("~/tztz/ddd"):
logger.info("TEST 08: OK")
else:
logger.info("TEST 08: FAILED, file not found")
failed = True
# deleting a non existing file
logger.info("TEST 09: delete not existing file")
try:
ftp.delete("zzz_unexistant")
logger.info("TEST 09: OK")
except:
logger.info("TEST 09: FAILED")
failed = True
# testing several sending options
logger.info("TEST 10: sending file lock = .filename")
cfg.inflight = '.'
tr.send(cfg)
try:
ftp.delete("ddd")
logger.info("TEST 10: OK")
except:
logger.info("TEST 10: FAILED")
failed = True
logger.info("TEST 11: sending file lock = filename.tmp")
cfg.inflight = '.tmp'
tr.send(cfg)
try:
ftp.delete("ddd")
logger.info("TEST 11: OK")
except:
logger.info("TEST 11: FAILED")
failed = True
# testing a number of sends
logger.info("TEST 12: numerous send for the same file")
tr.send(cfg)
tr.send(cfg)
tr.send(cfg)
tr.close()
logger.info("TEST 12: OK")
# do cleanup from previous tests
ftp = sr_ftp(cfg)
ftp.connect()
ftp.cd("tztz")
ftp.ls()
ftp.delete("ccc")
ftp.delete("ddd")
ftp.cd("")
ftp.rmdir("tztz")
ftp.close()
logger.info("")
if not failed:
logger.info("sr_ftp: TEST PASSED")
else:
logger.info("sr_ftp: TEST FAILED")
sys.exit(1)
def self_test():
logger = test_logger()
# config setup
cfg = sr_config()
cfg.defaults()
cfg.general()
#cfg.debug = True
opt1 = "destination sftp://localhost"
cfg.option( opt1.split() )
cfg.logger = logger
cfg.timeout = 5.0
# 1 bytes par 5 secs
#cfg.kbytes_ps = 0.0001
cfg.kbytes_ps = 0.01
try:
sftp = sr_sftp(cfg)
sftp.connect()
sftp.mkdir("tztz")
sftp.chmod(775,"tztz")
sftp.cd("tztz")
f = open("aaa","wb")
f.write(b"1\n")
f.write(b"2\n")
f.write(b"3\n")
f.close()
sftp.put("aaa", "bbb")
ls = sftp.ls()
logger.info("ls = %s" % ls )
sftp.chmod(775,"bbb")
ls = sftp.ls()
logger.info("ls = %s" % ls )
sftp.rename("bbb", "ccc")
ls = sftp.ls()
logger.info("ls = %s" % ls )
sftp.get("ccc", "bbb",0,0,6)
f = open("bbb","rb")
data = f.read()
f.close()
if data != b"1\n2\n3\n" :
logger.error("sr_sftp1 TEST FAILED")
sys.exit(1)
os.unlink("bbb")
msg = sr_message(cfg)
msg.start_timer()
msg.topic = "v02.post.test"
msg.notice = "notice"
msg.urlcred = "sftp://localhost/"
msg.urlstr = "sftp://localhost/tztz/ccc"
msg.url = urllib.parse.urlparse(msg.urlcred+"tztz/ccc")
msg.partflg = '1'
msg.offset = 0
msg.length = 0
msg.local_file = "bbb"
msg.local_offset = 0
msg.sumalgo = None
cfg.msg = msg
cfg.batch = 5
cfg.lock = None
dldr = sftp_transport()
dldr.download(cfg)
logger.debug("checksum = %s" % msg.onfly_checksum)
dldr.download(cfg)
dldr.download(cfg)
cfg.logger.info("lock .")
cfg.lock = '.'
dldr.download(cfg)
dldr.download(cfg)
msg.sumalgo = cfg.sumalgo
dldr.download(cfg)
logger.debug("checksum = %s" % msg.onfly_checksum)
cfg.logger.info("lock .tmp")
cfg.lock = '.tmp'
dldr.download(cfg)
dldr.download(cfg)
dldr.close()
dldr.close()
dldr.close()
dldr = sftp_transport()
cfg.local_file = "bbb"
cfg.local_path = "./bbb"
cfg.remote_file = "ddd"
cfg.remote_path = "tztz/ddd"
cfg.remote_urlstr = "sftp://localhost/tztz/ddd"
cfg.remote_dir = "tztz"
cfg.chmod = 775
cfg.lock = None
dldr.send(cfg)
dldr.sftp.delete("ddd")
cfg.lock = '.'
dldr.send(cfg)
dldr.sftp.delete("ddd")
cfg.lock = '.tmp'
dldr.send(cfg)
dldr.send(cfg)
dldr.send(cfg)
dldr.send(cfg)
dldr.send(cfg)
dldr.send(cfg)
dldr.close()
dldr.close()
dldr.close()
sftp = sr_sftp(cfg)
sftp.connect()
sftp.cd("tztz")
sftp.delete("ccc")
sftp.delete("ddd")
logger.info("%s" % sftp.originalDir)
sftp.cd("")
logger.info("%s" % sftp.sftp.getcwd())
sftp.rmdir("tztz")
sftp.put("aaa","bbb",0,0,2)
sftp.put("aaa","bbb",2,4,2)
sftp.put("aaa","bbb",4,2,2)
sftp.get("bbb","bbb",2,2,2)
sftp.delete("bbb")
f = open("bbb","rb")
data = f.read()
f.close()
if data != b"1\n3\n3\n" :
logger.error("sr_sftp TEST FAILED ")
sys.exit(1)
sftp.close()
except:
(stype, svalue, tb) = sys.exc_info()
logger.error("(Type: %s, Value: %s)" % (stype ,svalue))
logger.error("sr_sftp TEST FAILED")
sys.exit(2)
os.unlink('aaa')
os.unlink('bbb')
print("sr_sftp TEST PASSED")
sys.exit(0)
def self_test():
logger = test_logger()
# config setup
cfg = sr_config()
cfg.defaults()
cfg.general()
cfg.debug = True
opt1 = "destination ftp://localhost"
cfg.option(opt1.split())
cfg.logger = logger
cfg.timeout = 5
try:
ftp = sr_ftp(cfg)
ftp.connect()
ftp.mkdir("tztz")
ftp.chmod(775, "tztz")
ftp.cd("tztz")
ftp.umask()
f = open("aaa", "wb")
f.write(b"1\n")
f.write(b"2\n")
f.write(b"3\n")
f.close()
ftp.put("aaa", "bbb")
ls = ftp.ls()
logger.info("ls = %s" % ls)
ftp.chmod(775, "bbb")
ls = ftp.ls()
logger.info("ls = %s" % ls)
ftp.rename("bbb", "ccc")
ls = ftp.ls()
logger.info("ls = %s" % ls)
ftp.get("ccc", "bbb")
f = open("bbb", "rb")
data = f.read()
f.close()
ftp.close()
if data != b"1\n2\n3\n":
logger.error("sr_ftp TEST FAILED")
sys.exit(1)
os.unlink("bbb")
msg = sr_message(cfg)
msg.start_timer()
msg.topic = "v02.post.test"
msg.notice = "notice"
msg.urlcred = "ftp://localhost/"
msg.urlstr = "ftp://localhost/tztz/ccc"
msg.url = urllib.parse.urlparse(msg.urlcred + "tztz/ccc")
msg.partflg = '1'
msg.offset = 0
msg.length = 0
msg.sumalgo = None
msg.local_file = "bbb"
msg.local_offset = 0
cfg.msg = msg
cfg.batch = 5
cfg.kbytes_ps = 0.05
dldr = ftp_transport()
cfg.lock = None
dldr.download(cfg)
dldr.download(cfg)
cfg.lock = '.'
dldr.download(cfg)
dldr.download(cfg)
logger.debug("checksum = %s" % msg.onfly_checksum)
dldr.download(cfg)
cfg.lock = '.tmp'
dldr.download(cfg)
dldr.download(cfg)
msg.sumalgo = cfg.sumalgo
dldr.download(cfg)
logger.debug("checksum = %s" % msg.onfly_checksum)
logger.debug("change context")
ok, details = cfg.credentials.get(cfg.destination)
details.binary = False
cfg.credentials.credentials[cfg.destination] = details
dldr.download(cfg)
logger.debug("checksum = %s" % msg.onfly_checksum)
dldr.close()
dldr.close()
dldr.close()
dldr = ftp_transport()
cfg.local_file = "bbb"
cfg.local_path = "./bbb"
cfg.remote_file = "ddd"
cfg.remote_path = "tztz/ddd"
cfg.remote_urlstr = "ftp://localhost/tztz/ddd"
cfg.remote_dir = "tztz"
cfg.chmod = 775
cfg.lock = None
dldr.send(cfg)
dldr.ftp.delete("ddd")
cfg.lock = '.'
dldr.send(cfg)
dldr.ftp.delete("ddd")
cfg.lock = '.tmp'
dldr.send(cfg)
dldr.send(cfg)
dldr.send(cfg)
logger.debug("change context back")
ok, details = cfg.credentials.get(cfg.destination)
details.binary = True
cfg.credentials.credentials[cfg.destination] = details
dldr.send(cfg)
dldr.send(cfg)
dldr.send(cfg)
dldr.close()
dldr.close()
dldr.close()
ftp = sr_ftp(cfg)
ftp.connect()
ftp.cd("tztz")
ftp.delete("ccc")
ftp.delete("ddd")
logger.info("%s" % ftp.originalDir)
ftp.cd("")
logger.info("%s" % ftp.ftp.pwd())
ftp.rmdir("tztz")
ftp.close()
except:
logger.error("sr_ftp TEST FAILED")
sys.exit(2)
os.unlink('aaa')
os.unlink('bbb')
print("sr_ftp TEST PASSED")
sys.exit(0)
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
#
#
import sys, os, os.path, time, subprocess
try :
from sr_config import *
except :
from sarra.sr_config import *
cfg = sr_config()
# an sr_subscribe config will be under ~/.config/sarra/subscribe,
# will be sr_subscribe ~/.config/sarra/subscribe/file.conf "action"
def invoke(confpath):
parts = confpath.split('/')
program = 'sr_' + parts[-2]
config = re.sub(r'(\.conf)','',parts[-1])
try :
cfg.logger.info("%s %s %s" % (program,config,sys.argv[-1]))
subprocess.check_call([program,config,sys.argv[-1]])
except :
def self_test():
failed = False
logger = test_logger()
# creating a temporary cache directory/file
tmpdirname = tempfile.TemporaryDirectory().name
try:
os.mkdir(tmpdirname)
except:
pass
tmpfilname = 'cache_test_file'
tmppath = tmpdirname + os.sep + 'cache_test_file'
cfg = sr_config(config=None, args=['test', '--debug', 'False'])
cfg.logger = logger
cfg.config_name = "test"
cfg.debug = False
cfg.defaults()
cfg.debug = False
cfg.general()
optH = "caching 1"
cfg.option(optH.split())
# check creation addition close
cache = sr_cache(cfg)
cache.open(tmppath)
cache.check('key1', 'file1', 'part1')
cache.check('key2', 'file2', 'part2')
cache.check('key1', 'file1', 'part1')
cache.close()
cache = sr_cache(cfg)
cache.open(tmppath)
cache.load()
# one collision when adding so 2 entries
if len(cache.cache_dict) != 2:
logger.error("test 01: expecting 2 entries...")
failed = True
# expire previous and add 3
time.sleep(1)
cache.check('key3', 'file3', 'part3')
cache.check_expire()
cache.check('key4', 'file4', None)
cache.check('key5', 'file5', 'part5')
if len(cache.cache_dict) != 3:
logger.error("test 02: expecting 3 entries...")
failed = True
#checking cache internals ...
#logger.error("%s" % cache.cache_dict)
cache.close()
# expire previous
time.sleep(1)
cache = sr_cache(cfg)
cache.open(tmppath)
if len(cache.cache_dict) != 0:
logger.error("test 03: expecting 0 entry...")
failed = True
cache.close()
#add 100 entries
cache = sr_cache(cfg)
cache.open(tmppath)
cache.load()
i = 0
now = time.time()
while i < 100:
cache.check('key%d' % i, 'file%d' % i, 'part%d' % i)
i = i + 1
if len(cache.cache_dict) != 100:
logger.error("test 04: expecting 100 entries...")
failed = True
# free cache
cache.free()
if len(cache.cache_dict) != 0:
logger.error("test 05: expecting 0 entry...")
failed = True
cache.close()
#add 10 entries
cache = sr_cache(cfg)
cache.open(tmppath)
i = 0
while i < 10:
cache.check('key%d' % i, 'file%d' % i, 'part%d' % i)
cache.check('key%d' % i, 'file%d' % i, 'part0%d' % i)
cache.check('key%d' % i, 'file%d' % i, 'part1%d' % i)
cache.check('key%d' % i, 'file%d' % i, 'part2%d' % i)
i = i + 1
# delete one
cache.delete_path('file8')
if len(cache.cache_dict) != 9:
logger.error("test 06: expecting 9 entries...got %d" %
len(cache.cache_dict))
failed = True
# expire and clean
time.sleep(1)
cache.clean()
if len(cache.cache_dict) != 0:
logger.error("test 07: expecting 0 entry...")
failed = True
# add one and save
cache.check('key%d' % i, 'file%d' % i, 'part2%d' % i)
cache.save()
if len(cache.cache_dict) != 1:
logger.error("test 08: expecting 1 entry...")
failed = True
if not os.path.exists(tmppath):
logger.error("test 09: cache file should exists")
failed = True
# close and unlink
cache.close(unlink=True)
if os.path.exists(tmppath):
logger.error("test 10: cache file should have been deleted")
failed = True
if not failed:
print("sr_cache.py TEST PASSED")
else:
print("sr_cache.py TEST FAILED")
sys.exit(1)
def self_test():
logger = test_logger()
# config setup
cfg = sr_config()
cfg.defaults()
cfg.general()
cfg.debug = True
cfg.logger = logger
cfg.kbytes_ps = 10.0
#setup consumer to catch first post
#dd.weather has strange permissions, so queue declare fails. 'unknown method'
#ok, cfg.broker = cfg.validate_urlstr("amqp://[email protected]/")
ok, cfg.broker = cfg.validate_urlstr("amqp://tsub@localhost/")
cfg.bindings = [ ( 'xpublic', 'v02.post.#') ]
cfg.user_cache_dir = os.getcwd()
cfg.config_name = "test"
cfg.queue_name = None
cfg.reset = True
consumer = sr_consumer(cfg)
i = 0
while True :
ok, msg = consumer.consume()
if ok: break
cfg.msg = msg
cfg.msg.sumalgo = None
cfg.msg.local_file = "toto"
cfg.msg.local_offset = 0
try:
tr = http_transport()
cfg.lock = None
tr.download(cfg)
logger.debug("checksum = %s" % cfg.msg.onfly_checksum)
cfg.lock = '.'
tr.download(cfg)
logger.debug("checksum = %s" % cfg.msg.onfly_checksum)
cfg.lock = '.tmp'
tr.download(cfg)
logger.debug("checksum = %s" % cfg.msg.onfly_checksum)
cfg.msg.sumalgo = cfg.sumalgo
tr.download(cfg)
logger.debug("checksum = %s" % cfg.msg.onfly_checksum)
logger.debug("checksum = %s" % cfg.msg.checksum)
except:
logger.error("sr_http TEST FAILED")
(stype, svalue, tb) = sys.exc_info()
logger.error('Unexpected error Type: %s, Value: %s' % (stype, svalue))
sys.exit(2)
os.unlink(consumer.queuepath)
consumer.close()
print("sr_http TEST PASSED")
sys.exit(0)
def self_test():
failed = False
# test include
f = open("./bbb.inc","w")
f.write("randomize True\n")
f.close()
f = open("./aaa.conf","w")
f.write("include bbb.inc\n")
f.close()
# instantiation, test include and overwrite logs
logger = test_logger()
cfg = sr_config(config="aaa")
cfg.logger = logger
cfg.configure()
if not cfg.randomize :
cfg.logger.error("test 01 : problem with include")
failed = True
# back to defaults + check isTrue
cfg.defaults()
if not cfg.isTrue('true') or cfg.isTrue('false') :
cfg.logger.error("test 02: problem with module isTrue")
failed = True
# ===================================
# TESTING checksum
# ===================================
cfg.configure()
# creating a temporary directory with testfile test_chksum_file
tmpdirname = tempfile.TemporaryDirectory().name
try : os.mkdir(tmpdirname)
except : pass
tmpfilname = 'test_chksum_file'
tmppath = tmpdirname + os.sep + 'test_chksum_file'
f=open(tmppath,'wb')
f.write(b"0123456789")
f.write(b"abcdefghij")
f.write(b"ABCDEFGHIJ")
f.write(b"9876543210")
f.close()
# checksum_0
cfg.set_sumalgo('0')
chk0 = cfg.sumalgo
chk0.set_path(tmppath)
f = open(tmppath,'rb')
for i in 0,1,2,3 :
chunk = f.read(10)
chk0.update(chunk)
f.close()
v = int(chk0.get_value())
if v < 0 or v > 9999 :
print("test 01: checksum_0 did not work")
failed = True
# checksum_d
cfg.set_sumalgo('d')
chkd = cfg.sumalgo
chkd.set_path(tmppath)
f = open(tmppath,'rb')
for i in 0,1,2,3 :
chunk = f.read(10)
chkd.update(chunk)
f.close()
if chkd.get_value() != '7efaff9e615737b379a45646c755c492' :
print("test 02: checksum_d did not work")
failed = True
# checksum_n
cfg.set_sumalgo('n')
chkn = cfg.sumalgo
chkn.set_path(tmpfilname)
f = open(tmppath,'rb')
for i in 0,1,2,3 :
chunk = f.read(10)
chkn.update(chunk)
f.close()
v=chkn.get_value()
if chkn.get_value() != 'fd6b0296fe95e19fcef6f21f77efdfed' :
print("test 03: checksum_N did not work")
failed = True
# checksum_N
cfg.set_sumalgo('N')
chkN = cfg.sumalgo
chkN.set_path(tmpfilname,'i,1,256,1,0,0')
v = chkN.get_value()
# this should do nothing
chunk = 'aaa'
chkN.update(chunk)
if chkN.get_value() != 'a0847ab809f83cb573b76671bb500a430372d2e3d5bce4c4cd663c4ea1b5c40f5eda439c09c7776ff19e3cc30459acc2a387cf10d056296b9dc03a6556da291f' :
print("test 04: checksum_N did not work")
failed = True
# checksum_s
cfg.set_sumalgo('s')
chks = cfg.sumalgo
chks.set_path(tmppath)
f = open(tmppath,'rb')
for i in 0,1,2,3 :
chunk = f.read(10)
chks.update(chunk)
f.close()
if chks.get_value() != 'e0103da78bbff9a741116e04f8bea2a5b291605c00731dcf7d26c0848bccf76dd2caa6771cb4b909d5213d876ab85094654f36d15e265d322b68fea4b78efb33':
print("test 05: checksum_s did not work")
failed = True
os.unlink(tmppath)
os.rmdir(tmpdirname)
# pluggin script checking
f = open("./scrpt.py","w")
f.write("class Transformer(object): \n")
f.write(" def __init__(self):\n")
f.write(" pass\n")
f.write("\n")
f.write(" def perform(self,parent):\n")
f.write(" if parent.this_value != 0 : return False\n")
f.write(" parent.this_value = 1\n")
f.write(" return True\n")
f.write("\n")
f.write("transformer = Transformer()\n")
f.write("self.on_message = transformer.perform\n")
f.close()
# able to find the script
ok, path = cfg.config_path("plugins","scrpt.py",mandatory=True,ctype='py')
if not ok :
cfg.logger.error("test 03: problem with config_path script not found")
failed = True
# able to load the script
cfg.execfile("on_message",path)
if cfg.on_message == None :
cfg.logger.error("test 04: problem with module execfile script not loaded")
failed = True
# able to run the script
cfg.this_value = 0
cfg.on_message(cfg)
if cfg.this_value != 1 :
cfg.logger.error("test 05: problem to run the script ")
failed = True
os.unlink("./scrpt.py")
# general ...
cfg.general()
cfg.logger.info(cfg.user_cache_dir)
cfg.logger.info(cfg.user_log_dir)
cfg.logger.info(cfg.user_config_dir)
# args ...
cfg.randomize = False
cfg.assemble = False
cfg.logrotate = 5
cfg.expire = 0
expire_value = 1000*60*60*3
message_value = 1000*60*60*24*7*3
cfg.message_ttl = 0
cfg.args(['-expire','3h','-message_ttl','3W','--randomize', '--assemble', 'True', '-logrotate', '10'])
if not cfg.randomize :
cfg.logger.error("test 06: args problem randomize")
failed = True
if not cfg.inplace :
cfg.logger.error("test 07: args problem assemble")
failed = True
if cfg.logrotate !=10 :
cfg.logger.error("test 08: args problem logrotate %s" % cfg.logrotate)
failed = True
if cfg.expire != expire_value :
cfg.logger.error("test 09: args problem expire %s" % cfg.expire)
failed = True
if cfg.message_ttl != message_value :
cfg.logger.error("test 10: args problem message_ttl %s" % cfg.message_ttl)
failed = True
# has_vip...
cfg.args(['-vip', '127.0.0.1' ])
if not cfg.has_vip():
cfg.logger.error("test 11: has_vip failed")
failed = True
opt1 = "hostname toto"
opt2 = "broker amqp://a:b@${HOSTNAME}"
cfg.option(opt1.split())
cfg.option(opt2.split())
if cfg.broker.geturl() != "amqp://*****:*****@toto" :
cfg.logger.error("test 12: varsub problem with replacing HOSTNAME")
failed = True
opt1 = "parts i,128"
cfg.option(opt1.split())
if cfg.partflg != 'i' or cfg.blocksize != 128:
cfg.logger.error("test 13: option parts or module validate_parts")
failed = True
opt1 = "sum z,d"
cfg.option(opt1.split())
if cfg.sumflg != 'z,d' :
cfg.logger.error("test 14: option sum or module validate_sum")
failed = True
opt1 = "sum R,0"
cfg.option(opt1.split())
if cfg.sumflg != 'R,0' :
cfg.logger.error("test 15: option sum or module validate_sum")
failed = True
opt1 = "sum d"
cfg.option(opt1.split())
if cfg.sumflg != 'd' or cfg.sumalgo.registered_as() != 'd' :
cfg.logger.error("test 16: option sum or module validate_sum")
failed = True
opt1 = "sum 0"
cfg.option(opt1.split())
if cfg.sumflg != '0' or cfg.sumalgo.registered_as() != '0' :
cfg.logger.error("test 17: option sum or module validate_sum")
failed = True
opt1 = "sum n"
cfg.option(opt1.split())
if cfg.sumflg != 'n' or cfg.sumalgo.registered_as() != 'n' :
cfg.logger.error("test 18: option sum or module validate_sum")
failed = True
opt1 = "sum s"
cfg.option(opt1.split())
if cfg.sumflg != 's' or cfg.sumalgo.registered_as() != 's':
cfg.logger.error("test 19: option sum or module validate_sum")
failed = True
opt1 = "sum N"
cfg.option(opt1.split())
if cfg.sumflg != 'N' or cfg.sumalgo.registered_as() != 'N':
cfg.logger.error("test 19b: option sum or module validate_sum")
failed = True
opt1 = "move toto titi"
cfg.option(opt1.split())
if cfg.movepath[0] != 'toto' or cfg.movepath[1] != 'titi' :
cfg.logger.error("test 20: option move for sr_post does not work")
failed = True
opt1 = "path .. ."
cfg.option(opt1.split())
if cfg.postpath[0] != os.path.abspath('..') or cfg.postpath[1] != os.path.abspath('.') :
cfg.logger.error("test 21: option path for sr_post does not work")
failed = True
opt1 = "inflight ."
cfg.option(opt1.split())
if cfg.inflight != '.' :
cfg.logger.error("test 22: option inflight . does not work")
failed = True
opt1 = "inflight .tmp"
cfg.option(opt1.split())
if cfg.inflight != '.tmp' :
cfg.logger.error("test 23: option inflight .tmp does not work")
failed = True
opt1 = "inflight 1.5"
cfg.option(opt1.split())
if cfg.inflight != 1.5 :
cfg.logger.error("test 24: option inflight 1.5 does not work")
failed = True
opt1 = "prefetch 10"
cfg.option(opt1.split())
if cfg.prefetch != 10 :
cfg.logger.error("test 25: prefetch option did not work")
failed = True
# reexecuting the config aaa.conf
cfg.logger.debug("test 25b: reparsing aaa.conf that includes bbb.inc")
cfg.config(cfg.user_config)
os.unlink('aaa.conf')
os.unlink('bbb.inc')
cfg.logger.debug("test 25b: worked")
opt1 = "header toto1=titi1"
cfg.option(opt1.split())
opt2 = "header toto2=titi2"
cfg.option(opt2.split())
opt3 = "header tutu1=None"
cfg.option(opt3.split())
opt4 = "header tutu2=None"
cfg.option(opt4.split())
if not 'toto1' in cfg.headers_to_add or \
not 'toto2' in cfg.headers_to_add or \
cfg.headers_to_add['toto1'] != 'titi1' or \
cfg.headers_to_add['toto2'] != 'titi2' or \
len(cfg.headers_to_add) != 2 :
cfg.logger.error("test 26: option header adding entries did not work")
failed = True
if not 'tutu1' in cfg.headers_to_del or \
not 'tutu2' in cfg.headers_to_del or \
len(cfg.headers_to_del) != 2 :
cfg.logger.error("test 27: option header deleting entries did not work")
failed = True
# expire in ms
opt4 = "expire 10m"
cfg.option(opt4.split())
if cfg.expire != 600000 :
cfg.logger.error("test 28: option expire or module duration_from_str did not work")
failed = True
# message_ttl in ms
opt4 = "message_ttl 20m"
cfg.option(opt4.split())
if cfg.message_ttl != 1200000 :
cfg.logger.error("test 29: option message_ttl or module duration_from_str did not work")
failed = True
os.environ["VAR1"] = "michel"
os.environ["VAR2"] = "peter"
os.environ["VAR3"] = "jun"
opt4="directory ${VAR1}/${VAR2}/${VAR3}/blabla"
cfg.option(opt4.split())
if '$' in cfg.currentDir or cfg.currentDir != 'michel/peter/jun/blabla':
cfg.logger.error("test 30: env variable substitution failed %s" % cfg.currentDir)
failed = True
opt4='strip 4'
cfg.option(opt4.split())
if cfg.strip != 4 :
cfg.logger.error("test 31: option strip with integer failed")
failed = True
opt4='strip .*aaa'
cfg.option(opt4.split())
if cfg.pstrip != '.*aaa' :
cfg.logger.error("test 32: option strip with pattern failed")
failed = True
pika = cfg.use_pika
opt4='use_pika True'
cfg.option(opt4.split())
if not cfg.use_pika :
cfg.logger.error("test 33: option use_pika boolean set to true failed")
failed = True
opt4='use_pika False'
cfg.option(opt4.split())
if cfg.use_pika :
cfg.logger.error("test 34: option use_pika boolean set to false failed")
failed = True
opt4='use_pika'
cfg.option(opt4.split())
if not cfg.use_pika :
cfg.logger.error("test 35: option use_pika boolean set to true without value failed")
failed = True
opt4='statehost False'
cfg.option(opt4.split())
if cfg.statehost :
cfg.logger.error("test 35: option statehost boolean set to false failed")
failed = True
opt4='statehost True'
cfg.option(opt4.split())
if not cfg.statehost or cfg.hostform != 'short' :
cfg.logger.error("test 36: option statehost boolean set to true, hostform short, failed")
failed = True
opt4='statehost SHORT'
cfg.option(opt4.split())
if not cfg.statehost or cfg.hostform != 'short' :
cfg.logger.error("test 37: option statehost set to SHORT, hostform short, failed")
failed = True
opt4='statehost fqdn'
cfg.option(opt4.split())
if not cfg.statehost or cfg.hostform != 'fqdn' :
cfg.logger.error("test 38: option statehost set to fqdn, hostform fqdn, failed")
failed = True
opt4='statehost TOTO'
cfg.option(opt4.split())
if cfg.statehost and cfg.hostform != None:
cfg.logger.error("test 39: option statehost set badly ... did not react correctly, failed")
failed = True
# extended options
opt4='extended TOTO'
cfg.option(opt4.split())
cfg.declare_option('extended')
if not cfg.check_extended():
cfg.logger.error("test 40: extend with new option, option was declared, but check_extended complained(False)")
failed = True
opt4='extended_bad TITI'
cfg.option(opt4.split())
# modify this test... causes error to be printed out ... which is ok... but annoying for conformity tests
#if cfg.check_extended():
if cfg.extended_bad[0] != 'TITI' :
cfg.logger.error("test 41: extend with new option, option not declared, value should still be ok")
failed = True
opt1 = "surplus_opt surplus_value"
cfg.option(opt1.split())
if cfg.surplus_opt != [ "surplus_value" ] :
cfg.logger.error("test 42: extend option did not work")
failed = True
opt1 = "surplus_opt surplus_value2"
cfg.option(opt1.split())
if cfg.surplus_opt[0] != "surplus_value" or cfg.surplus_opt[1] != "surplus_value2":
cfg.logger.error("test 43: extend option list did not work")
failed = True
# more options testing
opt1 = "base_dir /home/aspymjg/dev/metpx-sarracenia/sarra"
cfg.option(opt1.split())
if cfg.base_dir != '/home/aspymjg/dev/metpx-sarracenia/sarra':
cfg.logger.error("test 44: string option base_dir did not work")
failed = True
opt1 = "post_base_dir /totot/toto"
cfg.option(opt1.split())
if cfg.post_base_dir != '/totot/toto':
cfg.logger.error("test 45: string option post_base_dir did not work")
failed = True
opt1 = "post_base_url file://toto"
cfg.option(opt1.split())
if cfg.post_base_url != 'file://toto':
cfg.logger.error("test 46: url option post_base_url did not work")
failed = True
if cfg.outlet != 'post' :
cfg.logger.error("test 47: default error outlet = %s" % self.outlet)
failed = True
opt1 = "outlet json"
cfg.option(opt1.split())
if cfg.outlet != 'json' :
cfg.logger.error("test 48: option outlet value json did not work")
failed = True
opt1 = "outlet url"
cfg.option(opt1.split())
if cfg.outlet != 'url' :
cfg.logger.error("test 49: option outlet value url did not work")
failed = True
opt1 = "outlet post"
cfg.option(opt1.split())
if cfg.outlet != 'post' :
cfg.logger.error("test 50: option outlet value post did not work")
failed = True
# bad option setting skipped ... its output confuses conformity...
# complains about an error... and it is ok to complain.
opt1 = "outlet toto"
#cfg.option(opt1.split())
#if cfg.outlet != 'post' :
# cfg.logger.error("test 51: option outlet with bad value did not work")
# failed = True
if not cfg.retry_mode :
cfg.logger.error("test 52: retry_mode should be the default")
failed = True
opt1 = "retry_mode false"
cfg.option(opt1.split())
if cfg.retry_mode :
cfg.logger.error("test 53: retry_mode should be false")
failed = True
# retry_ttl in mins
opt1 = "retry_ttl 1D"
cfg.option(opt1.split())
if cfg.retry_ttl != 86400 :
cfg.logger.error("test 54: option retry_ttl or module duration_from_str did not work")
failed = True
# exchange_suffix
opt1 = "exchange_suffix suffix1"
cfg.option(opt1.split())
opt1 = "post_exchange_suffix suffix2"
cfg.option(opt1.split())
if cfg.exchange_suffix != 'suffix1' or cfg.post_exchange_suffix != 'suffix2' :
cfg.logger.error("test 55: option exchange_suffix or post_exchange_suffix did not work")
failed = True
# internal variables substitution
opt1 = "broker amqp://michel:[email protected]"
cfg.option(opt1.split())
opt1 = "post_base_dir /${broker.hostname}/${broker.username}"
cfg.option(opt1.split())
if cfg.post_base_dir != '/testbroker.toto/michel':
cfg.logger.error("test 56: replacing internal ${broker.hostname} ${broker.username} did not work")
failed = True
cfg.toto = ['tutu1','tutu2']
opt1 = "post_base_dir /${toto[1]}/${broker.username}/aaa"
cfg.option(opt1.split())
if cfg.post_base_dir != '/tutu2/michel/aaa':
cfg.logger.error("test 57: replacing internal ${toto[1]} did not work")
failed = True
cfg.toto = ['tutu1','tutu2']
opt1 = "post_base_dir /${toto}/${broker.username}/aaa"
cfg.option(opt1.split())
if cfg.post_base_dir != '/tutu1/michel/aaa':
cfg.logger.error("test 58: replacing internal ${toto} did not work")
failed = True
# more config test to perform for full coverage...
# options 'accept','get','reject']
# option 'accept_unmatched'
# def isMatchingPattern(self, str, accept_unmatch = False):
#def sundew_getDestInfos(self, filename):
#def validate_urlstr(self,urlstr):
# example of output for next test : new_dir = /20180404140433/SACN04CWAO140251RRA
dDir = '/${RYYYY}${RMM}${RDD}${RHH}${RMM}${RSS}/${T1}${T2}${A1}${A2}${ii}${CCCC}${YY}${GG}${Gg}${BBB}/'
new_dir = cfg.sundew_dirPattern(urlstr='',basename='SACN04_CWAO_140251_RRA',destDir=dDir,destName='aaa')
if not new_dir.endswith('/SACN04CWAO140251RRA') :
cfg.logger.error("test 59: sundew_dirPattern new_dir %s should end with /SACN04CWAO140251RRA" % new_dir)
failed = True
# retry_ttl in mins
opt1 = "sanity_log_dead 1D"
cfg.option(opt1.split())
if cfg.sanity_log_dead != 86400 :
cfg.logger.error("test 60: option sanity_log_dead or module duration_from_str did not work")
failed = True
# retry_ttl in mins
opt1 = "heartbeat ${sanity_log_dead}"
cfg.option(opt1.split())
if cfg.heartbeat != 86400 :
cfg.logger.error("test 61: option evaluated self.${} did not work")
failed = True
# retry_ttl in mins
opt1 = "subtopic aaa.vv\ ww.hh##bb.aaa.#"
w = opt1.split()
w = cfg.backslash_space(w)
cfg.option(w)
if cfg.heartbeat != 86400 :
cfg.logger.error("test 61: option evaluated self.${} did not work")
failed = True
if not failed :
print("sr_config.py TEST PASSED")
else :
print("sr_config.py TEST FAILED")
sys.exit(1)
def self_test():
logger = test_logger()
# config setup
cfg = sr_config()
cfg.defaults()
cfg.general()
cfg.debug = True
opt1 = "destination ftp://localhost"
cfg.option( opt1.split() )
cfg.logger = logger
cfg.timeout = 5
try:
ftp = sr_ftp(cfg)
ftp.connect()
ftp.mkdir("tztz")
ftp.chmod(775,"tztz")
ftp.cd("tztz")
ftp.umask()
f = open("aaa","wb")
f.write(b"1\n")
f.write(b"2\n")
f.write(b"3\n")
f.close()
ftp.put("aaa", "bbb")
ls = ftp.ls()
logger.info("ls = %s" % ls )
ftp.chmod(775,"bbb")
ls = ftp.ls()
logger.info("ls = %s" % ls )
ftp.rename("bbb", "ccc")
ls = ftp.ls()
logger.info("ls = %s" % ls )
ftp.get("ccc", "bbb")
f = open("bbb","rb")
data = f.read()
f.close()
ftp.close()
if data != b"1\n2\n3\n" :
logger.error("sr_ftp TEST FAILED")
sys.exit(1)
os.unlink("bbb")
msg = sr_message(cfg)
msg.start_timer()
msg.topic = "v02.post.test"
msg.notice = "notice"
msg.urlcred = "ftp://localhost/"
msg.urlstr = "ftp://localhost/tztz/ccc"
msg.url = urllib.parse.urlparse(msg.urlcred+"tztz/ccc")
msg.partflg = '1'
msg.offset = 0
msg.length = 0
msg.sumalgo = None
msg.local_file = "bbb"
msg.local_offset = 0
cfg.msg = msg
cfg.batch = 5
cfg.kbytes_ps= 0.05
dldr = ftp_transport()
cfg.lock = None
dldr.download(cfg)
dldr.download(cfg)
cfg.lock = '.'
dldr.download(cfg)
dldr.download(cfg)
logger.debug("checksum = %s" % msg.onfly_checksum)
dldr.download(cfg)
cfg.lock = '.tmp'
dldr.download(cfg)
dldr.download(cfg)
msg.sumalgo = cfg.sumalgo
dldr.download(cfg)
logger.debug("checksum = %s" % msg.onfly_checksum)
logger.debug("change context")
ok, details = cfg.credentials.get(cfg.destination)
details.binary = False
cfg.credentials.credentials[cfg.destination] = details
dldr.download(cfg)
logger.debug("checksum = %s" % msg.onfly_checksum)
dldr.close()
dldr.close()
dldr.close()
dldr = ftp_transport()
cfg.local_file = "bbb"
cfg.local_path = "./bbb"
cfg.remote_file = "ddd"
cfg.remote_path = "tztz/ddd"
cfg.remote_urlstr = "ftp://localhost/tztz/ddd"
cfg.remote_dir = "tztz"
cfg.chmod = 775
cfg.lock = None
dldr.send(cfg)
dldr.ftp.delete("ddd")
cfg.lock = '.'
dldr.send(cfg)
dldr.ftp.delete("ddd")
cfg.lock = '.tmp'
dldr.send(cfg)
dldr.send(cfg)
dldr.send(cfg)
logger.debug("change context back")
ok, details = cfg.credentials.get(cfg.destination)
details.binary = True
cfg.credentials.credentials[cfg.destination] = details
dldr.send(cfg)
dldr.send(cfg)
dldr.send(cfg)
dldr.close()
dldr.close()
dldr.close()
ftp = sr_ftp(cfg)
ftp.connect()
ftp.cd("tztz")
ftp.delete("ccc")
ftp.delete("ddd")
logger.info("%s" % ftp.originalDir)
ftp.cd("")
logger.info("%s" % ftp.ftp.pwd())
ftp.rmdir("tztz")
ftp.close()
except:
logger.error("sr_ftp TEST FAILED")
sys.exit(2)
os.unlink('aaa')
os.unlink('bbb')
print("sr_ftp TEST PASSED")
sys.exit(0)
def self_test():
failed = False
cfg = sr_config()
cfg.configure()
credentials = sr_credentials(cfg.logger)
# covers : parse, credential_details obj, isTrue, add
line = "ftp://*****:*****@localhost active , binary"
credentials.parse(line)
# covers get with match
urlstr = "ftp://*****:*****@localhost"
ok, details = credentials.get(urlstr)
if not ok:
print("test 01: parsed %s and could not get %s " % (line, urlstr))
failed = True
# check details
if not details.passive == False or \
not details.binary == True :
print("test 02: parsed %s and passive = %s, binary = %s" %
(line, details.passive, details.binary))
failed = True
# covers get with resolve 1
urlstr = "ftp://localhost"
ok, details = credentials.get(urlstr)
if not ok:
print("test 03: parsed %s and could not get %s " % (line, urlstr))
failed = True
# covers get with resolve 2
urlstr = "ftp://guest@localhost"
ok, details = credentials.get(urlstr)
if not ok:
print("test 04: parsed %s and could not get %s " % (line, urlstr))
failed = True
# covers unresolve get
urlstr = "http://localhost"
ok, details = credentials.get(urlstr)
if ok:
print("test 05: parsed %s and could get %s " % (line, urlstr))
failed = True
# covers read
urlstr = "sftp://ruser@remote"
line = urlstr + " ssh_keyfile=/tmp/mytoto2\n"
f = open('/tmp/mytoto2', 'w')
f.write(line)
f.close()
credentials.read('/tmp/mytoto2')
ok, details = credentials.get(urlstr)
if not ok or details.ssh_keyfile != '/tmp/mytoto2':
print("test 06: read %s and could not get %s (ssh_keyfile = %s) " %
(line, urlstr, details.ssh_keyfile))
failed = True
os.unlink('/tmp/mytoto2')
# covers isValid
bad_urls = [
"ftp://host", "file://aaa/", "ftp://user@host", "ftp://user:@host",
"ftp://:pass@host"
]
for urlstr in bad_urls:
if credentials.isValid(urllib.parse.urlparse(urlstr)):
print("test 07: isValid %s returned True" % urlstr)
failed = True
# covers Valid sftp
ok_urls = [
"sftp://host", "sftp://aaa/", "sftp://user@host", "sftp://user:@host",
"sftp://:pass@host"
]
for urlstr in ok_urls:
if not credentials.isValid(urllib.parse.urlparse(urlstr)):
print("test 08: not isValid %s returned True" % urlstr)
failed = True
if not failed:
print("sr_credentials.py TEST PASSED")
else:
print("sr_credentials.py TEST FAILED")
sys.exit(1)
def self_test():
failed = False
# configuration
cfg = sr_config()
cfg.configure()
cfg.set_sumalgo('d')
print("sr_sftp: BEGIN TEST\n")
msg = sr_message(cfg)
msg.filesize = None
msg.lastchunk = True
msg.onfly_checksum = False
# 1 bytes par 5 secs
#cfg.kbytes_ps = 0.0001
cfg.timeout = 5.0
cfg.kbytes_ps = 0.01
cfg.chmod_dir = 0o775
cfg.msg = msg
opt1 = "destination sftp://${SFTPUSER}@localhost"
cfg.option(opt1.split())
# make sure test directory is removed before test startup
sftp_url = cfg.destination
sftpuser = sftp_url.split('/')[2].split('@')[0]
testdir = os.path.expanduser('~' + sftpuser) + '/tztz'
try:
shutil.rmtree(testdir)
except:
pass
print("TEST 01: instantiation and connection")
sftp = sr_sftp(cfg)
sftp.connect()
if sftp.check_is_connected():
print("TEST 01: OK")
else:
print("TEST 01: failed ... not connected")
failed = True
print("TEST 02: mkdir,rmdir,chmod,cd")
sftp.mkdir("txtx")
ls = sftp.ls()
if "txtx" in ls:
sftp.chmod(0o775, "txtx")
sftp.delete("txtx")
ls = sftp.ls()
if "txtx" in ls:
print("TEST 02: failed1")
failed = True
else:
print("TEST 02: OK1")
sftp.mkdir("tztz")
ls = sftp.ls()
if "tztz" in ls:
sftp.chmod(0o775, "tztz")
sftp.cd("tztz")
print("TEST 02: OK2")
else:
print("TEST 02: failed2")
failed = True
print("TEST 03: put local aaa to remote bbb, chmod bbb, rename bbb ccc")
f = open("aaa", "wb")
f.write(b"1\n")
f.write(b"2\n")
f.write(b"3\n")
f.close()
sftp.put("aaa", "bbb")
ls = sftp.ls()
if "bbb" in ls:
sftp.chmod(0o775, "bbb")
sftp.rename("bbb", "ccc")
ls = sftp.ls()
if "ccc" in ls:
print("TEST 03: OK")
else:
print("TEST 03: failed")
failed = True
print("TEST 04: get part of remote ccc into bbb")
sftp.get("ccc", "bbb", 1, 0, 4)
f = open("bbb", "rb")
data = f.read()
f.close()
if data == b"\n2\n3":
print("TEST 04: OK")
else:
print("TEST 04: failed")
failed = True
# build a message
msg.start_timer()
msg.topic = "v02.post.test"
msg.notice = "notice"
msg.pubtime = "20190109084420.788106"
msg.baseurl = sftp_url
msg.relpath = "tztz/ccc"
msg.partflg = '1'
msg.offset = 0
msg.length = 6
msg.local_file = "bbb"
msg.local_offset = 0
cfg.set_sumalgo('d')
msg.sumalgo = cfg.sumalgo
msg.new_file = "bbb"
msg.new_dir = "."
cfg.msg = msg
cfg.batch = 5
cfg.inflight = None
print("TEST 05: transport download with exact name and onfly_checksum")
try:
os.unlink("bbb")
except:
pass
msg.onfly_checksum = None
tr = sftp_transport()
tr.download(cfg)
if os.path.exists("bbb") and cfg.msg.onfly_checksum:
os.unlink("./bbb")
print("TEST 05: OK")
else:
print("TEST 05: failed")
failed = True
print("TEST 06: download file lock is .filename")
cfg.inflight = '.'
tr.download(cfg)
try:
os.unlink("./bbb")
print("TEST 06: OK")
except:
print("TEST 06: FAILED, file not found")
failed = True
print("TEST 07: download file lock is filename.tmp")
cfg.inflight = '.tmp'
tr.download(cfg)
try:
os.unlink("./bbb")
print("TEST 07: OK")
except:
print("TEST 07: FAILED, file not found")
failed = True
# download the file... it is sent below
tr.download(cfg)
# closing way too much
tr.close()
tr.close()
tr.close()
# configure sending
tr = sftp_transport()
cfg.local_file = "bbb"
msg.relpath = "./bbb"
msg.new_dir = "tztz"
msg.new_file = "ddd"
cfg.remote_file = "ddd"
cfg.remote_path = "tztz/ddd"
cfg.remote_urlstr = sftp_url + "/tztz/ddd"
cfg.remote_dir = "tztz"
cfg.chmod = 0o775
print("TEST 08: send file with exact name")
cfg.inflight = None
tr.send(cfg)
if os.path.exists(os.path.expanduser("~/tztz/ddd")):
print("TEST 08: OK")
else:
print("TEST 08: FAILED, file not found")
failed = True
print("TEST 09: delete remote file")
sftp = tr.proto
sftp.delete("ddd")
if not os.path.exists("~/tztz/ddd"):
print("TEST 09: OK")
else:
print("TEST 09: FAILED, file not found")
failed = True
# deleting a non existing file
print("TEST 10: delete not existing file")
try:
sftp.delete("zzz_unexistant")
print("TEST 10: OK")
except:
print("TEST 10: FAILED")
failed = True
# testing several sending options
print("TEST 11: sending file lock = .filename")
cfg.inflight = '.'
tr.send(cfg)
try:
sftp.delete("ddd")
print("TEST 11: OK")
except:
print("TEST 11: FAILED")
failed = True
print("TEST 12: sending file lock = filename.tmp")
cfg.inflight = '.tmp'
tr.send(cfg)
try:
sftp.delete("ddd")
print("TEST 12: OK")
except:
print("TEST 12: FAILED")
failed = True
# testing a number of sends
print("TEST 13: numerous send for the same file")
tr.send(cfg)
tr.send(cfg)
tr.send(cfg)
tr.close()
print("TEST 13: OK")
# do cleanup from previous tests
sftp = sr_sftp(cfg)
sftp.connect()
sftp.cd("tztz")
sftp.ls()
sftp.delete("ccc")
sftp.delete("ddd")
print("%s" % sftp.originalDir)
sftp.cd("")
print("%s" % sftp.getcwd())
sftp.rmdir("tztz")
sftp.close()
pwd = os.getcwd()
# as sftp do part stuff back and fore
print("TEST 14: testing parts get/put")
sftp = sr_sftp(cfg)
sftp.connect()
sftp.cd(pwd)
sftp.set_sumalgo(cfg.sumalgo)
sftp.put("aaa", "bbb", 0, 0, 2)
sftp.get("aaa", "bbb", 2, 2, 2)
sftp.put("aaa", "bbb", 4, 4, 2)
f = open("bbb", "rb")
data = f.read()
f.close()
if data == b"1\n2\n3\n":
print("TEST 14: OK")
else:
print("TEST 14: FAILED")
failed = True
sftp.delete("bbb")
sftp.delete("aaa")
sftp.close()
if not failed:
print("sr_sftp: TEST PASSED")
else:
print("sr_sftp: TEST FAILED")
sys.exit(1)
def self_test():
logger = test_logger()
# config setup
cfg = sr_config()
cfg.defaults()
cfg.general()
cfg.debug = True
cfg.logger = logger
cfg.kbytes_ps = 10.0
#setup consumer to catch first post
#dd.weather has strange permissions, so queue declare fails. 'unknown method'
#ok, cfg.broker = cfg.validate_urlstr("amqp://[email protected]/")
ok, cfg.broker = cfg.validate_urlstr("amqp://tsub@localhost/")
cfg.bindings = [('xpublic', 'v02.post.#')]
cfg.user_cache_dir = os.getcwd()
cfg.config_name = "test"
cfg.queue_name = None
cfg.reset = True
consumer = sr_consumer(cfg)
i = 0
while True:
ok, msg = consumer.consume()
if ok: break
cfg.msg = msg
cfg.msg.sumalgo = None
cfg.msg.local_file = "toto"
cfg.msg.local_offset = 0
try:
tr = http_transport()
cfg.lock = None
tr.download(cfg)
logger.debug("checksum = %s" % cfg.msg.onfly_checksum)
cfg.lock = '.'
tr.download(cfg)
logger.debug("checksum = %s" % cfg.msg.onfly_checksum)
cfg.lock = '.tmp'
tr.download(cfg)
logger.debug("checksum = %s" % cfg.msg.onfly_checksum)
cfg.msg.sumalgo = cfg.sumalgo
tr.download(cfg)
logger.debug("checksum = %s" % cfg.msg.onfly_checksum)
logger.debug("checksum = %s" % cfg.msg.checksum)
except:
logger.error("sr_http TEST FAILED")
(stype, svalue, tb) = sys.exc_info()
logger.error('Unexpected error Type: %s, Value: %s' % (stype, svalue))
sys.exit(2)
os.unlink(consumer.queuepath)
consumer.close()
print("sr_http TEST PASSED")
sys.exit(0)
