def connect(self):
# =============
# create message
# =============
self.msg = sr_message(self)
# =============
# consumer
# =============
self.consumer = sr_consumer(self)
self.msg.report_publisher = self.consumer.publish_back()
self.msg.report_exchange = self.report_exchange
self.msg.user = self.details.url.username
self.msg.host = self.details.url.scheme + '://' + self.details.url.hostname
self.msg.post_exchange_split = self.post_exchange_split
# =============
# poster
# =============
if self.post_broker:
self.poster = sr_poster(self)
self.msg.publisher = self.poster.publisher
self.msg.pub_exchange = self.post_exchange
self.msg.user = self.post_broker.username
self.connected = True
def connect(self):
# =============
# create message if needed
# =============
self.msg = sr_message(self)
# =============
# consumer queue_name : let consumer takes care of it
# =============
self.consumer = sr_consumer(self)
# =============
# publisher... (publish back to consumer)
# =============
self.publisher = self.consumer.publish_back()
# =============
# setup message publisher
# =============
self.msg.publisher = self.consumer.publisher
self.msg.post_exchange_split = self.post_exchange_split
def connect(self):
# =============
# create message
# =============
self.msg = sr_message(self)
# =============
# consumer queue_name : let consumer takes care of it
# =============
self.consumer = sr_consumer(self)
# =============
# publisher... on remote cluster
# =============
self.hc = HostConnect(logger=self.logger)
self.hc.set_url(self.cluster_broker)
self.hc.connect()
self.publisher = Publisher(self.hc)
self.publisher.build()
# =============
# setup message publisher
# =============
self.msg.publisher = self.publisher
self.msg.post_exchange_split = self.post_exchange_split
def get_message(self):
if not hasattr(self.parent, 'msg'):
self.parent.msg = sr_message(self.parent)
self.raw_msg = None
self.msg = self.parent.msg
self.msg.user = self.broker.username
def connect(self):
# =============
# create message if needed
# =============
self.msg = sr_message(self)
# =============
# consumer queue_name : let consumer takes care of it
# =============
self.consumer = sr_consumer(self)
# =============
# publisher... (publish back to consumer)
# =============
self.publisher = self.consumer.publish_back()
# =============
# setup message publisher
# =============
self.msg.publisher = self.consumer.publisher
self.msg.post_exchange_split = self.post_exchange_split
def connect(self):
# =============
# create message
# =============
self.msg = sr_message(self)
# =============
# consumer queue_name : let consumer takes care of it
# =============
self.consumer = sr_consumer(self)
# =============
# publisher... on remote cluster
# =============
self.hc = HostConnect( logger = self.logger )
self.hc.set_url( self.cluster_broker )
self.hc.connect()
self.publisher = Publisher(self.hc)
self.publisher.build()
# =============
# setup message publisher
# =============
self.msg.publisher = self.publisher
self.msg.post_exchange_split = self.post_exchange_split
def connect(self):
# =============
# create message
# =============
self.msg = sr_message(self)
# =============
# consumer
# =============
self.consumer = sr_consumer(self)
self.msg.report_publisher = self.consumer.publish_back()
self.msg.report_exchange = self.report_exchange
self.msg.user = self.details.url.username
self.msg.host = self.details.url.scheme + '://' + self.details.url.hostname
self.msg.post_exchange_split = self.post_exchange_split
# =============
# poster
# =============
if self.post_broker :
self.poster = sr_poster(self)
self.msg.publisher = self.poster.publisher
self.msg.pub_exchange = self.post_exchange
self.msg.user = self.post_broker.username
self.connected = True
def connect(self):
self.logger.debug("%s connect" % self.program_name)
# =============
# create message if needed
# =============
self.msg = sr_message(self)
# =============
# posting
# =============
loop = True
#20200404 - PS looping for connection has nothing to do with looping for watch.
# self.sleep says whether watch should loop.
# loop is about whether we try over and over to connect... two are not related.
# that's why following line is commented.
#if self.sleep <= 0 : loop = False
self.post_hc = HostConnect(logger=self.logger)
self.post_hc.choose_amqp_alternative(self.use_amqplib, self.use_pika)
self.post_hc.set_url(self.post_broker)
self.post_hc.loop = loop
if not self.post_hc.connect():
return
self.publisher = Publisher(self.post_hc)
self.publisher.build()
self.logger.info("Output AMQP broker(%s) user(%s) vhost(%s)" % \
(self.post_broker.hostname,self.post_broker.username,self.post_broker.path) )
# =============
# setup message publish
# =============
self.msg.user = self.post_broker.username
self.msg.publisher = self.publisher
self.msg.pub_exchange = self.post_exchange
self.msg.post_exchange_split = self.post_exchange_split
self.logger.info("Output AMQP exchange(%s)" % self.msg.pub_exchange)
# =============
# amqp resources
# =============
self.declare_exchanges()
# =============
# retransmit/restore_queue
# =============
if self.restore_queue != None:
self.publisher.restore_set(self)
self.msg.pub_exchange = self.publisher.restore_exchange
self.msg.post_exchange_split = 0
def get_message(self):
self.logger.debug("sr_consumer get_message")
if not hasattr(self.parent,'msg'):
self.parent.msg = sr_message(self.parent)
self.raw_msg = None
self.msg = self.parent.msg
self.msg.user = self.broker.username
def get_message(self):
#self.logger.debug("sr_consumer get_message")
if not hasattr(self.parent, 'msg'):
self.parent.msg = sr_message(self.parent)
self.raw_msg = None
self.msg = self.parent.msg
self.msg.user = self.broker.username
def get_message(self):
self.logger.debug("sr_poster get_message")
if not hasattr(self.parent, 'msg'):
self.parent.msg = sr_message(self.parent)
self.msg = self.parent.msg
self.msg.user = self.broker.username
self.msg.publisher = self.publisher
def get_message(self):
self.logger.debug("sr_poster get_message")
if not hasattr(self.parent,'msg'):
self.parent.msg = sr_message(self.parent)
self.msg = self.parent.msg
self.msg.user = self.broker.username
self.msg.publisher = self.publisher
def connect(self):
self.logger.debug("%s connect" % self.program_name)
# =============
# create message if needed
# =============
self.msg = sr_message(self)
# =============
# posting
# =============
loop = True
if self.sleep <= 0: loop = False
self.post_hc = HostConnect(logger=self.logger)
self.post_hc.choose_amqp_alternative(self.use_amqplib, self.use_pika)
self.post_hc.set_url(self.post_broker)
self.post_hc.loop = loop
if not self.post_hc.connect():
return
self.publisher = Publisher(self.post_hc)
self.publisher.build()
self.logger.info("Output AMQP broker(%s) user(%s) vhost(%s)" % \
(self.post_broker.hostname,self.post_broker.username,self.post_broker.path) )
# =============
# setup message publish
# =============
self.msg.user = self.post_broker.username
self.msg.publisher = self.publisher
self.msg.pub_exchange = self.post_exchange
self.msg.post_exchange_split = self.post_exchange_split
self.logger.info("Output AMQP exchange(%s)" % self.msg.pub_exchange)
# =============
# amqp resources
# =============
self.declare_exchanges()
# =============
# retransmit/restore_queue
# =============
if self.restore_queue != None:
self.publisher.restore_set(self)
self.msg.pub_exchange = self.publisher.restore_exchange
self.msg.post_exchange_split = 0
def connect(self):
# =============
# create message
# =============
self.msg = sr_message(self)
# =============
# consumer
# =============
self.consumer = sr_consumer(self)
self.msg.report_publisher = self.consumer.publish_back()
self.msg.report_exchange = self.report_exchange
self.msg.user = self.broker.username
self.logger.info(
"reading from to %s@%s, exchange: %s" %
(self.broker.username, self.broker.hostname, self.msg.exchange))
self.logger.info("report back to %s@%s, exchange: %s" %
(self.broker.username, self.broker.hostname,
self.msg.report_exchange))
# =============
# poster if post_broker different from broker
# =============
self.poster = None
if self.post_broker.geturl() != self.broker.geturl():
self.poster = sr_poster(self)
self.msg.publisher = self.poster.publisher
# =============
# publisher if post_broker is same as broker
# =============
else:
self.publisher = Publisher(self.consumer.hc)
self.publisher.build()
self.msg.publisher = self.publisher
self.logger.info("Output AMQP broker(%s) user(%s) vhost(%s)" % \
(self.broker.hostname,self.broker.username,self.broker.path) )
self.msg.pub_exchange = self.post_exchange
self.msg.post_exchange_split = self.post_exchange_split
self.logger.info("Output AMQP exchange(%s)" % self.msg.pub_exchange)
def connect(self):
# =============
# create message
# =============
self.msg = sr_message(self)
# =============
# consumer
# =============
self.consumer = sr_consumer(self)
self.msg.report_publisher = self.consumer.publish_back()
self.msg.report_exchange = self.report_exchange
self.msg.user = self.broker.username
self.logger.info("reading from to %s@%s, exchange: %s" %
( self.broker.username, self.broker.hostname, self.msg.exchange ) )
self.logger.info("report back to %s@%s, exchange: %s" %
( self.broker.username, self.broker.hostname, self.msg.report_exchange ) )
# =============
# poster if post_broker different from broker
# =============
self.poster = None
if self.post_broker.geturl() != self.broker.geturl() :
self.poster = sr_poster(self)
self.msg.publisher = self.poster.publisher
# =============
# publisher if post_broker is same as broker
# =============
else :
self.publisher = Publisher(self.consumer.hc)
self.publisher.build()
self.msg.publisher = self.publisher
self.logger.info("Output AMQP broker(%s) user(%s) vhost(%s)" % \
(self.broker.hostname,self.broker.username,self.broker.path) )
self.msg.pub_exchange = self.post_exchange
self.msg.post_exchange_split = self.post_exchange_split
self.logger.info("Output AMQP exchange(%s)" % self.msg.pub_exchange )
def connect(self):
self.logger.debug("sr_post connect")
# sr_post : no loop to reconnect to broker
self.loop = True
if self.program_name == 'sr_post' :
self.loop = False
# message
self.msg = sr_message( self )
# publisher
self.post_broker = self.broker
self.poster = sr_poster(self, self.loop)
self.msg.publisher = self.poster.publisher
self.msg.post_exchange_split = self.post_exchange_split
def connect(self):
self.logger.debug("sr_post connect")
# sr_post : no loop to reconnect to broker
self.loop = True
if self.program_name == 'sr_post':
self.loop = False
# message
self.msg = sr_message(self)
# publisher
self.post_broker = self.broker
self.poster = sr_poster(self, self.loop)
self.msg.publisher = self.poster.publisher
self.msg.post_exchange_split = self.post_exchange_split
def connect(self):
# =============
# create message
# =============
self.msg = sr_message(self)
# =============
# poster
# =============
self.post_broker = self.broker
self.post_exchange = self.exchange
self.poster = sr_poster(self)
self.msg.publisher = self.poster.publisher
self.msg.pub_exchange = self.post_exchange
self.connected = True
def connect(self):
# =============
# create message if needed
# =============
self.msg = sr_message(self)
# =============
# consumer
# =============
self.consumer = sr_consumer(self)
if self.reportback:
self.msg.report_publisher = self.consumer.publish_back()
self.msg.report_exchange = self.report_exchange
self.logger.info("report_back to %s@%s, exchange: %s" %
(self.broker.username, self.broker.hostname,
self.msg.report_exchange))
else:
self.logger.info("report_back suppressed")
# =============
# publisher
# =============
# publisher host
self.hc_pst = HostConnect(logger=self.logger)
self.hc_pst.set_url(self.post_broker)
self.hc_pst.connect()
# publisher
self.publisher = Publisher(self.hc_pst)
self.publisher.build()
self.msg.publisher = self.publisher
self.msg.post_exchange_split = self.post_exchange_split
def connect(self):
# =============
# create message
# =============
self.msg = sr_message(self)
# =============
# poster
# =============
self.post_broker = self.broker
self.post_exchange = self.exchange
self.poster = sr_poster(self)
self.msg.publisher = self.poster.publisher
self.msg.pub_exchange = self.post_exchange
self.msg.post_exchange_split = self.post_exchange_split
self.connected = True
def connect(self):
# =============
# create message if needed
# =============
self.msg = sr_message(self)
# =============
# consumer
# =============
self.consumer = sr_consumer(self)
if self.reportback :
self.msg.report_publisher = self.consumer.publish_back()
self.msg.report_exchange = self.report_exchange
self.logger.info("reportback to %s@%s, exchange: %s" %
( self.broker.username, self.broker.hostname, self.msg.report_exchange ) )
else:
self.logger.info( "reportback suppressed" )
# =============
# publisher
# =============
# publisher host
self.hc_pst = HostConnect( logger = self.logger )
self.hc_pst.set_url( self.post_broker )
self.hc_pst.connect()
# publisher
self.publisher = Publisher(self.hc_pst)
self.publisher.build()
self.msg.publisher = self.publisher
self.msg.post_exchange_split = self.post_exchange_split
def self_test():
failed = False
# configuration
cfg = sr_config()
cfg.configure()
cfg.set_sumalgo('d')
print("sr_sftp: BEGIN TEST\n")
msg = sr_message(cfg)
msg.filesize = None
msg.lastchunk = True
msg.onfly_checksum = False
# 1 bytes par 5 secs
#cfg.kbytes_ps = 0.0001
cfg.timeout = 5.0
cfg.kbytes_ps = 0.01
cfg.chmod_dir = 0o775
cfg.msg = msg
opt1 = "destination sftp://${SFTPUSER}@localhost"
cfg.option(opt1.split())
# make sure test directory is removed before test startup
sftp_url = cfg.destination
sftpuser = sftp_url.split('/')[2].split('@')[0]
testdir = os.path.expanduser('~' + sftpuser) + '/tztz'
try:
shutil.rmtree(testdir)
except:
pass
print("TEST 01: instantiation and connection")
sftp = sr_sftp(cfg)
sftp.connect()
if sftp.check_is_connected():
print("TEST 01: OK")
else:
print("TEST 01: failed ... not connected")
failed = True
print("TEST 02: mkdir,rmdir,chmod,cd")
sftp.mkdir("txtx")
ls = sftp.ls()
if "txtx" in ls:
sftp.chmod(0o775, "txtx")
sftp.delete("txtx")
ls = sftp.ls()
if "txtx" in ls:
print("TEST 02: failed1")
failed = True
else:
print("TEST 02: OK1")
sftp.mkdir("tztz")
ls = sftp.ls()
if "tztz" in ls:
sftp.chmod(0o775, "tztz")
sftp.cd("tztz")
print("TEST 02: OK2")
else:
print("TEST 02: failed2")
failed = True
print("TEST 03: put local aaa to remote bbb, chmod bbb, rename bbb ccc")
f = open("aaa", "wb")
f.write(b"1\n")
f.write(b"2\n")
f.write(b"3\n")
f.close()
sftp.put("aaa", "bbb")
ls = sftp.ls()
if "bbb" in ls:
sftp.chmod(0o775, "bbb")
sftp.rename("bbb", "ccc")
ls = sftp.ls()
if "ccc" in ls:
print("TEST 03: OK")
else:
print("TEST 03: failed")
failed = True
print("TEST 04: get part of remote ccc into bbb")
sftp.get("ccc", "bbb", 1, 0, 4)
f = open("bbb", "rb")
data = f.read()
f.close()
if data == b"\n2\n3":
print("TEST 04: OK")
else:
print("TEST 04: failed")
failed = True
# build a message
msg.start_timer()
msg.topic = "v02.post.test"
msg.notice = "notice"
msg.pubtime = "20190109084420.788106"
msg.baseurl = sftp_url
msg.relpath = "tztz/ccc"
msg.partflg = '1'
msg.offset = 0
msg.length = 6
msg.local_file = "bbb"
msg.local_offset = 0
cfg.set_sumalgo('d')
msg.sumalgo = cfg.sumalgo
msg.new_file = "bbb"
msg.new_dir = "."
cfg.msg = msg
cfg.batch = 5
cfg.inflight = None
print("TEST 05: transport download with exact name and onfly_checksum")
try:
os.unlink("bbb")
except:
pass
msg.onfly_checksum = None
tr = sftp_transport()
tr.download(cfg)
if os.path.exists("bbb") and cfg.msg.onfly_checksum:
os.unlink("./bbb")
print("TEST 05: OK")
else:
print("TEST 05: failed")
failed = True
print("TEST 06: download file lock is .filename")
cfg.inflight = '.'
tr.download(cfg)
try:
os.unlink("./bbb")
print("TEST 06: OK")
except:
print("TEST 06: FAILED, file not found")
failed = True
print("TEST 07: download file lock is filename.tmp")
cfg.inflight = '.tmp'
tr.download(cfg)
try:
os.unlink("./bbb")
print("TEST 07: OK")
except:
print("TEST 07: FAILED, file not found")
failed = True
# download the file... it is sent below
tr.download(cfg)
# closing way too much
tr.close()
tr.close()
tr.close()
# configure sending
tr = sftp_transport()
cfg.local_file = "bbb"
msg.relpath = "./bbb"
msg.new_dir = "tztz"
msg.new_file = "ddd"
cfg.remote_file = "ddd"
cfg.remote_path = "tztz/ddd"
cfg.remote_urlstr = sftp_url + "/tztz/ddd"
cfg.remote_dir = "tztz"
cfg.chmod = 0o775
print("TEST 08: send file with exact name")
cfg.inflight = None
tr.send(cfg)
if os.path.exists(os.path.expanduser("~/tztz/ddd")):
print("TEST 08: OK")
else:
print("TEST 08: FAILED, file not found")
failed = True
print("TEST 09: delete remote file")
sftp = tr.proto
sftp.delete("ddd")
if not os.path.exists("~/tztz/ddd"):
print("TEST 09: OK")
else:
print("TEST 09: FAILED, file not found")
failed = True
# deleting a non existing file
print("TEST 10: delete not existing file")
try:
sftp.delete("zzz_unexistant")
print("TEST 10: OK")
except:
print("TEST 10: FAILED")
failed = True
# testing several sending options
print("TEST 11: sending file lock = .filename")
cfg.inflight = '.'
tr.send(cfg)
try:
sftp.delete("ddd")
print("TEST 11: OK")
except:
print("TEST 11: FAILED")
failed = True
print("TEST 12: sending file lock = filename.tmp")
cfg.inflight = '.tmp'
tr.send(cfg)
try:
sftp.delete("ddd")
print("TEST 12: OK")
except:
print("TEST 12: FAILED")
failed = True
# testing a number of sends
print("TEST 13: numerous send for the same file")
tr.send(cfg)
tr.send(cfg)
tr.send(cfg)
tr.close()
print("TEST 13: OK")
# do cleanup from previous tests
sftp = sr_sftp(cfg)
sftp.connect()
sftp.cd("tztz")
sftp.ls()
sftp.delete("ccc")
sftp.delete("ddd")
print("%s" % sftp.originalDir)
sftp.cd("")
print("%s" % sftp.getcwd())
sftp.rmdir("tztz")
sftp.close()
pwd = os.getcwd()
# as sftp do part stuff back and fore
print("TEST 14: testing parts get/put")
sftp = sr_sftp(cfg)
sftp.connect()
sftp.cd(pwd)
sftp.set_sumalgo(cfg.sumalgo)
sftp.put("aaa", "bbb", 0, 0, 2)
sftp.get("aaa", "bbb", 2, 2, 2)
sftp.put("aaa", "bbb", 4, 4, 2)
f = open("bbb", "rb")
data = f.read()
f.close()
if data == b"1\n2\n3\n":
print("TEST 14: OK")
else:
print("TEST 14: FAILED")
failed = True
sftp.delete("bbb")
sftp.delete("aaa")
sftp.close()
if not failed:
print("sr_sftp: TEST PASSED")
else:
print("sr_sftp: TEST FAILED")
sys.exit(1)
def self_test():
logger = test_logger()
# config setup
cfg = sr_config()
cfg.defaults()
cfg.general()
cfg.debug = True
opt1 = "destination ftp://localhost"
cfg.option( opt1.split() )
cfg.logger = logger
cfg.timeout = 5
try:
ftp = sr_ftp(cfg)
ftp.connect()
ftp.mkdir("tztz")
ftp.chmod(775,"tztz")
ftp.cd("tztz")
ftp.umask()
f = open("aaa","wb")
f.write(b"1\n")
f.write(b"2\n")
f.write(b"3\n")
f.close()
ftp.put("aaa", "bbb")
ls = ftp.ls()
logger.info("ls = %s" % ls )
ftp.chmod(775,"bbb")
ls = ftp.ls()
logger.info("ls = %s" % ls )
ftp.rename("bbb", "ccc")
ls = ftp.ls()
logger.info("ls = %s" % ls )
ftp.get("ccc", "bbb")
f = open("bbb","rb")
data = f.read()
f.close()
ftp.close()
if data != b"1\n2\n3\n" :
logger.error("sr_ftp TEST FAILED")
sys.exit(1)
os.unlink("bbb")
msg = sr_message(cfg)
msg.start_timer()
msg.topic = "v02.post.test"
msg.notice = "notice"
msg.urlcred = "ftp://localhost/"
msg.urlstr = "ftp://localhost/tztz/ccc"
msg.url = urllib.parse.urlparse(msg.urlcred+"tztz/ccc")
msg.partflg = '1'
msg.offset = 0
msg.length = 0
msg.sumalgo = None
msg.local_file = "bbb"
msg.local_offset = 0
cfg.msg = msg
cfg.batch = 5
cfg.kbytes_ps= 0.05
dldr = ftp_transport()
cfg.lock = None
dldr.download(cfg)
dldr.download(cfg)
cfg.lock = '.'
dldr.download(cfg)
dldr.download(cfg)
logger.debug("checksum = %s" % msg.onfly_checksum)
dldr.download(cfg)
cfg.lock = '.tmp'
dldr.download(cfg)
dldr.download(cfg)
msg.sumalgo = cfg.sumalgo
dldr.download(cfg)
logger.debug("checksum = %s" % msg.onfly_checksum)
logger.debug("change context")
ok, details = cfg.credentials.get(cfg.destination)
details.binary = False
cfg.credentials.credentials[cfg.destination] = details
dldr.download(cfg)
logger.debug("checksum = %s" % msg.onfly_checksum)
dldr.close()
dldr.close()
dldr.close()
dldr = ftp_transport()
cfg.local_file = "bbb"
cfg.local_path = "./bbb"
cfg.remote_file = "ddd"
cfg.remote_path = "tztz/ddd"
cfg.remote_urlstr = "ftp://localhost/tztz/ddd"
cfg.remote_dir = "tztz"
cfg.chmod = 775
cfg.lock = None
dldr.send(cfg)
dldr.ftp.delete("ddd")
cfg.lock = '.'
dldr.send(cfg)
dldr.ftp.delete("ddd")
cfg.lock = '.tmp'
dldr.send(cfg)
dldr.send(cfg)
dldr.send(cfg)
logger.debug("change context back")
ok, details = cfg.credentials.get(cfg.destination)
details.binary = True
cfg.credentials.credentials[cfg.destination] = details
dldr.send(cfg)
dldr.send(cfg)
dldr.send(cfg)
dldr.close()
dldr.close()
dldr.close()
ftp = sr_ftp(cfg)
ftp.connect()
ftp.cd("tztz")
ftp.delete("ccc")
ftp.delete("ddd")
logger.info("%s" % ftp.originalDir)
ftp.cd("")
logger.info("%s" % ftp.ftp.pwd())
ftp.rmdir("tztz")
ftp.close()
except:
logger.error("sr_ftp TEST FAILED")
sys.exit(2)
os.unlink('aaa')
os.unlink('bbb')
print("sr_ftp TEST PASSED")
sys.exit(0)
def self_test():
logger = test_logger()
# config setup
cfg = sr_config()
cfg.defaults()
cfg.general()
cfg.debug = True
opt1 = "destination ftp://localhost"
cfg.option(opt1.split())
cfg.logger = logger
cfg.timeout = 5
try:
ftp = sr_ftp(cfg)
ftp.connect()
ftp.mkdir("tztz")
ftp.chmod(775, "tztz")
ftp.cd("tztz")
ftp.umask()
f = open("aaa", "wb")
f.write(b"1\n")
f.write(b"2\n")
f.write(b"3\n")
f.close()
ftp.put("aaa", "bbb")
ls = ftp.ls()
logger.info("ls = %s" % ls)
ftp.chmod(775, "bbb")
ls = ftp.ls()
logger.info("ls = %s" % ls)
ftp.rename("bbb", "ccc")
ls = ftp.ls()
logger.info("ls = %s" % ls)
ftp.get("ccc", "bbb")
f = open("bbb", "rb")
data = f.read()
f.close()
ftp.close()
if data != b"1\n2\n3\n":
logger.error("sr_ftp TEST FAILED")
sys.exit(1)
os.unlink("bbb")
msg = sr_message(cfg)
msg.start_timer()
msg.topic = "v02.post.test"
msg.notice = "notice"
msg.urlcred = "ftp://localhost/"
msg.urlstr = "ftp://localhost/tztz/ccc"
msg.url = urllib.parse.urlparse(msg.urlcred + "tztz/ccc")
msg.partflg = '1'
msg.offset = 0
msg.length = 0
msg.sumalgo = None
msg.local_file = "bbb"
msg.local_offset = 0
cfg.msg = msg
cfg.batch = 5
cfg.kbytes_ps = 0.05
dldr = ftp_transport()
cfg.lock = None
dldr.download(cfg)
dldr.download(cfg)
cfg.lock = '.'
dldr.download(cfg)
dldr.download(cfg)
logger.debug("checksum = %s" % msg.onfly_checksum)
dldr.download(cfg)
cfg.lock = '.tmp'
dldr.download(cfg)
dldr.download(cfg)
msg.sumalgo = cfg.sumalgo
dldr.download(cfg)
logger.debug("checksum = %s" % msg.onfly_checksum)
logger.debug("change context")
ok, details = cfg.credentials.get(cfg.destination)
details.binary = False
cfg.credentials.credentials[cfg.destination] = details
dldr.download(cfg)
logger.debug("checksum = %s" % msg.onfly_checksum)
dldr.close()
dldr.close()
dldr.close()
dldr = ftp_transport()
cfg.local_file = "bbb"
cfg.local_path = "./bbb"
cfg.remote_file = "ddd"
cfg.remote_path = "tztz/ddd"
cfg.remote_urlstr = "ftp://localhost/tztz/ddd"
cfg.remote_dir = "tztz"
cfg.chmod = 775
cfg.lock = None
dldr.send(cfg)
dldr.ftp.delete("ddd")
cfg.lock = '.'
dldr.send(cfg)
dldr.ftp.delete("ddd")
cfg.lock = '.tmp'
dldr.send(cfg)
dldr.send(cfg)
dldr.send(cfg)
logger.debug("change context back")
ok, details = cfg.credentials.get(cfg.destination)
details.binary = True
cfg.credentials.credentials[cfg.destination] = details
dldr.send(cfg)
dldr.send(cfg)
dldr.send(cfg)
dldr.close()
dldr.close()
dldr.close()
ftp = sr_ftp(cfg)
ftp.connect()
ftp.cd("tztz")
ftp.delete("ccc")
ftp.delete("ddd")
logger.info("%s" % ftp.originalDir)
ftp.cd("")
logger.info("%s" % ftp.ftp.pwd())
ftp.rmdir("tztz")
ftp.close()
except:
logger.error("sr_ftp TEST FAILED")
sys.exit(2)
os.unlink('aaa')
os.unlink('bbb')
print("sr_ftp TEST PASSED")
sys.exit(0)
def self_test():
logger = test_logger()
# config setup
cfg = sr_config()
cfg.defaults()
cfg.general()
#cfg.debug = True
opt1 = "destination sftp://localhost"
cfg.option(opt1.split())
cfg.logger = logger
cfg.timeout = 5.0
# 1 bytes par 5 secs
#cfg.kbytes_ps = 0.0001
cfg.kbytes_ps = 0.01
try:
sftp = sr_sftp(cfg)
sftp.connect()
sftp.mkdir("tztz")
sftp.chmod(775, "tztz")
sftp.cd("tztz")
f = open("aaa", "wb")
f.write(b"1\n")
f.write(b"2\n")
f.write(b"3\n")
f.close()
sftp.put("aaa", "bbb")
ls = sftp.ls()
logger.info("ls = %s" % ls)
sftp.chmod(775, "bbb")
ls = sftp.ls()
logger.info("ls = %s" % ls)
sftp.rename("bbb", "ccc")
ls = sftp.ls()
logger.info("ls = %s" % ls)
sftp.get("ccc", "bbb", 0, 0, 6)
f = open("bbb", "rb")
data = f.read()
f.close()
if data != b"1\n2\n3\n":
logger.error("sr_sftp1 TEST FAILED")
sys.exit(1)
os.unlink("bbb")
msg = sr_message(cfg)
msg.start_timer()
msg.topic = "v02.post.test"
msg.notice = "notice"
msg.urlcred = "sftp://localhost/"
msg.urlstr = "sftp://localhost/tztz/ccc"
msg.url = urllib.parse.urlparse(msg.urlcred + "tztz/ccc")
msg.partflg = '1'
msg.offset = 0
msg.length = 0
msg.local_file = "bbb"
msg.local_offset = 0
msg.sumalgo = None
cfg.msg = msg
cfg.batch = 5
cfg.lock = None
dldr = sftp_transport()
dldr.download(cfg)
logger.debug("checksum = %s" % msg.onfly_checksum)
dldr.download(cfg)
dldr.download(cfg)
cfg.logger.info("lock .")
cfg.lock = '.'
dldr.download(cfg)
dldr.download(cfg)
msg.sumalgo = cfg.sumalgo
dldr.download(cfg)
logger.debug("checksum = %s" % msg.onfly_checksum)
cfg.logger.info("lock .tmp")
cfg.lock = '.tmp'
dldr.download(cfg)
dldr.download(cfg)
dldr.close()
dldr.close()
dldr.close()
dldr = sftp_transport()
cfg.local_file = "bbb"
cfg.local_path = "./bbb"
cfg.remote_file = "ddd"
cfg.remote_path = "tztz/ddd"
cfg.remote_urlstr = "sftp://localhost/tztz/ddd"
cfg.remote_dir = "tztz"
cfg.chmod = 775
cfg.lock = None
dldr.send(cfg)
dldr.sftp.delete("ddd")
cfg.lock = '.'
dldr.send(cfg)
dldr.sftp.delete("ddd")
cfg.lock = '.tmp'
dldr.send(cfg)
dldr.send(cfg)
dldr.send(cfg)
dldr.send(cfg)
dldr.send(cfg)
dldr.send(cfg)
dldr.close()
dldr.close()
dldr.close()
sftp = sr_sftp(cfg)
sftp.connect()
sftp.cd("tztz")
sftp.delete("ccc")
sftp.delete("ddd")
logger.info("%s" % sftp.originalDir)
sftp.cd("")
logger.info("%s" % sftp.sftp.getcwd())
sftp.rmdir("tztz")
sftp.put("aaa", "bbb", 0, 0, 2)
sftp.put("aaa", "bbb", 2, 4, 2)
sftp.put("aaa", "bbb", 4, 2, 2)
sftp.get("bbb", "bbb", 2, 2, 2)
sftp.delete("bbb")
f = open("bbb", "rb")
data = f.read()
f.close()
if data != b"1\n3\n3\n":
logger.error("sr_sftp TEST FAILED ")
sys.exit(1)
sftp.close()
except:
(stype, svalue, tb) = sys.exc_info()
logger.error("(Type: %s, Value: %s)" % (stype, svalue))
logger.error("sr_sftp TEST FAILED")
sys.exit(2)
os.unlink('aaa')
os.unlink('bbb')
print("sr_sftp TEST PASSED")
sys.exit(0)
def self_test():
logger = test_logger()
# config setup
cfg = sr_config()
cfg.defaults()
cfg.general()
#cfg.debug = True
opt1 = "destination sftp://localhost"
cfg.option( opt1.split() )
cfg.logger = logger
cfg.timeout = 5.0
# 1 bytes par 5 secs
#cfg.kbytes_ps = 0.0001
cfg.kbytes_ps = 0.01
try:
sftp = sr_sftp(cfg)
sftp.connect()
sftp.mkdir("tztz")
sftp.chmod(775,"tztz")
sftp.cd("tztz")
f = open("aaa","wb")
f.write(b"1\n")
f.write(b"2\n")
f.write(b"3\n")
f.close()
sftp.put("aaa", "bbb")
ls = sftp.ls()
logger.info("ls = %s" % ls )
sftp.chmod(775,"bbb")
ls = sftp.ls()
logger.info("ls = %s" % ls )
sftp.rename("bbb", "ccc")
ls = sftp.ls()
logger.info("ls = %s" % ls )
sftp.get("ccc", "bbb",0,0,6)
f = open("bbb","rb")
data = f.read()
f.close()
if data != b"1\n2\n3\n" :
logger.error("sr_sftp1 TEST FAILED")
sys.exit(1)
os.unlink("bbb")
msg = sr_message(cfg)
msg.start_timer()
msg.topic = "v02.post.test"
msg.notice = "notice"
msg.urlcred = "sftp://localhost/"
msg.urlstr = "sftp://localhost/tztz/ccc"
msg.url = urllib.parse.urlparse(msg.urlcred+"tztz/ccc")
msg.partflg = '1'
msg.offset = 0
msg.length = 0
msg.local_file = "bbb"
msg.local_offset = 0
msg.sumalgo = None
cfg.msg = msg
cfg.batch = 5
cfg.lock = None
dldr = sftp_transport()
dldr.download(cfg)
logger.debug("checksum = %s" % msg.onfly_checksum)
dldr.download(cfg)
dldr.download(cfg)
cfg.logger.info("lock .")
cfg.lock = '.'
dldr.download(cfg)
dldr.download(cfg)
msg.sumalgo = cfg.sumalgo
dldr.download(cfg)
logger.debug("checksum = %s" % msg.onfly_checksum)
cfg.logger.info("lock .tmp")
cfg.lock = '.tmp'
dldr.download(cfg)
dldr.download(cfg)
dldr.close()
dldr.close()
dldr.close()
dldr = sftp_transport()
cfg.local_file = "bbb"
cfg.local_path = "./bbb"
cfg.remote_file = "ddd"
cfg.remote_path = "tztz/ddd"
cfg.remote_urlstr = "sftp://localhost/tztz/ddd"
cfg.remote_dir = "tztz"
cfg.chmod = 775
cfg.lock = None
dldr.send(cfg)
dldr.sftp.delete("ddd")
cfg.lock = '.'
dldr.send(cfg)
dldr.sftp.delete("ddd")
cfg.lock = '.tmp'
dldr.send(cfg)
dldr.send(cfg)
dldr.send(cfg)
dldr.send(cfg)
dldr.send(cfg)
dldr.send(cfg)
dldr.close()
dldr.close()
dldr.close()
sftp = sr_sftp(cfg)
sftp.connect()
sftp.cd("tztz")
sftp.delete("ccc")
sftp.delete("ddd")
logger.info("%s" % sftp.originalDir)
sftp.cd("")
logger.info("%s" % sftp.sftp.getcwd())
sftp.rmdir("tztz")
sftp.put("aaa","bbb",0,0,2)
sftp.put("aaa","bbb",2,4,2)
sftp.put("aaa","bbb",4,2,2)
sftp.get("bbb","bbb",2,2,2)
sftp.delete("bbb")
f = open("bbb","rb")
data = f.read()
f.close()
if data != b"1\n3\n3\n" :
logger.error("sr_sftp TEST FAILED ")
sys.exit(1)
sftp.close()
except:
(stype, svalue, tb) = sys.exc_info()
logger.error("(Type: %s, Value: %s)" % (stype ,svalue))
logger.error("sr_sftp TEST FAILED")
sys.exit(2)
os.unlink('aaa')
os.unlink('bbb')
print("sr_sftp TEST PASSED")
sys.exit(0)
def self_test():
failed = False
logger = test_logger()
logger.info("sr_ftp : BEGIN TEST\n")
# configuration
cfg = sr_config()
cfg.logger = logger
cfg.configure()
cfg.set_sumalgo('d')
msg = sr_message(cfg)
msg.filesize = None
msg.onfly_checksum = False
# 1 bytes par 5 secs
#cfg.kbytes_ps = 0.0001
cfg.timeout = 5.0
cfg.kbytes_ps = 0.01
cfg.chmod_dir = 0o775
cfg.msg = msg
opt1 = "destination ftp://aspymjg@localhost"
cfg.option(opt1.split())
# make sure test directory is removed before test startup
ftp_url = cfg.destination
ftpuser = ftp_url.split('/')[2].split('@')[0]
testdir = os.path.expanduser('~' + ftpuser) + '/tztz'
try:
shutil.rmtree(testdir)
except:
pass
logger.info("TEST 01: instantiation and connection")
ftp = sr_ftp(cfg)
ftp.connect()
if ftp.check_is_connected():
logger.info("TEST 01: OK")
else:
logger.info("TEST 01: failed ... not connected")
failed = True
logger.info("TEST 02: mkdir,chmod,cd")
ftp.mkdir("tztz")
ls = ftp.ls()
if "tztz" in ls:
ftp.chmod(0o775, "tztz")
ftp.cd("tztz")
logger.info("TEST 02: OK")
else:
logger.info("TEST 02: failed")
failed = True
logger.info(
"TEST 03: put local aaa to remote bbb, chmod bbb, rename bbb ccc")
f = open("aaa", "wb")
f.write(b"1\n")
f.write(b"2\n")
f.write(b"3\n")
f.close()
ftp.put("aaa", "bbb")
ls = ftp.ls()
if "bbb" in ls:
ftp.chmod(0o775, "bbb")
ftp.rename("bbb", "ccc")
ls = ftp.ls()
if "ccc" in ls:
logger.info("TEST 03: OK")
else:
logger.info("TEST 03: failed")
failed = True
# build a message
msg.start_timer()
msg.topic = "v02.post.test"
msg.notice = "notice"
msg.baseurl = ftp_url
msg.relpath = "tztz/ccc"
msg.partflg = '1'
msg.offset = 0
msg.length = 6
msg.local_file = "bbb"
msg.local_offset = 0
cfg.set_sumalgo('d')
msg.sumalgo = cfg.sumalgo
msg.logger = logger
msg.new_file = "bbb"
msg.new_dir = "."
cfg.msg = msg
cfg.batch = 5
cfg.inflight = None
logger.info(
"TEST 04: transport download with exact name and onfly_checksum")
try:
os.unlink("bbb")
except:
pass
msg.onfly_checksum = None
tr = ftp_transport()
tr.download(cfg)
if os.path.exists("bbb") and cfg.msg.onfly_checksum:
os.unlink("./bbb")
logger.info("TEST 04: OK")
else:
logger.info("TEST 04: failed")
failed = True
logger.info("TEST 05: download file lock is .filename")
cfg.inflight = '.'
tr.download(cfg)
try:
os.unlink("./bbb")
logger.info("TEST 05: OK")
except:
logger.info("TEST 05: FAILED, file not found")
failed = True
logger.info("TEST 06: download file lock is filename.tmp")
cfg.inflight = '.tmp'
tr.download(cfg)
try:
os.unlink("./bbb")
logger.info("TEST 06: OK")
except:
logger.info("TEST 06: FAILED, file not found")
failed = True
# download the file... it is sent below
tr.download(cfg)
# closing way too much
tr.close()
tr.close()
tr.close()
tr = None
# configure sending
tr = ftp_transport()
cfg.local_file = "bbb"
msg.relpath = "./bbb"
msg.new_dir = "tztz"
msg.new_file = "ddd"
cfg.remote_file = "ddd"
cfg.remote_path = "tztz/ddd"
cfg.remote_urlstr = ftp_url + "/tztz/ddd"
cfg.remote_dir = "tztz"
cfg.chmod = 0o775
logger.info("TEST 07: send file with exact name")
cfg.inflight = None
tr.send(cfg)
if os.path.exists(testdir + os.sep + 'ddd'):
logger.info("TEST 07: OK")
else:
logger.info("TEST 07: FAILED, file not found")
failed = True
logger.debug = logger.silence
logger.info("TEST 08: delete remote file")
ftp = tr.proto
ftp.delete("ddd")
if not os.path.exists("~/tztz/ddd"):
logger.info("TEST 08: OK")
else:
logger.info("TEST 08: FAILED, file not found")
failed = True
# deleting a non existing file
logger.info("TEST 09: delete not existing file")
try:
ftp.delete("zzz_unexistant")
logger.info("TEST 09: OK")
except:
logger.info("TEST 09: FAILED")
failed = True
# testing several sending options
logger.info("TEST 10: sending file lock = .filename")
cfg.inflight = '.'
tr.send(cfg)
try:
ftp.delete("ddd")
logger.info("TEST 10: OK")
except:
logger.info("TEST 10: FAILED")
failed = True
logger.info("TEST 11: sending file lock = filename.tmp")
cfg.inflight = '.tmp'
tr.send(cfg)
try:
ftp.delete("ddd")
logger.info("TEST 11: OK")
except:
logger.info("TEST 11: FAILED")
failed = True
# testing a number of sends
logger.info("TEST 12: numerous send for the same file")
tr.send(cfg)
tr.send(cfg)
tr.send(cfg)
tr.close()
logger.info("TEST 12: OK")
# do cleanup from previous tests
ftp = sr_ftp(cfg)
ftp.connect()
ftp.cd("tztz")
ftp.ls()
ftp.delete("ccc")
ftp.delete("ddd")
ftp.cd("")
ftp.rmdir("tztz")
ftp.close()
logger.info("")
if not failed:
logger.info("sr_ftp: TEST PASSED")
else:
logger.info("sr_ftp: TEST FAILED")
sys.exit(1)
