def connect(self): # ============= # create message # ============= self.msg = sr_message(self) # ============= # consumer # ============= self.consumer = sr_consumer(self) self.msg.report_publisher = self.consumer.publish_back() self.msg.report_exchange = self.report_exchange self.msg.user = self.details.url.username self.msg.host = self.details.url.scheme + '://' + self.details.url.hostname self.msg.post_exchange_split = self.post_exchange_split # ============= # poster # ============= if self.post_broker: self.poster = sr_poster(self) self.msg.publisher = self.poster.publisher self.msg.pub_exchange = self.post_exchange self.msg.user = self.post_broker.username self.connected = True
def connect(self): # ============= # create message if needed # ============= self.msg = sr_message(self) # ============= # consumer queue_name : let consumer takes care of it # ============= self.consumer = sr_consumer(self) # ============= # publisher... (publish back to consumer) # ============= self.publisher = self.consumer.publish_back() # ============= # setup message publisher # ============= self.msg.publisher = self.consumer.publisher self.msg.post_exchange_split = self.post_exchange_split
def connect(self): # ============= # create message # ============= self.msg = sr_message(self) # ============= # consumer queue_name : let consumer takes care of it # ============= self.consumer = sr_consumer(self) # ============= # publisher... on remote cluster # ============= self.hc = HostConnect(logger=self.logger) self.hc.set_url(self.cluster_broker) self.hc.connect() self.publisher = Publisher(self.hc) self.publisher.build() # ============= # setup message publisher # ============= self.msg.publisher = self.publisher self.msg.post_exchange_split = self.post_exchange_split
def get_message(self): if not hasattr(self.parent, 'msg'): self.parent.msg = sr_message(self.parent) self.raw_msg = None self.msg = self.parent.msg self.msg.user = self.broker.username
def connect(self): # ============= # create message # ============= self.msg = sr_message(self) # ============= # consumer queue_name : let consumer takes care of it # ============= self.consumer = sr_consumer(self) # ============= # publisher... on remote cluster # ============= self.hc = HostConnect( logger = self.logger ) self.hc.set_url( self.cluster_broker ) self.hc.connect() self.publisher = Publisher(self.hc) self.publisher.build() # ============= # setup message publisher # ============= self.msg.publisher = self.publisher self.msg.post_exchange_split = self.post_exchange_split
def connect(self): # ============= # create message # ============= self.msg = sr_message(self) # ============= # consumer # ============= self.consumer = sr_consumer(self) self.msg.report_publisher = self.consumer.publish_back() self.msg.report_exchange = self.report_exchange self.msg.user = self.details.url.username self.msg.host = self.details.url.scheme + '://' + self.details.url.hostname self.msg.post_exchange_split = self.post_exchange_split # ============= # poster # ============= if self.post_broker : self.poster = sr_poster(self) self.msg.publisher = self.poster.publisher self.msg.pub_exchange = self.post_exchange self.msg.user = self.post_broker.username self.connected = True
def connect(self): self.logger.debug("%s connect" % self.program_name) # ============= # create message if needed # ============= self.msg = sr_message(self) # ============= # posting # ============= loop = True #20200404 - PS looping for connection has nothing to do with looping for watch. # self.sleep says whether watch should loop. # loop is about whether we try over and over to connect... two are not related. # that's why following line is commented. #if self.sleep <= 0 : loop = False self.post_hc = HostConnect(logger=self.logger) self.post_hc.choose_amqp_alternative(self.use_amqplib, self.use_pika) self.post_hc.set_url(self.post_broker) self.post_hc.loop = loop if not self.post_hc.connect(): return self.publisher = Publisher(self.post_hc) self.publisher.build() self.logger.info("Output AMQP broker(%s) user(%s) vhost(%s)" % \ (self.post_broker.hostname,self.post_broker.username,self.post_broker.path) ) # ============= # setup message publish # ============= self.msg.user = self.post_broker.username self.msg.publisher = self.publisher self.msg.pub_exchange = self.post_exchange self.msg.post_exchange_split = self.post_exchange_split self.logger.info("Output AMQP exchange(%s)" % self.msg.pub_exchange) # ============= # amqp resources # ============= self.declare_exchanges() # ============= # retransmit/restore_queue # ============= if self.restore_queue != None: self.publisher.restore_set(self) self.msg.pub_exchange = self.publisher.restore_exchange self.msg.post_exchange_split = 0
def get_message(self): self.logger.debug("sr_consumer get_message") if not hasattr(self.parent,'msg'): self.parent.msg = sr_message(self.parent) self.raw_msg = None self.msg = self.parent.msg self.msg.user = self.broker.username
def get_message(self): #self.logger.debug("sr_consumer get_message") if not hasattr(self.parent, 'msg'): self.parent.msg = sr_message(self.parent) self.raw_msg = None self.msg = self.parent.msg self.msg.user = self.broker.username
def get_message(self): self.logger.debug("sr_poster get_message") if not hasattr(self.parent, 'msg'): self.parent.msg = sr_message(self.parent) self.msg = self.parent.msg self.msg.user = self.broker.username self.msg.publisher = self.publisher
def get_message(self): self.logger.debug("sr_poster get_message") if not hasattr(self.parent,'msg'): self.parent.msg = sr_message(self.parent) self.msg = self.parent.msg self.msg.user = self.broker.username self.msg.publisher = self.publisher
def connect(self): self.logger.debug("%s connect" % self.program_name) # ============= # create message if needed # ============= self.msg = sr_message(self) # ============= # posting # ============= loop = True if self.sleep <= 0: loop = False self.post_hc = HostConnect(logger=self.logger) self.post_hc.choose_amqp_alternative(self.use_amqplib, self.use_pika) self.post_hc.set_url(self.post_broker) self.post_hc.loop = loop if not self.post_hc.connect(): return self.publisher = Publisher(self.post_hc) self.publisher.build() self.logger.info("Output AMQP broker(%s) user(%s) vhost(%s)" % \ (self.post_broker.hostname,self.post_broker.username,self.post_broker.path) ) # ============= # setup message publish # ============= self.msg.user = self.post_broker.username self.msg.publisher = self.publisher self.msg.pub_exchange = self.post_exchange self.msg.post_exchange_split = self.post_exchange_split self.logger.info("Output AMQP exchange(%s)" % self.msg.pub_exchange) # ============= # amqp resources # ============= self.declare_exchanges() # ============= # retransmit/restore_queue # ============= if self.restore_queue != None: self.publisher.restore_set(self) self.msg.pub_exchange = self.publisher.restore_exchange self.msg.post_exchange_split = 0
def connect(self): # ============= # create message # ============= self.msg = sr_message(self) # ============= # consumer # ============= self.consumer = sr_consumer(self) self.msg.report_publisher = self.consumer.publish_back() self.msg.report_exchange = self.report_exchange self.msg.user = self.broker.username self.logger.info( "reading from to %s@%s, exchange: %s" % (self.broker.username, self.broker.hostname, self.msg.exchange)) self.logger.info("report back to %s@%s, exchange: %s" % (self.broker.username, self.broker.hostname, self.msg.report_exchange)) # ============= # poster if post_broker different from broker # ============= self.poster = None if self.post_broker.geturl() != self.broker.geturl(): self.poster = sr_poster(self) self.msg.publisher = self.poster.publisher # ============= # publisher if post_broker is same as broker # ============= else: self.publisher = Publisher(self.consumer.hc) self.publisher.build() self.msg.publisher = self.publisher self.logger.info("Output AMQP broker(%s) user(%s) vhost(%s)" % \ (self.broker.hostname,self.broker.username,self.broker.path) ) self.msg.pub_exchange = self.post_exchange self.msg.post_exchange_split = self.post_exchange_split self.logger.info("Output AMQP exchange(%s)" % self.msg.pub_exchange)
def connect(self): # ============= # create message # ============= self.msg = sr_message(self) # ============= # consumer # ============= self.consumer = sr_consumer(self) self.msg.report_publisher = self.consumer.publish_back() self.msg.report_exchange = self.report_exchange self.msg.user = self.broker.username self.logger.info("reading from to %s@%s, exchange: %s" % ( self.broker.username, self.broker.hostname, self.msg.exchange ) ) self.logger.info("report back to %s@%s, exchange: %s" % ( self.broker.username, self.broker.hostname, self.msg.report_exchange ) ) # ============= # poster if post_broker different from broker # ============= self.poster = None if self.post_broker.geturl() != self.broker.geturl() : self.poster = sr_poster(self) self.msg.publisher = self.poster.publisher # ============= # publisher if post_broker is same as broker # ============= else : self.publisher = Publisher(self.consumer.hc) self.publisher.build() self.msg.publisher = self.publisher self.logger.info("Output AMQP broker(%s) user(%s) vhost(%s)" % \ (self.broker.hostname,self.broker.username,self.broker.path) ) self.msg.pub_exchange = self.post_exchange self.msg.post_exchange_split = self.post_exchange_split self.logger.info("Output AMQP exchange(%s)" % self.msg.pub_exchange )
def connect(self): self.logger.debug("sr_post connect") # sr_post : no loop to reconnect to broker self.loop = True if self.program_name == 'sr_post' : self.loop = False # message self.msg = sr_message( self ) # publisher self.post_broker = self.broker self.poster = sr_poster(self, self.loop) self.msg.publisher = self.poster.publisher self.msg.post_exchange_split = self.post_exchange_split
def connect(self): self.logger.debug("sr_post connect") # sr_post : no loop to reconnect to broker self.loop = True if self.program_name == 'sr_post': self.loop = False # message self.msg = sr_message(self) # publisher self.post_broker = self.broker self.poster = sr_poster(self, self.loop) self.msg.publisher = self.poster.publisher self.msg.post_exchange_split = self.post_exchange_split
def connect(self): # ============= # create message # ============= self.msg = sr_message(self) # ============= # poster # ============= self.post_broker = self.broker self.post_exchange = self.exchange self.poster = sr_poster(self) self.msg.publisher = self.poster.publisher self.msg.pub_exchange = self.post_exchange self.connected = True
def connect(self): # ============= # create message if needed # ============= self.msg = sr_message(self) # ============= # consumer # ============= self.consumer = sr_consumer(self) if self.reportback: self.msg.report_publisher = self.consumer.publish_back() self.msg.report_exchange = self.report_exchange self.logger.info("report_back to %s@%s, exchange: %s" % (self.broker.username, self.broker.hostname, self.msg.report_exchange)) else: self.logger.info("report_back suppressed") # ============= # publisher # ============= # publisher host self.hc_pst = HostConnect(logger=self.logger) self.hc_pst.set_url(self.post_broker) self.hc_pst.connect() # publisher self.publisher = Publisher(self.hc_pst) self.publisher.build() self.msg.publisher = self.publisher self.msg.post_exchange_split = self.post_exchange_split
def connect(self): # ============= # create message # ============= self.msg = sr_message(self) # ============= # poster # ============= self.post_broker = self.broker self.post_exchange = self.exchange self.poster = sr_poster(self) self.msg.publisher = self.poster.publisher self.msg.pub_exchange = self.post_exchange self.msg.post_exchange_split = self.post_exchange_split self.connected = True
def connect(self): # ============= # create message if needed # ============= self.msg = sr_message(self) # ============= # consumer # ============= self.consumer = sr_consumer(self) if self.reportback : self.msg.report_publisher = self.consumer.publish_back() self.msg.report_exchange = self.report_exchange self.logger.info("reportback to %s@%s, exchange: %s" % ( self.broker.username, self.broker.hostname, self.msg.report_exchange ) ) else: self.logger.info( "reportback suppressed" ) # ============= # publisher # ============= # publisher host self.hc_pst = HostConnect( logger = self.logger ) self.hc_pst.set_url( self.post_broker ) self.hc_pst.connect() # publisher self.publisher = Publisher(self.hc_pst) self.publisher.build() self.msg.publisher = self.publisher self.msg.post_exchange_split = self.post_exchange_split
def self_test(): failed = False # configuration cfg = sr_config() cfg.configure() cfg.set_sumalgo('d') print("sr_sftp: BEGIN TEST\n") msg = sr_message(cfg) msg.filesize = None msg.lastchunk = True msg.onfly_checksum = False # 1 bytes par 5 secs #cfg.kbytes_ps = 0.0001 cfg.timeout = 5.0 cfg.kbytes_ps = 0.01 cfg.chmod_dir = 0o775 cfg.msg = msg opt1 = "destination sftp://${SFTPUSER}@localhost" cfg.option(opt1.split()) # make sure test directory is removed before test startup sftp_url = cfg.destination sftpuser = sftp_url.split('/')[2].split('@')[0] testdir = os.path.expanduser('~' + sftpuser) + '/tztz' try: shutil.rmtree(testdir) except: pass print("TEST 01: instantiation and connection") sftp = sr_sftp(cfg) sftp.connect() if sftp.check_is_connected(): print("TEST 01: OK") else: print("TEST 01: failed ... not connected") failed = True print("TEST 02: mkdir,rmdir,chmod,cd") sftp.mkdir("txtx") ls = sftp.ls() if "txtx" in ls: sftp.chmod(0o775, "txtx") sftp.delete("txtx") ls = sftp.ls() if "txtx" in ls: print("TEST 02: failed1") failed = True else: print("TEST 02: OK1") sftp.mkdir("tztz") ls = sftp.ls() if "tztz" in ls: sftp.chmod(0o775, "tztz") sftp.cd("tztz") print("TEST 02: OK2") else: print("TEST 02: failed2") failed = True print("TEST 03: put local aaa to remote bbb, chmod bbb, rename bbb ccc") f = open("aaa", "wb") f.write(b"1\n") f.write(b"2\n") f.write(b"3\n") f.close() sftp.put("aaa", "bbb") ls = sftp.ls() if "bbb" in ls: sftp.chmod(0o775, "bbb") sftp.rename("bbb", "ccc") ls = sftp.ls() if "ccc" in ls: print("TEST 03: OK") else: print("TEST 03: failed") failed = True print("TEST 04: get part of remote ccc into bbb") sftp.get("ccc", "bbb", 1, 0, 4) f = open("bbb", "rb") data = f.read() f.close() if data == b"\n2\n3": print("TEST 04: OK") else: print("TEST 04: failed") failed = True # build a message msg.start_timer() msg.topic = "v02.post.test" msg.notice = "notice" msg.pubtime = "20190109084420.788106" msg.baseurl = sftp_url msg.relpath = "tztz/ccc" msg.partflg = '1' msg.offset = 0 msg.length = 6 msg.local_file = "bbb" msg.local_offset = 0 cfg.set_sumalgo('d') msg.sumalgo = cfg.sumalgo msg.new_file = "bbb" msg.new_dir = "." cfg.msg = msg cfg.batch = 5 cfg.inflight = None print("TEST 05: transport download with exact name and onfly_checksum") try: os.unlink("bbb") except: pass msg.onfly_checksum = None tr = sftp_transport() tr.download(cfg) if os.path.exists("bbb") and cfg.msg.onfly_checksum: os.unlink("./bbb") print("TEST 05: OK") else: print("TEST 05: failed") failed = True print("TEST 06: download file lock is .filename") cfg.inflight = '.' tr.download(cfg) try: os.unlink("./bbb") print("TEST 06: OK") except: print("TEST 06: FAILED, file not found") failed = True print("TEST 07: download file lock is filename.tmp") cfg.inflight = '.tmp' tr.download(cfg) try: os.unlink("./bbb") print("TEST 07: OK") except: print("TEST 07: FAILED, file not found") failed = True # download the file... it is sent below tr.download(cfg) # closing way too much tr.close() tr.close() tr.close() # configure sending tr = sftp_transport() cfg.local_file = "bbb" msg.relpath = "./bbb" msg.new_dir = "tztz" msg.new_file = "ddd" cfg.remote_file = "ddd" cfg.remote_path = "tztz/ddd" cfg.remote_urlstr = sftp_url + "/tztz/ddd" cfg.remote_dir = "tztz" cfg.chmod = 0o775 print("TEST 08: send file with exact name") cfg.inflight = None tr.send(cfg) if os.path.exists(os.path.expanduser("~/tztz/ddd")): print("TEST 08: OK") else: print("TEST 08: FAILED, file not found") failed = True print("TEST 09: delete remote file") sftp = tr.proto sftp.delete("ddd") if not os.path.exists("~/tztz/ddd"): print("TEST 09: OK") else: print("TEST 09: FAILED, file not found") failed = True # deleting a non existing file print("TEST 10: delete not existing file") try: sftp.delete("zzz_unexistant") print("TEST 10: OK") except: print("TEST 10: FAILED") failed = True # testing several sending options print("TEST 11: sending file lock = .filename") cfg.inflight = '.' tr.send(cfg) try: sftp.delete("ddd") print("TEST 11: OK") except: print("TEST 11: FAILED") failed = True print("TEST 12: sending file lock = filename.tmp") cfg.inflight = '.tmp' tr.send(cfg) try: sftp.delete("ddd") print("TEST 12: OK") except: print("TEST 12: FAILED") failed = True # testing a number of sends print("TEST 13: numerous send for the same file") tr.send(cfg) tr.send(cfg) tr.send(cfg) tr.close() print("TEST 13: OK") # do cleanup from previous tests sftp = sr_sftp(cfg) sftp.connect() sftp.cd("tztz") sftp.ls() sftp.delete("ccc") sftp.delete("ddd") print("%s" % sftp.originalDir) sftp.cd("") print("%s" % sftp.getcwd()) sftp.rmdir("tztz") sftp.close() pwd = os.getcwd() # as sftp do part stuff back and fore print("TEST 14: testing parts get/put") sftp = sr_sftp(cfg) sftp.connect() sftp.cd(pwd) sftp.set_sumalgo(cfg.sumalgo) sftp.put("aaa", "bbb", 0, 0, 2) sftp.get("aaa", "bbb", 2, 2, 2) sftp.put("aaa", "bbb", 4, 4, 2) f = open("bbb", "rb") data = f.read() f.close() if data == b"1\n2\n3\n": print("TEST 14: OK") else: print("TEST 14: FAILED") failed = True sftp.delete("bbb") sftp.delete("aaa") sftp.close() if not failed: print("sr_sftp: TEST PASSED") else: print("sr_sftp: TEST FAILED") sys.exit(1)
def self_test(): logger = test_logger() # config setup cfg = sr_config() cfg.defaults() cfg.general() cfg.debug = True opt1 = "destination ftp://localhost" cfg.option( opt1.split() ) cfg.logger = logger cfg.timeout = 5 try: ftp = sr_ftp(cfg) ftp.connect() ftp.mkdir("tztz") ftp.chmod(775,"tztz") ftp.cd("tztz") ftp.umask() f = open("aaa","wb") f.write(b"1\n") f.write(b"2\n") f.write(b"3\n") f.close() ftp.put("aaa", "bbb") ls = ftp.ls() logger.info("ls = %s" % ls ) ftp.chmod(775,"bbb") ls = ftp.ls() logger.info("ls = %s" % ls ) ftp.rename("bbb", "ccc") ls = ftp.ls() logger.info("ls = %s" % ls ) ftp.get("ccc", "bbb") f = open("bbb","rb") data = f.read() f.close() ftp.close() if data != b"1\n2\n3\n" : logger.error("sr_ftp TEST FAILED") sys.exit(1) os.unlink("bbb") msg = sr_message(cfg) msg.start_timer() msg.topic = "v02.post.test" msg.notice = "notice" msg.urlcred = "ftp://localhost/" msg.urlstr = "ftp://localhost/tztz/ccc" msg.url = urllib.parse.urlparse(msg.urlcred+"tztz/ccc") msg.partflg = '1' msg.offset = 0 msg.length = 0 msg.sumalgo = None msg.local_file = "bbb" msg.local_offset = 0 cfg.msg = msg cfg.batch = 5 cfg.kbytes_ps= 0.05 dldr = ftp_transport() cfg.lock = None dldr.download(cfg) dldr.download(cfg) cfg.lock = '.' dldr.download(cfg) dldr.download(cfg) logger.debug("checksum = %s" % msg.onfly_checksum) dldr.download(cfg) cfg.lock = '.tmp' dldr.download(cfg) dldr.download(cfg) msg.sumalgo = cfg.sumalgo dldr.download(cfg) logger.debug("checksum = %s" % msg.onfly_checksum) logger.debug("change context") ok, details = cfg.credentials.get(cfg.destination) details.binary = False cfg.credentials.credentials[cfg.destination] = details dldr.download(cfg) logger.debug("checksum = %s" % msg.onfly_checksum) dldr.close() dldr.close() dldr.close() dldr = ftp_transport() cfg.local_file = "bbb" cfg.local_path = "./bbb" cfg.remote_file = "ddd" cfg.remote_path = "tztz/ddd" cfg.remote_urlstr = "ftp://localhost/tztz/ddd" cfg.remote_dir = "tztz" cfg.chmod = 775 cfg.lock = None dldr.send(cfg) dldr.ftp.delete("ddd") cfg.lock = '.' dldr.send(cfg) dldr.ftp.delete("ddd") cfg.lock = '.tmp' dldr.send(cfg) dldr.send(cfg) dldr.send(cfg) logger.debug("change context back") ok, details = cfg.credentials.get(cfg.destination) details.binary = True cfg.credentials.credentials[cfg.destination] = details dldr.send(cfg) dldr.send(cfg) dldr.send(cfg) dldr.close() dldr.close() dldr.close() ftp = sr_ftp(cfg) ftp.connect() ftp.cd("tztz") ftp.delete("ccc") ftp.delete("ddd") logger.info("%s" % ftp.originalDir) ftp.cd("") logger.info("%s" % ftp.ftp.pwd()) ftp.rmdir("tztz") ftp.close() except: logger.error("sr_ftp TEST FAILED") sys.exit(2) os.unlink('aaa') os.unlink('bbb') print("sr_ftp TEST PASSED") sys.exit(0)
def self_test(): logger = test_logger() # config setup cfg = sr_config() cfg.defaults() cfg.general() cfg.debug = True opt1 = "destination ftp://localhost" cfg.option(opt1.split()) cfg.logger = logger cfg.timeout = 5 try: ftp = sr_ftp(cfg) ftp.connect() ftp.mkdir("tztz") ftp.chmod(775, "tztz") ftp.cd("tztz") ftp.umask() f = open("aaa", "wb") f.write(b"1\n") f.write(b"2\n") f.write(b"3\n") f.close() ftp.put("aaa", "bbb") ls = ftp.ls() logger.info("ls = %s" % ls) ftp.chmod(775, "bbb") ls = ftp.ls() logger.info("ls = %s" % ls) ftp.rename("bbb", "ccc") ls = ftp.ls() logger.info("ls = %s" % ls) ftp.get("ccc", "bbb") f = open("bbb", "rb") data = f.read() f.close() ftp.close() if data != b"1\n2\n3\n": logger.error("sr_ftp TEST FAILED") sys.exit(1) os.unlink("bbb") msg = sr_message(cfg) msg.start_timer() msg.topic = "v02.post.test" msg.notice = "notice" msg.urlcred = "ftp://localhost/" msg.urlstr = "ftp://localhost/tztz/ccc" msg.url = urllib.parse.urlparse(msg.urlcred + "tztz/ccc") msg.partflg = '1' msg.offset = 0 msg.length = 0 msg.sumalgo = None msg.local_file = "bbb" msg.local_offset = 0 cfg.msg = msg cfg.batch = 5 cfg.kbytes_ps = 0.05 dldr = ftp_transport() cfg.lock = None dldr.download(cfg) dldr.download(cfg) cfg.lock = '.' dldr.download(cfg) dldr.download(cfg) logger.debug("checksum = %s" % msg.onfly_checksum) dldr.download(cfg) cfg.lock = '.tmp' dldr.download(cfg) dldr.download(cfg) msg.sumalgo = cfg.sumalgo dldr.download(cfg) logger.debug("checksum = %s" % msg.onfly_checksum) logger.debug("change context") ok, details = cfg.credentials.get(cfg.destination) details.binary = False cfg.credentials.credentials[cfg.destination] = details dldr.download(cfg) logger.debug("checksum = %s" % msg.onfly_checksum) dldr.close() dldr.close() dldr.close() dldr = ftp_transport() cfg.local_file = "bbb" cfg.local_path = "./bbb" cfg.remote_file = "ddd" cfg.remote_path = "tztz/ddd" cfg.remote_urlstr = "ftp://localhost/tztz/ddd" cfg.remote_dir = "tztz" cfg.chmod = 775 cfg.lock = None dldr.send(cfg) dldr.ftp.delete("ddd") cfg.lock = '.' dldr.send(cfg) dldr.ftp.delete("ddd") cfg.lock = '.tmp' dldr.send(cfg) dldr.send(cfg) dldr.send(cfg) logger.debug("change context back") ok, details = cfg.credentials.get(cfg.destination) details.binary = True cfg.credentials.credentials[cfg.destination] = details dldr.send(cfg) dldr.send(cfg) dldr.send(cfg) dldr.close() dldr.close() dldr.close() ftp = sr_ftp(cfg) ftp.connect() ftp.cd("tztz") ftp.delete("ccc") ftp.delete("ddd") logger.info("%s" % ftp.originalDir) ftp.cd("") logger.info("%s" % ftp.ftp.pwd()) ftp.rmdir("tztz") ftp.close() except: logger.error("sr_ftp TEST FAILED") sys.exit(2) os.unlink('aaa') os.unlink('bbb') print("sr_ftp TEST PASSED") sys.exit(0)
def self_test(): logger = test_logger() # config setup cfg = sr_config() cfg.defaults() cfg.general() #cfg.debug = True opt1 = "destination sftp://localhost" cfg.option(opt1.split()) cfg.logger = logger cfg.timeout = 5.0 # 1 bytes par 5 secs #cfg.kbytes_ps = 0.0001 cfg.kbytes_ps = 0.01 try: sftp = sr_sftp(cfg) sftp.connect() sftp.mkdir("tztz") sftp.chmod(775, "tztz") sftp.cd("tztz") f = open("aaa", "wb") f.write(b"1\n") f.write(b"2\n") f.write(b"3\n") f.close() sftp.put("aaa", "bbb") ls = sftp.ls() logger.info("ls = %s" % ls) sftp.chmod(775, "bbb") ls = sftp.ls() logger.info("ls = %s" % ls) sftp.rename("bbb", "ccc") ls = sftp.ls() logger.info("ls = %s" % ls) sftp.get("ccc", "bbb", 0, 0, 6) f = open("bbb", "rb") data = f.read() f.close() if data != b"1\n2\n3\n": logger.error("sr_sftp1 TEST FAILED") sys.exit(1) os.unlink("bbb") msg = sr_message(cfg) msg.start_timer() msg.topic = "v02.post.test" msg.notice = "notice" msg.urlcred = "sftp://localhost/" msg.urlstr = "sftp://localhost/tztz/ccc" msg.url = urllib.parse.urlparse(msg.urlcred + "tztz/ccc") msg.partflg = '1' msg.offset = 0 msg.length = 0 msg.local_file = "bbb" msg.local_offset = 0 msg.sumalgo = None cfg.msg = msg cfg.batch = 5 cfg.lock = None dldr = sftp_transport() dldr.download(cfg) logger.debug("checksum = %s" % msg.onfly_checksum) dldr.download(cfg) dldr.download(cfg) cfg.logger.info("lock .") cfg.lock = '.' dldr.download(cfg) dldr.download(cfg) msg.sumalgo = cfg.sumalgo dldr.download(cfg) logger.debug("checksum = %s" % msg.onfly_checksum) cfg.logger.info("lock .tmp") cfg.lock = '.tmp' dldr.download(cfg) dldr.download(cfg) dldr.close() dldr.close() dldr.close() dldr = sftp_transport() cfg.local_file = "bbb" cfg.local_path = "./bbb" cfg.remote_file = "ddd" cfg.remote_path = "tztz/ddd" cfg.remote_urlstr = "sftp://localhost/tztz/ddd" cfg.remote_dir = "tztz" cfg.chmod = 775 cfg.lock = None dldr.send(cfg) dldr.sftp.delete("ddd") cfg.lock = '.' dldr.send(cfg) dldr.sftp.delete("ddd") cfg.lock = '.tmp' dldr.send(cfg) dldr.send(cfg) dldr.send(cfg) dldr.send(cfg) dldr.send(cfg) dldr.send(cfg) dldr.close() dldr.close() dldr.close() sftp = sr_sftp(cfg) sftp.connect() sftp.cd("tztz") sftp.delete("ccc") sftp.delete("ddd") logger.info("%s" % sftp.originalDir) sftp.cd("") logger.info("%s" % sftp.sftp.getcwd()) sftp.rmdir("tztz") sftp.put("aaa", "bbb", 0, 0, 2) sftp.put("aaa", "bbb", 2, 4, 2) sftp.put("aaa", "bbb", 4, 2, 2) sftp.get("bbb", "bbb", 2, 2, 2) sftp.delete("bbb") f = open("bbb", "rb") data = f.read() f.close() if data != b"1\n3\n3\n": logger.error("sr_sftp TEST FAILED ") sys.exit(1) sftp.close() except: (stype, svalue, tb) = sys.exc_info() logger.error("(Type: %s, Value: %s)" % (stype, svalue)) logger.error("sr_sftp TEST FAILED") sys.exit(2) os.unlink('aaa') os.unlink('bbb') print("sr_sftp TEST PASSED") sys.exit(0)
def self_test(): logger = test_logger() # config setup cfg = sr_config() cfg.defaults() cfg.general() #cfg.debug = True opt1 = "destination sftp://localhost" cfg.option( opt1.split() ) cfg.logger = logger cfg.timeout = 5.0 # 1 bytes par 5 secs #cfg.kbytes_ps = 0.0001 cfg.kbytes_ps = 0.01 try: sftp = sr_sftp(cfg) sftp.connect() sftp.mkdir("tztz") sftp.chmod(775,"tztz") sftp.cd("tztz") f = open("aaa","wb") f.write(b"1\n") f.write(b"2\n") f.write(b"3\n") f.close() sftp.put("aaa", "bbb") ls = sftp.ls() logger.info("ls = %s" % ls ) sftp.chmod(775,"bbb") ls = sftp.ls() logger.info("ls = %s" % ls ) sftp.rename("bbb", "ccc") ls = sftp.ls() logger.info("ls = %s" % ls ) sftp.get("ccc", "bbb",0,0,6) f = open("bbb","rb") data = f.read() f.close() if data != b"1\n2\n3\n" : logger.error("sr_sftp1 TEST FAILED") sys.exit(1) os.unlink("bbb") msg = sr_message(cfg) msg.start_timer() msg.topic = "v02.post.test" msg.notice = "notice" msg.urlcred = "sftp://localhost/" msg.urlstr = "sftp://localhost/tztz/ccc" msg.url = urllib.parse.urlparse(msg.urlcred+"tztz/ccc") msg.partflg = '1' msg.offset = 0 msg.length = 0 msg.local_file = "bbb" msg.local_offset = 0 msg.sumalgo = None cfg.msg = msg cfg.batch = 5 cfg.lock = None dldr = sftp_transport() dldr.download(cfg) logger.debug("checksum = %s" % msg.onfly_checksum) dldr.download(cfg) dldr.download(cfg) cfg.logger.info("lock .") cfg.lock = '.' dldr.download(cfg) dldr.download(cfg) msg.sumalgo = cfg.sumalgo dldr.download(cfg) logger.debug("checksum = %s" % msg.onfly_checksum) cfg.logger.info("lock .tmp") cfg.lock = '.tmp' dldr.download(cfg) dldr.download(cfg) dldr.close() dldr.close() dldr.close() dldr = sftp_transport() cfg.local_file = "bbb" cfg.local_path = "./bbb" cfg.remote_file = "ddd" cfg.remote_path = "tztz/ddd" cfg.remote_urlstr = "sftp://localhost/tztz/ddd" cfg.remote_dir = "tztz" cfg.chmod = 775 cfg.lock = None dldr.send(cfg) dldr.sftp.delete("ddd") cfg.lock = '.' dldr.send(cfg) dldr.sftp.delete("ddd") cfg.lock = '.tmp' dldr.send(cfg) dldr.send(cfg) dldr.send(cfg) dldr.send(cfg) dldr.send(cfg) dldr.send(cfg) dldr.close() dldr.close() dldr.close() sftp = sr_sftp(cfg) sftp.connect() sftp.cd("tztz") sftp.delete("ccc") sftp.delete("ddd") logger.info("%s" % sftp.originalDir) sftp.cd("") logger.info("%s" % sftp.sftp.getcwd()) sftp.rmdir("tztz") sftp.put("aaa","bbb",0,0,2) sftp.put("aaa","bbb",2,4,2) sftp.put("aaa","bbb",4,2,2) sftp.get("bbb","bbb",2,2,2) sftp.delete("bbb") f = open("bbb","rb") data = f.read() f.close() if data != b"1\n3\n3\n" : logger.error("sr_sftp TEST FAILED ") sys.exit(1) sftp.close() except: (stype, svalue, tb) = sys.exc_info() logger.error("(Type: %s, Value: %s)" % (stype ,svalue)) logger.error("sr_sftp TEST FAILED") sys.exit(2) os.unlink('aaa') os.unlink('bbb') print("sr_sftp TEST PASSED") sys.exit(0)
def self_test(): failed = False logger = test_logger() logger.info("sr_ftp : BEGIN TEST\n") # configuration cfg = sr_config() cfg.logger = logger cfg.configure() cfg.set_sumalgo('d') msg = sr_message(cfg) msg.filesize = None msg.onfly_checksum = False # 1 bytes par 5 secs #cfg.kbytes_ps = 0.0001 cfg.timeout = 5.0 cfg.kbytes_ps = 0.01 cfg.chmod_dir = 0o775 cfg.msg = msg opt1 = "destination ftp://aspymjg@localhost" cfg.option(opt1.split()) # make sure test directory is removed before test startup ftp_url = cfg.destination ftpuser = ftp_url.split('/')[2].split('@')[0] testdir = os.path.expanduser('~' + ftpuser) + '/tztz' try: shutil.rmtree(testdir) except: pass logger.info("TEST 01: instantiation and connection") ftp = sr_ftp(cfg) ftp.connect() if ftp.check_is_connected(): logger.info("TEST 01: OK") else: logger.info("TEST 01: failed ... not connected") failed = True logger.info("TEST 02: mkdir,chmod,cd") ftp.mkdir("tztz") ls = ftp.ls() if "tztz" in ls: ftp.chmod(0o775, "tztz") ftp.cd("tztz") logger.info("TEST 02: OK") else: logger.info("TEST 02: failed") failed = True logger.info( "TEST 03: put local aaa to remote bbb, chmod bbb, rename bbb ccc") f = open("aaa", "wb") f.write(b"1\n") f.write(b"2\n") f.write(b"3\n") f.close() ftp.put("aaa", "bbb") ls = ftp.ls() if "bbb" in ls: ftp.chmod(0o775, "bbb") ftp.rename("bbb", "ccc") ls = ftp.ls() if "ccc" in ls: logger.info("TEST 03: OK") else: logger.info("TEST 03: failed") failed = True # build a message msg.start_timer() msg.topic = "v02.post.test" msg.notice = "notice" msg.baseurl = ftp_url msg.relpath = "tztz/ccc" msg.partflg = '1' msg.offset = 0 msg.length = 6 msg.local_file = "bbb" msg.local_offset = 0 cfg.set_sumalgo('d') msg.sumalgo = cfg.sumalgo msg.logger = logger msg.new_file = "bbb" msg.new_dir = "." cfg.msg = msg cfg.batch = 5 cfg.inflight = None logger.info( "TEST 04: transport download with exact name and onfly_checksum") try: os.unlink("bbb") except: pass msg.onfly_checksum = None tr = ftp_transport() tr.download(cfg) if os.path.exists("bbb") and cfg.msg.onfly_checksum: os.unlink("./bbb") logger.info("TEST 04: OK") else: logger.info("TEST 04: failed") failed = True logger.info("TEST 05: download file lock is .filename") cfg.inflight = '.' tr.download(cfg) try: os.unlink("./bbb") logger.info("TEST 05: OK") except: logger.info("TEST 05: FAILED, file not found") failed = True logger.info("TEST 06: download file lock is filename.tmp") cfg.inflight = '.tmp' tr.download(cfg) try: os.unlink("./bbb") logger.info("TEST 06: OK") except: logger.info("TEST 06: FAILED, file not found") failed = True # download the file... it is sent below tr.download(cfg) # closing way too much tr.close() tr.close() tr.close() tr = None # configure sending tr = ftp_transport() cfg.local_file = "bbb" msg.relpath = "./bbb" msg.new_dir = "tztz" msg.new_file = "ddd" cfg.remote_file = "ddd" cfg.remote_path = "tztz/ddd" cfg.remote_urlstr = ftp_url + "/tztz/ddd" cfg.remote_dir = "tztz" cfg.chmod = 0o775 logger.info("TEST 07: send file with exact name") cfg.inflight = None tr.send(cfg) if os.path.exists(testdir + os.sep + 'ddd'): logger.info("TEST 07: OK") else: logger.info("TEST 07: FAILED, file not found") failed = True logger.debug = logger.silence logger.info("TEST 08: delete remote file") ftp = tr.proto ftp.delete("ddd") if not os.path.exists("~/tztz/ddd"): logger.info("TEST 08: OK") else: logger.info("TEST 08: FAILED, file not found") failed = True # deleting a non existing file logger.info("TEST 09: delete not existing file") try: ftp.delete("zzz_unexistant") logger.info("TEST 09: OK") except: logger.info("TEST 09: FAILED") failed = True # testing several sending options logger.info("TEST 10: sending file lock = .filename") cfg.inflight = '.' tr.send(cfg) try: ftp.delete("ddd") logger.info("TEST 10: OK") except: logger.info("TEST 10: FAILED") failed = True logger.info("TEST 11: sending file lock = filename.tmp") cfg.inflight = '.tmp' tr.send(cfg) try: ftp.delete("ddd") logger.info("TEST 11: OK") except: logger.info("TEST 11: FAILED") failed = True # testing a number of sends logger.info("TEST 12: numerous send for the same file") tr.send(cfg) tr.send(cfg) tr.send(cfg) tr.close() logger.info("TEST 12: OK") # do cleanup from previous tests ftp = sr_ftp(cfg) ftp.connect() ftp.cd("tztz") ftp.ls() ftp.delete("ccc") ftp.delete("ddd") ftp.cd("") ftp.rmdir("tztz") ftp.close() logger.info("") if not failed: logger.info("sr_ftp: TEST PASSED") else: logger.info("sr_ftp: TEST FAILED") sys.exit(1)