def testAddressAlreadyInUseRoundRobinPortReservation(): """ #38 - Address already in use FDT Java https://trac.hep.caltech.edu/trac/fdtcp/ticket/38 Address already in use problem was seen during #5:comment:20 https://trac.hep.caltech.edu/trac/fdtcp/ticket/5#comment:20 2 times out of 338 transfer (attempts). Probably, when there is traffic the port can't be bound immediately even if it was released very short ago by the previous process. This test could not reproduce the problem (when reusing immediately the same port number for the next request), so FDTD.getFreePort() was reimplemented to reserver ports on round-robin basis. """ hostName = os.uname()[1] f = getTempFile(functionalFDTDConfiguration) inputOption = "--config=%s" % f.name conf = ConfigFDTD(inputOption.split()) conf.sanitize() testName = inspect.stack()[0][3] logger = Logger(name=testName, logFile="/tmp/fdtdtest-%s.log" % testName, level=logging.DEBUG) apMon = None fdtd = FDTD(conf, apMon, logger) # launch two subsequent ReceivingServerAction, second will likely fail # to bind the same, just very short ago, released port serverId = "%s" % testName testAction = TestAction(hostName, hostName) testAction.id = serverId # do TestAction r = fdtd.service.service(testAction) assert r.status == 0 options = dict(gridUserDest="someuserDest", clientIP=os.uname()[1], destFiles=[]) recvServerAction = ReceivingServerAction(testAction.id, options) # do ReceivingServerAction - start FDT Java server r = fdtd.service.service(recvServerAction) print(r.msg) assert r.status == 0 assert r.serverPort == 54321 cleanupAction = CleanupProcessesAction(serverId, timeout=0, waitTimeout=False) # do CleanupProcessesAction - shut FDT Java server, port shall be # released r = fdtd.service.service(cleanupAction) print(r.msg) assert r.status == 0 # do another ReceivingServerAction - start FDT Java server r = fdtd.service.service(recvServerAction) print(r.msg) assert r.status == 0 # will not get the same port, but the next one in the range assert r.serverPort == 54322 # in fact, if separate log files are enabled, after this last # ReceivingServerAction, there is a separate log file open. # taking the the service down, it should also closed it's related # to open files #41 problem fdtd.shutdown() fdtd.pyroDaemon.closedown() logger.close()
def testFDTDServiceOpenFiles(): """ #41 - Too many open files (fdtd side) """ hostName = os.uname()[1] f = getTempFile(functionalFDTDConfiguration) inputOption = "--config=%s" % f.name conf = ConfigFDTD(inputOption.split()) conf.sanitize() testName = inspect.stack()[0][3] logger = Logger(name=testName, logFile="/tmp/fdtdtest-%s.log" % testName, level=logging.DEBUG) apMon = None fdtd = FDTD(conf, apMon, logger) proc = Process(os.getpid()) initStateNumOpenFiles = len(proc.get_open_files()) for testAction in [TestAction("fakeSrc", "fakeDst") for i in range(3)]: r = fdtd.service.service(testAction) logger.debug("Result: %s" % r) assert r.status == 0 # after TestAction, there should not be left behind any open files numOpenFilesNow = len(proc.get_open_files()) assert initStateNumOpenFiles == numOpenFilesNow # test on ReceivingServerAction - it's action after which the # separate logger is not closed, test the number of open files went +1, # send CleanupProcessesAction and shall again remain # initStateNumOpenFiles send appropriate TestAction first (like in real) serverId = "server-id" testAction = TestAction(hostName, hostName) testAction.id = serverId r = fdtd.service.service(testAction) assert r.status == 0 options = dict(gridUserDest="someuserDest", clientIP=os.uname()[1], destFiles=[]) recvServerAction = ReceivingServerAction(testAction.id, options) r = fdtd.service.service(recvServerAction) print r.msg assert r.status == 0 numOpenFilesNow = len(proc.get_open_files()) # there should be only 1 extra opened file now assert initStateNumOpenFiles == numOpenFilesNow - 1 cleanupAction = CleanupProcessesAction(serverId, timeout=2) r = fdtd.service.service(cleanupAction) print r.msg assert r.status == 0 numOpenFilesNow = len(proc.get_open_files()) assert initStateNumOpenFiles == numOpenFilesNow fdtd.shutdown() fdtd.pyroDaemon.closedown() logger.close()
def testFDTDServiceOpenFiles(): """ #41 - Too many open files (fdtd side) """ hostName = os.uname()[1] f = getTempFile(functionalFDTDConfiguration) inputOption = "--config=%s" % f.name conf = ConfigFDTD(inputOption.split()) conf.sanitize() testName = inspect.stack()[0][3] logger = Logger(name=testName, logFile="/tmp/fdtdtest-%s.log" % testName, level=logging.DEBUG) apMon = None fdtd = FDTD(conf, apMon, logger) proc = Process(os.getpid()) initStateNumOpenFiles = len(proc.get_open_files()) for testAction in [TestAction("fakeSrc", "fakeDst") for i in range(3)]: r = fdtd.service.service(testAction) logger.debug("Result: %s" % r) assert r.status == 0 # after TestAction, there should not be left behind any open files numOpenFilesNow = len(proc.get_open_files()) assert initStateNumOpenFiles == numOpenFilesNow # test on ReceivingServerAction - it's action after which the # separate logger is not closed, test the number of open files went +1, # send CleanupProcessesAction and shall again remain # initStateNumOpenFiles send appropriate TestAction first (like in real) serverId = "server-id" testAction = TestAction(hostName, hostName) testAction.id = serverId r = fdtd.service.service(testAction) assert r.status == 0 options = dict(gridUserDest="someuserDest", clientIP=os.uname()[1], destFiles=[]) recvServerAction = ReceivingServerAction(testAction.id, options) r = fdtd.service.service(recvServerAction) print(r.msg) assert r.status == 0 numOpenFilesNow = len(proc.get_open_files()) # there should be only 1 extra opened file now assert initStateNumOpenFiles == numOpenFilesNow - 1 cleanupAction = CleanupProcessesAction(serverId, timeout=2) r = fdtd.service.service(cleanupAction) print(r.msg) assert r.status == 0 numOpenFilesNow = len(proc.get_open_files()) assert initStateNumOpenFiles == numOpenFilesNow fdtd.shutdown() fdtd.pyroDaemon.closedown() logger.close()
def testTestAction(): ta = TestAction("src", "dest", 5) assert ta.timeout == 5 r = ta.execute() assert r.status == 0 assert r.id == ta.id
def testTestActionId(): t = TestAction("src", "dest", timeout=4) assert t.timeout == 4
def performTransfer(self): self.logger.info("Starting transfer %s" % self) if self.result != None: self.logger.warn("Skipping transfer %s (already failed)." % self) return # timeout is used only on TestActions (testing mutual connection) # id of the initial action (this test (TestAction)) is later used # in all subsequent actions related to this particular transfer timeout = int(self.conf.get("timeout")) testAction = TestAction(self.hostSrc, self.hostDest, timeout=timeout) self.id = testAction.id self.logger.info("Testing remote parties availability, " "transfer id: '%s' ..." % testAction.id) self.receiver.perform(testAction) self.sender.perform(testAction) # remote services are available, authenticate with both # remoteGridUserSrc - user's name at the source site # remoteGridUserDest - user's name at the destination site authStartTime = datetime.datetime.now() remoteGridUserSrc, remoteGridUserDest = self._runAuthChain() authEndTime = datetime.datetime.now() authTime = (authEndTime - authStartTime).seconds par = dict(id=self.id, authentication=authTime) self.logger.debug("Authentication lasted: %s [s]." % authTime) if self.apMon: self.logger.debug("Sending data to ApMon ...") self.apMon.sendParameters("fdtcp", None, par) # find out client IP address, server will need # it (-f <allowedIPsList>) when starting clientIP = socket.gethostbyname(self.hostSrc) self.logger.debug("Client IP address is: '%s'" % clientIP) # since the separate log file may not be closed properly once the # ReceivingServer request is send, register the request for possible # cleaning now (issue found during ticket:5#comment:24) # no exception was raised, remote process must be running, register # for clean up (remote process may however terminate normally, then # shall be removed from the container of processes at the remote # party) self.toCleanup.append(self.receiver.uri) # start receiving server first, information on its port will need # the client # destFiles - list if files at destination - just check (#36) destFiles = [f.fileDest for f in self.files] options = dict(gridUserDest=remoteGridUserDest, clientIP=clientIP, destFiles=destFiles) recvServerAction = ReceivingServerAction(self.id, options) result = self.receiver.perform(recvServerAction) serverFDTPort = result.serverPort self.logger.info("Remote FDT server: %s:%s" % (result.host, serverFDTPort)) # calling the client is synchronous action - waiting until it finishes # if the client hangs without progress - may want to kill such remote # process, thus register the remote URI into clean up local container # if something goes wrong, this remote URI later receives # CleanupAction to kill its running processes self.toCleanup.append(self.sender.uri) # start sending FDT client which initiates the transfer process options = dict(port=serverFDTPort, hostDest=self.hostDest, transferFiles=self.files, gridUserSrc=remoteGridUserSrc) sndClientAction = SendingClientAction(self.id, options) # this command will stop the execution flow, for very very long if # the transfer data is bulky - status monitoring via MonALISA # using the transfer id result = self.sender.perform(sndClientAction) self.logger.info("Transfer result: %s, logs (FDT sending " "client):\n%s" % (result, result.log)) self.result = result.status # don't put the log there for now, but in case of failure it puts the # log into result and then into report file # self.log = result.log # clean up remote processes # the server - FDT Java server is run with -S - it gets # automatically shut once the transfer is over # the client - if everything was all right (client not hanging), it # has already successfully finished (and the key of the process # has been removed from the container with the client) # yet call the cleanup at both sides explicitly self.performCleanup(waitTimeout=True)
def testInputCopyJobFileTranslationIntoFDTFileList(): # correct data - input copyjobfile - all must be the same source host, # destination host pairs, otherwise such input copyjobfile will break # into a number of FDT fileList files on corresponding source hosts inputData = \ """fdt://localhost:111/tmp/file fdt://localhost:222/tmp/fileX1 fdt://localhost:111/tmp/fileGY fdt://localhost:222/tmp/fileX2 fdt://localhost:111/tmp/fileWA fdt://localhost:222/tmp/fileX fdt://localhost:111/tmp/fileWQ fdt://localhost:222/tmp/fileTY fdt://localhost:111/tmp/file9Y fdt://localhost:222/tmp/fileIO""" # desired output fileList for FDT client outputData = \ """/tmp/file / /tmp/fileX1 /tmp/fileGY / /tmp/fileX2 /tmp/fileWA / /tmp/fileX /tmp/fileWQ / /tmp/fileTY /tmp/file9Y / /tmp/fileIO""" logger = Logger("test logger", level=logging.DEBUG) copyJobFile = tempfile.NamedTemporaryFile("w+") # read / write copyJobFile.write(inputData) copyJobFile.flush() copyJobFile.seek(0) inputOption = "--copyjobfile=%s" % copyJobFile.name conf = ConfigFDTCopy(inputOption.split()) apMon = None transfers = Transfers(conf, apMon, logger) # since having the same source host, destination host pair, should have # only one transfer job t = transfers.transfers["localhost:111-localhost:222"] assert len(transfers.transfers) == 1 assert len(t.files) == 5 # do relevant stuff from fdtcp performTransfer method now testAction = TestAction(t.hostSrc, t.hostDest, timeout=5) t.id = testAction.id options = dict(port="some_port", hostDest=t.hostDest, transferFiles=t.files) sndClientAction = SendingClientAction(testAction.id, options) assert sndClientAction.options["port"] == "some_port" assert sndClientAction.options["hostDest"] == t.hostDest # fileList is constructed at the side of fdtd service (remote site) # simulate this process ... class MockFDTDConfig(Mock): # mock class only to satisfy sndClientAction._setUp() call - when # it checks for log file def get(self, what): return "/tmp/logfile" # this method is called on fdtd service site sndClientAction._setUp(MockFDTDConfig()) assert sndClientAction.options["fileList"] == \ "/tmp/fileLists/fdt-fileList-%s" % sndClientAction.id # now check content of the fileList file - shall be as the output data data = open(sndClientAction.options["fileList"], 'r').readlines() for line1, line2 in zip(data, outputData.split('\n')): assert line1.strip() == line2 # clean up after test, only if succeeded os.unlink(sndClientAction.options["fileList"])
def testFDTDServiceOpenFilesFullTransfer(): """ #41:comment:8 - Too many open files (fdtd side) SendingClient actually removed itself from the executors container once it finishes so subsequent CleanupProcessesAction doesn't know about this process, nor about its open separate log file, which doesn't get closed. Simulate a simple successful transfer, send all actions and check number of open files - does all as it happens in fdtd.service() """ hostName = os.uname()[1] testName = inspect.stack()[0][3] initStateNumOpenFilesTestStart, filesStr = getOpenFilesList() print("%s: test 0: open files: %s items:\n%s" % (testName, initStateNumOpenFilesTestStart, filesStr)) # there should not be any open files now assert initStateNumOpenFilesTestStart == 0 f = getTempFile(functionalFDTDConfiguration) inputOption = "--config=%s --port=10001" % f.name confServer = ConfigFDTD(inputOption.split()) confServer.sanitize() loggerServer = Logger(name=testName, logFile="/tmp/fdtdtest-%s-writer.log" % testName, level=logging.DEBUG) apMon = None fdtdServer = FDTD(confServer, apMon, loggerServer) inputOption = "--config=%s --port=10002" % f.name confReader = ConfigFDTD(inputOption.split()) confReader.sanitize() loggerReader = Logger(name=testName, logFile="/tmp/fdtdtest-%s-reader.log" % testName, level=logging.DEBUG) apMon = None fdtdReader = FDTD(confReader, apMon, loggerReader) # -2 open log files, additional -1 is the temp config file initStateNumOpenFiles, filesStr = getOpenFilesList() print("%s: test 1: open files: %s items:\n%s" % (testName, initStateNumOpenFiles, filesStr)) assert initStateNumOpenFilesTestStart == initStateNumOpenFiles - 2 - 1 testActionServer = TestAction(hostName, hostName) testActionServer.id = testActionServer.id + "-writer" r = fdtdServer.service.service(testActionServer) assert r.status == 0 options = dict(gridUserDest="someuserDest", clientIP=os.uname()[1], destFiles=["/dev/null"]) recvServerAction = ReceivingServerAction(testActionServer.id, options) r = fdtdServer.service.service(recvServerAction) print r.msg assert r.status == 0 serverFDTPort = r.serverPort # there should be only 1 extra opened file now - ReceivingServerAction # separate log numOpenFilesNow, filesStr = getOpenFilesList() print("%s: test 2: open files: %s items:\n%s" % (testName, numOpenFilesNow, filesStr)) assert initStateNumOpenFiles == numOpenFilesNow - 1 testActionReader = TestAction(hostName, hostName) testActionReader.id = testActionReader.id + "-reader" r = fdtdReader.service.service(testActionReader) assert r.status == 0 files = [TransferFile("/etc/passwd", "/dev/null")] # list of TransferFile options = dict(port=serverFDTPort, hostDest=os.uname()[1], transferFiles=files, gridUserSrc="soemuserSrc") sndClientAction = SendingClientAction(testActionReader.id, options) r = fdtdReader.service.service(sndClientAction) assert r.status == 0 # there should be +2 extra - for separate both server and client numOpenFilesNow, filesStr = getOpenFilesList() print("%s: test 3: open files: %s items:\n%s" % (testName, numOpenFilesNow, filesStr)) # 2 extra files - separate transfer log at both ends assert initStateNumOpenFiles == numOpenFilesNow - 2 # now the transfer is over, both server (writer) and sender (reader) # parties kept their separate log files open, CleanupProcessesAction # will close them print "going to clean up" cl = CleanupProcessesAction(testActionReader.id, waitTimeout=False) r = fdtdReader.service.service(cl) assert r.status == 0 # one shall be closed now numOpenFilesNow, filesStr = getOpenFilesList() print("%s: test 4: open files: %s items:\n%s" % (testName, numOpenFilesNow, filesStr)) assert initStateNumOpenFiles == numOpenFilesNow - 1 cl = CleanupProcessesAction(testActionServer.id, waitTimeout=False) r = fdtdServer.service.service(cl) assert r.status == 0 # both separate log files should be closed now # problem #41:comment:8 was here - server behaved correctly, but # reader kept its separate log file open numOpenFilesNow, filesStr = getOpenFilesList() print("%s: test 5: open files: %s items:\n%s" % (testName, numOpenFilesNow, filesStr)) assert initStateNumOpenFiles == numOpenFilesNow fdtdServer.shutdown() fdtdServer.pyroDaemon.closedown() loggerServer.close() fdtdReader.shutdown() fdtdReader.pyroDaemon.closedown() loggerReader.close() # after even log files were closed, etc numOpenFilesNow, filesStr = getOpenFilesList() print("%s: test 6: open files: %s items:\n%s" % (testName, numOpenFilesNow, filesStr)) # -1: the temp configuration file is still open assert initStateNumOpenFilesTestStart == numOpenFilesNow - 1
def testAddressAlreadyInUseRoundRobinPortReservation(): """ #38 - Address already in use FDT Java https://trac.hep.caltech.edu/trac/fdtcp/ticket/38 Address already in use problem was seen during #5:comment:20 https://trac.hep.caltech.edu/trac/fdtcp/ticket/5#comment:20 2 times out of 338 transfer (attempts). Probably, when there is traffic the port can't be bound immediately even if it was released very short ago by the previous process. This test could not reproduce the problem (when reusing immediately the same port number for the next request), so FDTD.getFreePort() was reimplemented to reserver ports on round-robin basis. """ hostName = os.uname()[1] f = getTempFile(functionalFDTDConfiguration) inputOption = "--config=%s" % f.name conf = ConfigFDTD(inputOption.split()) conf.sanitize() testName = inspect.stack()[0][3] logger = Logger(name=testName, logFile="/tmp/fdtdtest-%s.log" % testName, level=logging.DEBUG) apMon = None fdtd = FDTD(conf, apMon, logger) # launch two subsequent ReceivingServerAction, second will likely fail # to bind the same, just very short ago, released port serverId = "%s" % testName testAction = TestAction(hostName, hostName) testAction.id = serverId # do TestAction r = fdtd.service.service(testAction) assert r.status == 0 options = dict(gridUserDest="someuserDest", clientIP=os.uname()[1], destFiles=[]) recvServerAction = ReceivingServerAction(testAction.id, options) # do ReceivingServerAction - start FDT Java server r = fdtd.service.service(recvServerAction) print r.msg assert r.status == 0 assert r.serverPort == 54321 cleanupAction = CleanupProcessesAction(serverId, timeout=0, waitTimeout=False) # do CleanupProcessesAction - shut FDT Java server, port shall be # released r = fdtd.service.service(cleanupAction) print r.msg assert r.status == 0 # do another ReceivingServerAction - start FDT Java server r = fdtd.service.service(recvServerAction) print r.msg assert r.status == 0 # will not get the same port, but the next one in the range assert r.serverPort == 54322 # in fact, if separate log files are enabled, after this last # ReceivingServerAction, there is a separate log file open. # taking the the service down, it should also closed it's related # to open files #41 problem fdtd.shutdown() fdtd.pyroDaemon.closedown() logger.close()
def testTestAction(): ta = TestAction(5) assert ta.timeout == 5 r = ta.execute() assert r.status == 0 assert r.id == ta.id
def testFDTDServiceOpenFilesFullTransfer(): """ #41:comment:8 - Too many open files (fdtd side) SendingClient actually removed itself from the executors container once it finishes so subsequent CleanupProcessesAction doesn't know about this process, nor about its open separate log file, which doesn't get closed. Simulate a simple successful transfer, send all actions and check number of open files - does all as it happens in fdtd.service() """ hostName = os.uname()[1] testName = inspect.stack()[0][3] initStateNumOpenFilesTestStart, filesStr = getOpenFilesList() print("%s: test 0: open files: %s items:\n%s" % (testName, initStateNumOpenFilesTestStart, filesStr)) # there should not be any open files now assert initStateNumOpenFilesTestStart == 0 f = getTempFile(functionalFDTDConfiguration) inputOption = "--config=%s --port=10001" % f.name confServer = ConfigFDTD(inputOption.split()) confServer.sanitize() loggerServer = Logger(name=testName, logFile="/tmp/fdtdtest-%s-writer.log" % testName, level=logging.DEBUG) apMon = None fdtdServer = FDTD(confServer, apMon, loggerServer) inputOption = "--config=%s --port=10002" % f.name confReader = ConfigFDTD(inputOption.split()) confReader.sanitize() loggerReader = Logger(name=testName, logFile="/tmp/fdtdtest-%s-reader.log" % testName, level=logging.DEBUG) apMon = None fdtdReader = FDTD(confReader, apMon, loggerReader) # -2 open log files, additional -1 is the temp config file initStateNumOpenFiles, filesStr = getOpenFilesList() print("%s: test 1: open files: %s items:\n%s" % (testName, initStateNumOpenFiles, filesStr)) assert initStateNumOpenFilesTestStart == initStateNumOpenFiles - 2 - 1 testActionServer = TestAction(hostName, hostName) testActionServer.id = testActionServer.id + "-writer" r = fdtdServer.service.service(testActionServer) assert r.status == 0 options = dict(gridUserDest="someuserDest", clientIP=os.uname()[1], destFiles=["/dev/null"]) recvServerAction = ReceivingServerAction(testActionServer.id, options) r = fdtdServer.service.service(recvServerAction) print(r.msg) assert r.status == 0 serverFDTPort = r.serverPort # there should be only 1 extra opened file now - ReceivingServerAction # separate log numOpenFilesNow, filesStr = getOpenFilesList() print("%s: test 2: open files: %s items:\n%s" % (testName, numOpenFilesNow, filesStr)) assert initStateNumOpenFiles == numOpenFilesNow - 1 testActionReader = TestAction(hostName, hostName) testActionReader.id = testActionReader.id + "-reader" r = fdtdReader.service.service(testActionReader) assert r.status == 0 files = [TransferFile("/etc/passwd", "/dev/null")] # list of TransferFile options = dict(port=serverFDTPort, hostDest=os.uname()[1], transferFiles=files, gridUserSrc="soemuserSrc") sndClientAction = SendingClientAction(testActionReader.id, options) r = fdtdReader.service.service(sndClientAction) assert r.status == 0 # there should be +2 extra - for separate both server and client numOpenFilesNow, filesStr = getOpenFilesList() print("%s: test 3: open files: %s items:\n%s" % (testName, numOpenFilesNow, filesStr)) # 2 extra files - separate transfer log at both ends assert initStateNumOpenFiles == numOpenFilesNow - 2 # now the transfer is over, both server (writer) and sender (reader) # parties kept their separate log files open, CleanupProcessesAction # will close them print("going to clean up") cl = CleanupProcessesAction(testActionReader.id, waitTimeout=False) r = fdtdReader.service.service(cl) assert r.status == 0 # one shall be closed now numOpenFilesNow, filesStr = getOpenFilesList() print("%s: test 4: open files: %s items:\n%s" % (testName, numOpenFilesNow, filesStr)) assert initStateNumOpenFiles == numOpenFilesNow - 1 cl = CleanupProcessesAction(testActionServer.id, waitTimeout=False) r = fdtdServer.service.service(cl) assert r.status == 0 # both separate log files should be closed now # problem #41:comment:8 was here - server behaved correctly, but # reader kept its separate log file open numOpenFilesNow, filesStr = getOpenFilesList() print("%s: test 5: open files: %s items:\n%s" % (testName, numOpenFilesNow, filesStr)) assert initStateNumOpenFiles == numOpenFilesNow fdtdServer.shutdown() fdtdServer.pyroDaemon.closedown() loggerServer.close() fdtdReader.shutdown() fdtdReader.pyroDaemon.closedown() loggerReader.close() # after even log files were closed, etc numOpenFilesNow, filesStr = getOpenFilesList() print("%s: test 6: open files: %s items:\n%s" % (testName, numOpenFilesNow, filesStr)) # -1: the temp configuration file is still open assert initStateNumOpenFilesTestStart == numOpenFilesNow - 1