def testFDTDKillProcess2(): c = """ [general] port = 6700 debug = DEBUG killCommand = ../wrapper_kill.sh %(pid)s portRangeFDTServer = 54321,54400 """ f = getTempFile(c) inputOption = "--config=%s" % f.name conf = ConfigFDTD(inputOption.split()) testName = inspect.stack()[0][3] logger = Logger(name=testName, level=logging.DEBUG) apMon = None daemon = FDTD(conf, apMon, logger) # need long-running, non blocking process command = "dd if=/dev/zero of=/dev/null count=100000000 bs=102400" executor = Executor("some_id", command, blocking=False, caller=daemon, logger=logger, killTimeout=0) try: executor.execute() daemon.killProcess("some_id", logger) finally: # definitely release port and kill the process daemon.shutdown() daemon.pyroDaemon.closedown() try: p = Process(executor.proc.pid) m = "FAIL: Process PID:%s should have been " "killed." % executor.proc.pid logger.debug(m) py.test.fail(m) except NoSuchProcess as ex: logger.debug("OK: Process PID:%s doesn't exist now." % executor.proc.pid)
def testFDTDServiceOpenFiles(): """ #41 - Too many open files (fdtd side) """ hostName = os.uname()[1] f = getTempFile(functionalFDTDConfiguration) inputOption = "--config=%s" % f.name conf = ConfigFDTD(inputOption.split()) conf.sanitize() testName = inspect.stack()[0][3] logger = Logger(name=testName, logFile="/tmp/fdtdtest-%s.log" % testName, level=logging.DEBUG) apMon = None fdtd = FDTD(conf, apMon, logger) proc = Process(os.getpid()) initStateNumOpenFiles = len(proc.get_open_files()) for testAction in [TestAction("fakeSrc", "fakeDst") for i in range(3)]: r = fdtd.service.service(testAction) logger.debug("Result: %s" % r) assert r.status == 0 # after TestAction, there should not be left behind any open files numOpenFilesNow = len(proc.get_open_files()) assert initStateNumOpenFiles == numOpenFilesNow # test on ReceivingServerAction - it's action after which the # separate logger is not closed, test the number of open files went +1, # send CleanupProcessesAction and shall again remain # initStateNumOpenFiles send appropriate TestAction first (like in real) serverId = "server-id" testAction = TestAction(hostName, hostName) testAction.id = serverId r = fdtd.service.service(testAction) assert r.status == 0 options = dict(gridUserDest="someuserDest", clientIP=os.uname()[1], destFiles=[]) recvServerAction = ReceivingServerAction(testAction.id, options) r = fdtd.service.service(recvServerAction) print r.msg assert r.status == 0 numOpenFilesNow = len(proc.get_open_files()) # there should be only 1 extra opened file now assert initStateNumOpenFiles == numOpenFilesNow - 1 cleanupAction = CleanupProcessesAction(serverId, timeout=2) r = fdtd.service.service(cleanupAction) print r.msg assert r.status == 0 numOpenFilesNow = len(proc.get_open_files()) assert initStateNumOpenFiles == numOpenFilesNow fdtd.shutdown() fdtd.pyroDaemon.closedown() logger.close()
def testLogger(): logger = Logger("test logger", level=logging.DEBUG) logger.info("info message") logger.warn("warn message") logger.warning("warning message") logger.error("error message") logger.critical("critical message") logger.fatal("fatal message") logger.debug("debug message") logger.close()
def testFDTDServiceOpenFiles(): """ #41 - Too many open files (fdtd side) """ hostName = os.uname()[1] f = getTempFile(functionalFDTDConfiguration) inputOption = "--config=%s" % f.name conf = ConfigFDTD(inputOption.split()) conf.sanitize() testName = inspect.stack()[0][3] logger = Logger(name=testName, logFile="/tmp/fdtdtest-%s.log" % testName, level=logging.DEBUG) apMon = None fdtd = FDTD(conf, apMon, logger) proc = Process(os.getpid()) initStateNumOpenFiles = len(proc.get_open_files()) for testAction in [TestAction("fakeSrc", "fakeDst") for i in range(3)]: r = fdtd.service.service(testAction) logger.debug("Result: %s" % r) assert r.status == 0 # after TestAction, there should not be left behind any open files numOpenFilesNow = len(proc.get_open_files()) assert initStateNumOpenFiles == numOpenFilesNow # test on ReceivingServerAction - it's action after which the # separate logger is not closed, test the number of open files went +1, # send CleanupProcessesAction and shall again remain # initStateNumOpenFiles send appropriate TestAction first (like in real) serverId = "server-id" testAction = TestAction(hostName, hostName) testAction.id = serverId r = fdtd.service.service(testAction) assert r.status == 0 options = dict(gridUserDest="someuserDest", clientIP=os.uname()[1], destFiles=[]) recvServerAction = ReceivingServerAction(testAction.id, options) r = fdtd.service.service(recvServerAction) print(r.msg) assert r.status == 0 numOpenFilesNow = len(proc.get_open_files()) # there should be only 1 extra opened file now assert initStateNumOpenFiles == numOpenFilesNow - 1 cleanupAction = CleanupProcessesAction(serverId, timeout=2) r = fdtd.service.service(cleanupAction) print(r.msg) assert r.status == 0 numOpenFilesNow = len(proc.get_open_files()) assert initStateNumOpenFiles == numOpenFilesNow fdtd.shutdown() fdtd.pyroDaemon.closedown() logger.close()
def testReceivingServerActionCheckTargetFileNames(): logger = Logger() files = ["/mnt/data", "/etc/passwd", "/etc/something/nonsence", "/tmp"] options = dict(port=1000, gridUserDest="someuser", destFiles=files) a = ReceivingServerAction("some_id", options) logger.info("%s - checking presence of files at target location ..." % a.__class__.__name__) r = a._checkTargetFileNames(files) logger.debug("Results:\n%s" % r) expected = \ """ exists True: /mnt/data exists False: /mnt/.data exists True: /etc/passwd exists False: /etc/.passwd exists False: /etc/something/nonsence exists False: /etc/something/.nonsence exists True: /tmp exists False: /.tmp """ assert r == expected
def testFDTDKillProcess2(): c = """ [general] port = 6700 debug = DEBUG killCommand = ../wrapper_kill.sh %(pid)s portRangeFDTServer = 54321,54400 """ f = getTempFile(c) inputOption = "--config=%s" % f.name conf = ConfigFDTD(inputOption.split()) testName = inspect.stack()[0][3] logger = Logger(name=testName, level=logging.DEBUG) apMon = None daemon = FDTD(conf, apMon, logger) # need long-running, non blocking process command = "dd if=/dev/zero of=/dev/null count=100000000 bs=102400" executor = Executor("some_id", command, blocking=False, caller=daemon, logger=logger, killTimeout=0) try: executor.execute() daemon.killProcess("some_id", logger) finally: # definitely release port and kill the process daemon.shutdown() daemon.pyroDaemon.closedown() try: p = Process(executor.proc.pid) m = ("FAIL: Process PID:%s should have been " "killed." % executor.proc.pid) logger.debug(m) py.test.fail(m) except NoSuchProcess, ex: logger.debug("OK: Process PID:%s doesn't exist now." % executor.proc.pid)
def testLoggerTraceBackTrueOnNoException(): logger = Logger("test file logger", level=logging.DEBUG) logger.debug("my msg", traceBack=True)
conf = ConfigFDTCopy(sys.argv[1:]) conf.sanitize() except ConfigurationException, ex: print "fdtcp failed to start, reason: %s" % ex sys.exit(1) logger = Logger(name="fdtcp", logFile=conf.get("logFile"), level=conf.get("debug")) # ticket #35 - mercurial expandable keywords in the code # information from the SCM (expandable keywords) versionInfo = dict(Revision="$Revision: cf25e731af0a $", Tags="$Tags: tip $") logger.info("fdtcp starting ... version: %s" % logger.pprintFormat(versionInfo)) logger.debug("Search sys.path:\n%s\n" % logger.pprintFormat(sys.path)) logger.debug("PYRO_STORAGE: '%s'" % os.environ.get("PYRO_STORAGE")) logger.debug("Input command line arguments: '%s'" % optBackup) logger.debug("Configuration values (processed):\n%s\n" % logger.pprintFormat(conf._options)) apMon = None apMonDestConf = conf.get("apMonDestinations") if apMonDestConf: apMonDestinations = tuple(apMonDestConf.split(',')) logger.info("Initializing MonALISA ApMon, destinations: %s ..." % (apMonDestinations, )) apMon = apmon.ApMon(apMonDestinations) apMon.enableBgMonitoring(True) else:
def testMainLogFileOpeningDuringDaemonisation(): py.test.skip("Test skipped, messes up with input/output streams for " "other test, better to run it stand-alone: " "py.test fdtcplib/test/test_fdtd.py -s -k testMainLogFileOpeningDuringDaemonisation") """ As described in the problem #41:comment:9, the main log file remains open thrice (under different descriptor) after initial daemonisation. In order to include this particular test into whole test suite running (issue with file descriptors being redirected, etc in fdtd.daemonize() function), perhaps after running this test, the file descriptors can be put back from backup sys.__stdout__, etc? """ c = """ [general] logFile = /tmp/fdtd.log pidFile = /tmp/fdtd.pid """ testName = inspect.stack()[0][3] f = getTempFile(c) inputOption = "--config=%s" % f.name conf = ConfigFDTD(inputOption.split()) f.close() logFile = "%s-%s" % (conf.get("logFile"), testName) pidFile = conf.get("pidFile") if os.path.exists(logFile): print("test: %s: file '%s' exists, removing ..." % (testName, logFile)) os.remove(logFile) if os.path.exists(pidFile): print("test: %s: file '%s' exists, removing ..." % (testName, pidFile)) os.remove(pidFile) logger = Logger(name=testName, logFile=logFile, level=logging.DEBUG) logger.debug("Before daemonization ...") numFiles, filesStr = getOpenFilesList() logger.debug("Logging open files: %s items:\n%s" % (numFiles, filesStr)) try: daemonize(conf, logger) except SystemExit: # previous leading process continues here return # here continues the newly created background daemon f = open(pidFile, 'r') rc = f.read() f.close() rc = rc.strip() pid = int(rc) numFiles, filesStr = getOpenFilesList() logger.debug("Logging open files: %s items:\n%s" % (numFiles, filesStr)) logger.debug("Before finishing ... ") logger.close() # the log file may be open more times due to streams descriptor # duplication as done in fdtd.daemonization() but now, once is # closed, there should not any other outstanding open file assert numFiles == 0
def testMainLogFileOpeningDuringDaemonisation(): py.test.skip( "Test skipped, messes up with input/output streams for " "other test, better to run it stand-alone: " "py.test fdtcplib/test/test_fdtd.py -s -k testMainLogFileOpeningDuringDaemonisation" ) """ As described in the problem #41:comment:9, the main log file remains open thrice (under different descriptor) after initial daemonisation. In order to include this particular test into whole test suite running (issue with file descriptors being redirected, etc in fdtd.daemonize() function), perhaps after running this test, the file descriptors can be put back from backup sys.__stdout__, etc? """ c = """ [general] logFile = /tmp/fdtd.log pidFile = /tmp/fdtd.pid """ testName = inspect.stack()[0][3] f = getTempFile(c) inputOption = "--config=%s" % f.name conf = ConfigFDTD(inputOption.split()) f.close() logFile = "%s-%s" % (conf.get("logFile"), testName) pidFile = conf.get("pidFile") if os.path.exists(logFile): print("test: %s: file '%s' exists, removing ..." % (testName, logFile)) os.remove(logFile) if os.path.exists(pidFile): print("test: %s: file '%s' exists, removing ..." % (testName, pidFile)) os.remove(pidFile) logger = Logger(name=testName, logFile=logFile, level=logging.DEBUG) logger.debug("Before daemonization ...") numFiles, filesStr = getOpenFilesList() logger.debug("Logging open files: %s items:\n%s" % (numFiles, filesStr)) try: daemonize(conf, logger) except SystemExit: # previous leading process continues here return # here continues the newly created background daemon f = open(pidFile, "r") rc = f.read() f.close() rc = rc.strip() pid = int(rc) numFiles, filesStr = getOpenFilesList() logger.debug("Logging open files: %s items:\n%s" % (numFiles, filesStr)) logger.debug("Before finishing ... ") logger.close() # the log file may be open more times due to streams descriptor # duplication as done in fdtd.daemonization() but now, once is # closed, there should not any other outstanding open file assert numFiles == 0
def main(): # all values and action information held in the conf object optBackup = sys.argv[:] try: conf = ConfigFDTCopy(sys.argv[1:]) conf.sanitize() except ConfigurationException, ex: print "fdtcp failed to start, reason: %s" % ex sys.exit(1) logger = Logger(name = "fdtcp", logFile = conf.get("logFile"), level = conf.get("debug")) logger.info("fdtcp starting ...") logger.debug("Search sys.path: '%s'" % sys.path) logger.debug("PYRO_STORAGE: '%s'" % os.environ.get("PYRO_STORAGE")) logger.debug("Input command line arguments: '%s'" % optBackup) logger.debug("Configuration values (processed): '%s'" % conf._options) apMonDestConf = conf.get("apMonDestinations") apMonDestinations = tuple(apMonDestConf.split(',')) logger.info("Initializing MonALISA ApMon, destinations: %s ..." % (apMonDestinations,)) apMon = apmon.ApMon(apMonDestinations) apMon.enableBgMonitoring(True) # use DNS names rather than IP address Pyro.config.PYRO_DNS_URI = True
from fdtcplib.utils.Logger import Logger transferFiles = [ '/mnt/hadoop/user/uscms01/pnfs/unl.edu/data4/cms/store/phedex_monarctest/Nebraska/LoadTest07_Nebraska_65 / /mnt/hadoop/store/PhEDEx_LoadTest07/LoadTest07_Debug_US_Nebraska/US_Caltech/3962/LoadTest07_Nebraska_65_uprBpUmMbtrlF910_3962', '/mnt/hadoop/user/uscms01/pnfs/unl.edu/data4/cms/store/phedex_monarctest/Nebraska/LoadTest07_Nebraska_AB / /mnt/hadoop/store/PhEDEx_LoadTest07/LoadTest07_Debug_US_Nebraska/US_Caltech/3962/LoadTest07_Nebraska_AB_VBggLIppuOd19CPC_3962', '/mnt/hadoop/user/uscms01/pnfs/unl.edu/data4/cms/store/phedex_monarctest/Nebraska/LoadTest07_Nebraska_E3 / /mnt/hadoop/store/PhEDEx_LoadTest07/LoadTest07_Debug_US_Nebraska/US_Caltech/3962/LoadTest07_Nebraska_E3_nxS6pVoM1of60fNW_3962', '/mnt/hadoop/user/uscms01/pnfs/unl.edu/data4/cms/store/phedex_monarctest/Nebraska/LoadTest07_Nebraska_0B / /mnt/hadoop/store/PhEDEx_LoadTest07/LoadTest07_Debug_US_Nebraska/US_Caltech/3962/LoadTest07_Nebraska_0B_QIG6aG9hpal8XTJD_3962', '/mnt/hadoop/user/uscms01/pnfs/unl.edu/data4/cms/store/phedex_monarctest/Nebraska/LoadTest07_Nebraska_1B / /mnt/hadoop/store/PhEDEx_LoadTest07/LoadTest07_Debug_US_Nebraska/US_Caltech/3962/LoadTest07_Nebraska_1B_but3hjz0eT7bWHrr_3962', '/mnt/hadoop/user/uscms01/pnfs/unl.edu/data4/cms/store/phedex_monarctest/Nebraska/LoadTest07_Nebraska_39 / /mnt/hadoop/store/PhEDEx_LoadTest07/LoadTest07_Debug_US_Nebraska/US_Caltech/3962/LoadTest07_Nebraska_39_mcBzPcNt1pJcaAkc_3962', '/mnt/hadoop/user/uscms01/pnfs/unl.edu/data4/cms/store/phedex_monarctest/Nebraska/LoadTest07_Nebraska_81 / /mnt/hadoop/store/PhEDEx_LoadTest07/LoadTest07_Debug_US_Nebraska/US_Caltech/3962/LoadTest07_Nebraska_81_QueqvduaTBMjk3pu_3962', '/mnt/hadoop/user/uscms01/pnfs/unl.edu/data4/cms/store/phedex_monarctest/Nebraska/LoadTest07_Nebraska_CB / /mnt/hadoop/store/PhEDEx_LoadTest07/LoadTest07_Debug_US_Nebraska/US_Caltech/3962/LoadTest07_Nebraska_CB_YbQNLdVZnUIEukVe_3962', '/mnt/hadoop/user/uscms01/pnfs/unl.edu/data4/cms/store/phedex_monarctest/Nebraska/LoadTest07_Nebraska_1C / /mnt/hadoop/store/PhEDEx_LoadTest07/LoadTest07_Debug_US_Nebraska/US_Caltech/3962/LoadTest07_Nebraska_1C_3iyt9ymMVRpAVEWz_3962', '/mnt/hadoop/user/uscms01/pnfs/unl.edu/data4/cms/store/phedex_monarctest/Nebraska/LoadTest07_Nebraska_62 / /mnt/hadoop/store/PhEDEx_LoadTest07/LoadTest07_Debug_US_Nebraska/US_Caltech/3962/LoadTest07_Nebraska_62_wboDVRwpaSU8jU0O_3962' ] options = dict(port=54321, hostDest='gridftp05.ultralight.org', transferFiles=transferFiles, gridUserSrc='cmsphedex') s = SendingClientAction( 'fdtcp-cithep249.ultralight.org-phedex-2011-04-08--06h:19m:21s:822724mics-epjiu', options) #logger = Logger(name = "fdtcp", logFile = "/tmp/logfile") logger = Logger(name="fdtcp") uri = "some.machine.edu:333/PYRO" logger.debug("Calling '%s' request: %s\n%s ..." % (uri, s.__class__.__name__, s)) logger.close() logger.error("erroneous message")
to be run from the project root directory: python fdtcplib/common/test/logtest.py """ from fdtcplib.common.actions import SendingClientAction from fdtcplib.utils.Logger import Logger transferFiles = [ "/mnt/hadoop/user/uscms01/pnfs/unl.edu/data4/cms/store/phedex_monarctest/Nebraska/LoadTest07_Nebraska_65 / /mnt/hadoop/store/PhEDEx_LoadTest07/LoadTest07_Debug_US_Nebraska/US_Caltech/3962/LoadTest07_Nebraska_65_uprBpUmMbtrlF910_3962", "/mnt/hadoop/user/uscms01/pnfs/unl.edu/data4/cms/store/phedex_monarctest/Nebraska/LoadTest07_Nebraska_AB / /mnt/hadoop/store/PhEDEx_LoadTest07/LoadTest07_Debug_US_Nebraska/US_Caltech/3962/LoadTest07_Nebraska_AB_VBggLIppuOd19CPC_3962", "/mnt/hadoop/user/uscms01/pnfs/unl.edu/data4/cms/store/phedex_monarctest/Nebraska/LoadTest07_Nebraska_E3 / /mnt/hadoop/store/PhEDEx_LoadTest07/LoadTest07_Debug_US_Nebraska/US_Caltech/3962/LoadTest07_Nebraska_E3_nxS6pVoM1of60fNW_3962", "/mnt/hadoop/user/uscms01/pnfs/unl.edu/data4/cms/store/phedex_monarctest/Nebraska/LoadTest07_Nebraska_0B / /mnt/hadoop/store/PhEDEx_LoadTest07/LoadTest07_Debug_US_Nebraska/US_Caltech/3962/LoadTest07_Nebraska_0B_QIG6aG9hpal8XTJD_3962", "/mnt/hadoop/user/uscms01/pnfs/unl.edu/data4/cms/store/phedex_monarctest/Nebraska/LoadTest07_Nebraska_1B / /mnt/hadoop/store/PhEDEx_LoadTest07/LoadTest07_Debug_US_Nebraska/US_Caltech/3962/LoadTest07_Nebraska_1B_but3hjz0eT7bWHrr_3962", "/mnt/hadoop/user/uscms01/pnfs/unl.edu/data4/cms/store/phedex_monarctest/Nebraska/LoadTest07_Nebraska_39 / /mnt/hadoop/store/PhEDEx_LoadTest07/LoadTest07_Debug_US_Nebraska/US_Caltech/3962/LoadTest07_Nebraska_39_mcBzPcNt1pJcaAkc_3962", "/mnt/hadoop/user/uscms01/pnfs/unl.edu/data4/cms/store/phedex_monarctest/Nebraska/LoadTest07_Nebraska_81 / /mnt/hadoop/store/PhEDEx_LoadTest07/LoadTest07_Debug_US_Nebraska/US_Caltech/3962/LoadTest07_Nebraska_81_QueqvduaTBMjk3pu_3962", "/mnt/hadoop/user/uscms01/pnfs/unl.edu/data4/cms/store/phedex_monarctest/Nebraska/LoadTest07_Nebraska_CB / /mnt/hadoop/store/PhEDEx_LoadTest07/LoadTest07_Debug_US_Nebraska/US_Caltech/3962/LoadTest07_Nebraska_CB_YbQNLdVZnUIEukVe_3962", "/mnt/hadoop/user/uscms01/pnfs/unl.edu/data4/cms/store/phedex_monarctest/Nebraska/LoadTest07_Nebraska_1C / /mnt/hadoop/store/PhEDEx_LoadTest07/LoadTest07_Debug_US_Nebraska/US_Caltech/3962/LoadTest07_Nebraska_1C_3iyt9ymMVRpAVEWz_3962", "/mnt/hadoop/user/uscms01/pnfs/unl.edu/data4/cms/store/phedex_monarctest/Nebraska/LoadTest07_Nebraska_62 / /mnt/hadoop/store/PhEDEx_LoadTest07/LoadTest07_Debug_US_Nebraska/US_Caltech/3962/LoadTest07_Nebraska_62_wboDVRwpaSU8jU0O_3962", ] options = dict(port=54321, hostDest="gridftp05.ultralight.org", transferFiles=transferFiles, gridUserSrc="cmsphedex") s = SendingClientAction("fdtcp-cithep249.ultralight.org-phedex-2011-04-08--06h:19m:21s:822724mics-epjiu", options) # logger = Logger(name = "fdtcp", logFile = "/tmp/logfile") logger = Logger(name="fdtcp") uri = "some.machine.edu:333/PYRO" logger.debug("Calling '%s' request: %s\n%s ..." % (uri, s.__class__.__name__, s)) logger.close() logger.error("erroneous message")