def testFDTDKillProcess2():
c = """
[general]
port = 6700
debug = DEBUG
killCommand = ../wrapper_kill.sh %(pid)s
portRangeFDTServer = 54321,54400
"""
f = getTempFile(c)
inputOption = "--config=%s" % f.name
conf = ConfigFDTD(inputOption.split())
testName = inspect.stack()[0][3]
logger = Logger(name=testName, level=logging.DEBUG)
apMon = None
daemon = FDTD(conf, apMon, logger)
# need long-running, non blocking process
command = "dd if=/dev/zero of=/dev/null count=100000000 bs=102400"
executor = Executor("some_id", command, blocking=False, caller=daemon, logger=logger, killTimeout=0)
try:
executor.execute()
daemon.killProcess("some_id", logger)
finally:
# definitely release port and kill the process
daemon.shutdown()
daemon.pyroDaemon.closedown()
try:
p = Process(executor.proc.pid)
m = "FAIL: Process PID:%s should have been " "killed." % executor.proc.pid
logger.debug(m)
py.test.fail(m)
except NoSuchProcess as ex:
logger.debug("OK: Process PID:%s doesn't exist now." % executor.proc.pid)
def testFDTDServiceOpenFiles():
"""
#41 - Too many open files (fdtd side)
"""
hostName = os.uname()[1]
f = getTempFile(functionalFDTDConfiguration)
inputOption = "--config=%s" % f.name
conf = ConfigFDTD(inputOption.split())
conf.sanitize()
testName = inspect.stack()[0][3]
logger = Logger(name=testName,
logFile="/tmp/fdtdtest-%s.log" % testName,
level=logging.DEBUG)
apMon = None
fdtd = FDTD(conf, apMon, logger)
proc = Process(os.getpid())
initStateNumOpenFiles = len(proc.get_open_files())
for testAction in [TestAction("fakeSrc", "fakeDst") for i in range(3)]:
r = fdtd.service.service(testAction)
logger.debug("Result: %s" % r)
assert r.status == 0
# after TestAction, there should not be left behind any open files
numOpenFilesNow = len(proc.get_open_files())
assert initStateNumOpenFiles == numOpenFilesNow
# test on ReceivingServerAction - it's action after which the
# separate logger is not closed, test the number of open files went +1,
# send CleanupProcessesAction and shall again remain
# initStateNumOpenFiles send appropriate TestAction first (like in real)
serverId = "server-id"
testAction = TestAction(hostName, hostName)
testAction.id = serverId
r = fdtd.service.service(testAction)
assert r.status == 0
options = dict(gridUserDest="someuserDest",
clientIP=os.uname()[1],
destFiles=[])
recvServerAction = ReceivingServerAction(testAction.id, options)
r = fdtd.service.service(recvServerAction)
print r.msg
assert r.status == 0
numOpenFilesNow = len(proc.get_open_files())
# there should be only 1 extra opened file now
assert initStateNumOpenFiles == numOpenFilesNow - 1
cleanupAction = CleanupProcessesAction(serverId, timeout=2)
r = fdtd.service.service(cleanupAction)
print r.msg
assert r.status == 0
numOpenFilesNow = len(proc.get_open_files())
assert initStateNumOpenFiles == numOpenFilesNow
fdtd.shutdown()
fdtd.pyroDaemon.closedown()
logger.close()
def testLogger():
logger = Logger("test logger", level=logging.DEBUG)
logger.info("info message")
logger.warn("warn message")
logger.warning("warning message")
logger.error("error message")
logger.critical("critical message")
logger.fatal("fatal message")
logger.debug("debug message")
logger.close()
def testLogger():
logger = Logger("test logger", level=logging.DEBUG)
logger.info("info message")
logger.warn("warn message")
logger.warning("warning message")
logger.error("error message")
logger.critical("critical message")
logger.fatal("fatal message")
logger.debug("debug message")
logger.close()
def testFDTDServiceOpenFiles():
"""
#41 - Too many open files (fdtd side)
"""
hostName = os.uname()[1]
f = getTempFile(functionalFDTDConfiguration)
inputOption = "--config=%s" % f.name
conf = ConfigFDTD(inputOption.split())
conf.sanitize()
testName = inspect.stack()[0][3]
logger = Logger(name=testName, logFile="/tmp/fdtdtest-%s.log" % testName, level=logging.DEBUG)
apMon = None
fdtd = FDTD(conf, apMon, logger)
proc = Process(os.getpid())
initStateNumOpenFiles = len(proc.get_open_files())
for testAction in [TestAction("fakeSrc", "fakeDst") for i in range(3)]:
r = fdtd.service.service(testAction)
logger.debug("Result: %s" % r)
assert r.status == 0
# after TestAction, there should not be left behind any open files
numOpenFilesNow = len(proc.get_open_files())
assert initStateNumOpenFiles == numOpenFilesNow
# test on ReceivingServerAction - it's action after which the
# separate logger is not closed, test the number of open files went +1,
# send CleanupProcessesAction and shall again remain
# initStateNumOpenFiles send appropriate TestAction first (like in real)
serverId = "server-id"
testAction = TestAction(hostName, hostName)
testAction.id = serverId
r = fdtd.service.service(testAction)
assert r.status == 0
options = dict(gridUserDest="someuserDest", clientIP=os.uname()[1], destFiles=[])
recvServerAction = ReceivingServerAction(testAction.id, options)
r = fdtd.service.service(recvServerAction)
print(r.msg)
assert r.status == 0
numOpenFilesNow = len(proc.get_open_files())
# there should be only 1 extra opened file now
assert initStateNumOpenFiles == numOpenFilesNow - 1
cleanupAction = CleanupProcessesAction(serverId, timeout=2)
r = fdtd.service.service(cleanupAction)
print(r.msg)
assert r.status == 0
numOpenFilesNow = len(proc.get_open_files())
assert initStateNumOpenFiles == numOpenFilesNow
fdtd.shutdown()
fdtd.pyroDaemon.closedown()
logger.close()
def testReceivingServerActionCheckTargetFileNames():
logger = Logger()
files = ["/mnt/data", "/etc/passwd", "/etc/something/nonsence", "/tmp"]
options = dict(port=1000, gridUserDest="someuser", destFiles=files)
a = ReceivingServerAction("some_id", options)
logger.info("%s - checking presence of files at target location ..." %
a.__class__.__name__)
r = a._checkTargetFileNames(files)
logger.debug("Results:\n%s" % r)
expected = \
""" exists True: /mnt/data
exists False: /mnt/.data
exists True: /etc/passwd
exists False: /etc/.passwd
exists False: /etc/something/nonsence
exists False: /etc/something/.nonsence
exists True: /tmp
exists False: /.tmp
"""
assert r == expected
def testReceivingServerActionCheckTargetFileNames():
logger = Logger()
files = ["/mnt/data", "/etc/passwd", "/etc/something/nonsence", "/tmp"]
options = dict(port=1000, gridUserDest="someuser", destFiles=files)
a = ReceivingServerAction("some_id", options)
logger.info("%s - checking presence of files at target location ..." %
a.__class__.__name__)
r = a._checkTargetFileNames(files)
logger.debug("Results:\n%s" % r)
expected = \
""" exists True: /mnt/data
exists False: /mnt/.data
exists True: /etc/passwd
exists False: /etc/.passwd
exists False: /etc/something/nonsence
exists False: /etc/something/.nonsence
exists True: /tmp
exists False: /.tmp
"""
assert r == expected
def testFDTDKillProcess2():
c = """
[general]
port = 6700
debug = DEBUG
killCommand = ../wrapper_kill.sh %(pid)s
portRangeFDTServer = 54321,54400
"""
f = getTempFile(c)
inputOption = "--config=%s" % f.name
conf = ConfigFDTD(inputOption.split())
testName = inspect.stack()[0][3]
logger = Logger(name=testName, level=logging.DEBUG)
apMon = None
daemon = FDTD(conf, apMon, logger)
# need long-running, non blocking process
command = "dd if=/dev/zero of=/dev/null count=100000000 bs=102400"
executor = Executor("some_id",
command,
blocking=False,
caller=daemon,
logger=logger,
killTimeout=0)
try:
executor.execute()
daemon.killProcess("some_id", logger)
finally:
# definitely release port and kill the process
daemon.shutdown()
daemon.pyroDaemon.closedown()
try:
p = Process(executor.proc.pid)
m = ("FAIL: Process PID:%s should have been "
"killed." % executor.proc.pid)
logger.debug(m)
py.test.fail(m)
except NoSuchProcess, ex:
logger.debug("OK: Process PID:%s doesn't exist now." %
executor.proc.pid)
def testLoggerTraceBackTrueOnNoException():
logger = Logger("test file logger", level=logging.DEBUG)
logger.debug("my msg", traceBack=True)
conf = ConfigFDTCopy(sys.argv[1:])
conf.sanitize()
except ConfigurationException, ex:
print "fdtcp failed to start, reason: %s" % ex
sys.exit(1)
logger = Logger(name="fdtcp",
logFile=conf.get("logFile"),
level=conf.get("debug"))
# ticket #35 - mercurial expandable keywords in the code
# information from the SCM (expandable keywords)
versionInfo = dict(Revision="$Revision: cf25e731af0a $",
Tags="$Tags: tip $")
logger.info("fdtcp starting ... version: %s" %
logger.pprintFormat(versionInfo))
logger.debug("Search sys.path:\n%s\n" % logger.pprintFormat(sys.path))
logger.debug("PYRO_STORAGE: '%s'" % os.environ.get("PYRO_STORAGE"))
logger.debug("Input command line arguments: '%s'" % optBackup)
logger.debug("Configuration values (processed):\n%s\n" %
logger.pprintFormat(conf._options))
apMon = None
apMonDestConf = conf.get("apMonDestinations")
if apMonDestConf:
apMonDestinations = tuple(apMonDestConf.split(','))
logger.info("Initializing MonALISA ApMon, destinations: %s ..." %
(apMonDestinations, ))
apMon = apmon.ApMon(apMonDestinations)
apMon.enableBgMonitoring(True)
else:
def testLoggerTraceBackTrueOnNoException():
logger = Logger("test file logger", level=logging.DEBUG)
logger.debug("my msg", traceBack=True)
def testMainLogFileOpeningDuringDaemonisation():
py.test.skip("Test skipped, messes up with input/output streams for "
"other test, better to run it stand-alone: "
"py.test fdtcplib/test/test_fdtd.py -s -k testMainLogFileOpeningDuringDaemonisation")
"""
As described in the problem #41:comment:9, the main log file remains
open thrice (under different descriptor) after initial daemonisation.
In order to include this particular test into whole test suite running
(issue with file descriptors being redirected, etc in fdtd.daemonize()
function), perhaps after running this test, the file descriptors
can be put back from backup sys.__stdout__, etc?
"""
c = """
[general]
logFile = /tmp/fdtd.log
pidFile = /tmp/fdtd.pid
"""
testName = inspect.stack()[0][3]
f = getTempFile(c)
inputOption = "--config=%s" % f.name
conf = ConfigFDTD(inputOption.split())
f.close()
logFile = "%s-%s" % (conf.get("logFile"), testName)
pidFile = conf.get("pidFile")
if os.path.exists(logFile):
print("test: %s: file '%s' exists, removing ..." %
(testName, logFile))
os.remove(logFile)
if os.path.exists(pidFile):
print("test: %s: file '%s' exists, removing ..." %
(testName, pidFile))
os.remove(pidFile)
logger = Logger(name=testName, logFile=logFile, level=logging.DEBUG)
logger.debug("Before daemonization ...")
numFiles, filesStr = getOpenFilesList()
logger.debug("Logging open files: %s items:\n%s" % (numFiles, filesStr))
try:
daemonize(conf, logger)
except SystemExit:
# previous leading process continues here
return
# here continues the newly created background daemon
f = open(pidFile, 'r')
rc = f.read()
f.close()
rc = rc.strip()
pid = int(rc)
numFiles, filesStr = getOpenFilesList()
logger.debug("Logging open files: %s items:\n%s" % (numFiles, filesStr))
logger.debug("Before finishing ... ")
logger.close()
# the log file may be open more times due to streams descriptor
# duplication as done in fdtd.daemonization() but now, once is
# closed, there should not any other outstanding open file
assert numFiles == 0
def testMainLogFileOpeningDuringDaemonisation():
py.test.skip(
"Test skipped, messes up with input/output streams for "
"other test, better to run it stand-alone: "
"py.test fdtcplib/test/test_fdtd.py -s -k testMainLogFileOpeningDuringDaemonisation"
)
"""
As described in the problem #41:comment:9, the main log file remains
open thrice (under different descriptor) after initial daemonisation.
In order to include this particular test into whole test suite running
(issue with file descriptors being redirected, etc in fdtd.daemonize()
function), perhaps after running this test, the file descriptors
can be put back from backup sys.__stdout__, etc?
"""
c = """
[general]
logFile = /tmp/fdtd.log
pidFile = /tmp/fdtd.pid
"""
testName = inspect.stack()[0][3]
f = getTempFile(c)
inputOption = "--config=%s" % f.name
conf = ConfigFDTD(inputOption.split())
f.close()
logFile = "%s-%s" % (conf.get("logFile"), testName)
pidFile = conf.get("pidFile")
if os.path.exists(logFile):
print("test: %s: file '%s' exists, removing ..." % (testName, logFile))
os.remove(logFile)
if os.path.exists(pidFile):
print("test: %s: file '%s' exists, removing ..." % (testName, pidFile))
os.remove(pidFile)
logger = Logger(name=testName, logFile=logFile, level=logging.DEBUG)
logger.debug("Before daemonization ...")
numFiles, filesStr = getOpenFilesList()
logger.debug("Logging open files: %s items:\n%s" % (numFiles, filesStr))
try:
daemonize(conf, logger)
except SystemExit:
# previous leading process continues here
return
# here continues the newly created background daemon
f = open(pidFile, "r")
rc = f.read()
f.close()
rc = rc.strip()
pid = int(rc)
numFiles, filesStr = getOpenFilesList()
logger.debug("Logging open files: %s items:\n%s" % (numFiles, filesStr))
logger.debug("Before finishing ... ")
logger.close()
# the log file may be open more times due to streams descriptor
# duplication as done in fdtd.daemonization() but now, once is
# closed, there should not any other outstanding open file
assert numFiles == 0
def main():
# all values and action information held in the conf object
optBackup = sys.argv[:]
try:
conf = ConfigFDTCopy(sys.argv[1:])
conf.sanitize()
except ConfigurationException, ex:
print "fdtcp failed to start, reason: %s" % ex
sys.exit(1)
logger = Logger(name = "fdtcp", logFile = conf.get("logFile"),
level = conf.get("debug"))
logger.info("fdtcp starting ...")
logger.debug("Search sys.path: '%s'" % sys.path)
logger.debug("PYRO_STORAGE: '%s'" % os.environ.get("PYRO_STORAGE"))
logger.debug("Input command line arguments: '%s'" % optBackup)
logger.debug("Configuration values (processed): '%s'" % conf._options)
apMonDestConf = conf.get("apMonDestinations")
apMonDestinations = tuple(apMonDestConf.split(','))
logger.info("Initializing MonALISA ApMon, destinations: %s ..." % (apMonDestinations,))
apMon = apmon.ApMon(apMonDestinations)
apMon.enableBgMonitoring(True)
# use DNS names rather than IP address
Pyro.config.PYRO_DNS_URI = True
from fdtcplib.utils.Logger import Logger
transferFiles = [
'/mnt/hadoop/user/uscms01/pnfs/unl.edu/data4/cms/store/phedex_monarctest/Nebraska/LoadTest07_Nebraska_65 / /mnt/hadoop/store/PhEDEx_LoadTest07/LoadTest07_Debug_US_Nebraska/US_Caltech/3962/LoadTest07_Nebraska_65_uprBpUmMbtrlF910_3962',
'/mnt/hadoop/user/uscms01/pnfs/unl.edu/data4/cms/store/phedex_monarctest/Nebraska/LoadTest07_Nebraska_AB / /mnt/hadoop/store/PhEDEx_LoadTest07/LoadTest07_Debug_US_Nebraska/US_Caltech/3962/LoadTest07_Nebraska_AB_VBggLIppuOd19CPC_3962',
'/mnt/hadoop/user/uscms01/pnfs/unl.edu/data4/cms/store/phedex_monarctest/Nebraska/LoadTest07_Nebraska_E3 / /mnt/hadoop/store/PhEDEx_LoadTest07/LoadTest07_Debug_US_Nebraska/US_Caltech/3962/LoadTest07_Nebraska_E3_nxS6pVoM1of60fNW_3962',
'/mnt/hadoop/user/uscms01/pnfs/unl.edu/data4/cms/store/phedex_monarctest/Nebraska/LoadTest07_Nebraska_0B / /mnt/hadoop/store/PhEDEx_LoadTest07/LoadTest07_Debug_US_Nebraska/US_Caltech/3962/LoadTest07_Nebraska_0B_QIG6aG9hpal8XTJD_3962',
'/mnt/hadoop/user/uscms01/pnfs/unl.edu/data4/cms/store/phedex_monarctest/Nebraska/LoadTest07_Nebraska_1B / /mnt/hadoop/store/PhEDEx_LoadTest07/LoadTest07_Debug_US_Nebraska/US_Caltech/3962/LoadTest07_Nebraska_1B_but3hjz0eT7bWHrr_3962',
'/mnt/hadoop/user/uscms01/pnfs/unl.edu/data4/cms/store/phedex_monarctest/Nebraska/LoadTest07_Nebraska_39 / /mnt/hadoop/store/PhEDEx_LoadTest07/LoadTest07_Debug_US_Nebraska/US_Caltech/3962/LoadTest07_Nebraska_39_mcBzPcNt1pJcaAkc_3962',
'/mnt/hadoop/user/uscms01/pnfs/unl.edu/data4/cms/store/phedex_monarctest/Nebraska/LoadTest07_Nebraska_81 / /mnt/hadoop/store/PhEDEx_LoadTest07/LoadTest07_Debug_US_Nebraska/US_Caltech/3962/LoadTest07_Nebraska_81_QueqvduaTBMjk3pu_3962',
'/mnt/hadoop/user/uscms01/pnfs/unl.edu/data4/cms/store/phedex_monarctest/Nebraska/LoadTest07_Nebraska_CB / /mnt/hadoop/store/PhEDEx_LoadTest07/LoadTest07_Debug_US_Nebraska/US_Caltech/3962/LoadTest07_Nebraska_CB_YbQNLdVZnUIEukVe_3962',
'/mnt/hadoop/user/uscms01/pnfs/unl.edu/data4/cms/store/phedex_monarctest/Nebraska/LoadTest07_Nebraska_1C / /mnt/hadoop/store/PhEDEx_LoadTest07/LoadTest07_Debug_US_Nebraska/US_Caltech/3962/LoadTest07_Nebraska_1C_3iyt9ymMVRpAVEWz_3962',
'/mnt/hadoop/user/uscms01/pnfs/unl.edu/data4/cms/store/phedex_monarctest/Nebraska/LoadTest07_Nebraska_62 / /mnt/hadoop/store/PhEDEx_LoadTest07/LoadTest07_Debug_US_Nebraska/US_Caltech/3962/LoadTest07_Nebraska_62_wboDVRwpaSU8jU0O_3962'
]
options = dict(port=54321,
hostDest='gridftp05.ultralight.org',
transferFiles=transferFiles,
gridUserSrc='cmsphedex')
s = SendingClientAction(
'fdtcp-cithep249.ultralight.org-phedex-2011-04-08--06h:19m:21s:822724mics-epjiu',
options)
#logger = Logger(name = "fdtcp", logFile = "/tmp/logfile")
logger = Logger(name="fdtcp")
uri = "some.machine.edu:333/PYRO"
logger.debug("Calling '%s' request: %s\n%s ..." %
(uri, s.__class__.__name__, s))
logger.close()
logger.error("erroneous message")
to be run from the project root directory:
python fdtcplib/common/test/logtest.py
"""
from fdtcplib.common.actions import SendingClientAction
from fdtcplib.utils.Logger import Logger
transferFiles = [
"/mnt/hadoop/user/uscms01/pnfs/unl.edu/data4/cms/store/phedex_monarctest/Nebraska/LoadTest07_Nebraska_65 / /mnt/hadoop/store/PhEDEx_LoadTest07/LoadTest07_Debug_US_Nebraska/US_Caltech/3962/LoadTest07_Nebraska_65_uprBpUmMbtrlF910_3962",
"/mnt/hadoop/user/uscms01/pnfs/unl.edu/data4/cms/store/phedex_monarctest/Nebraska/LoadTest07_Nebraska_AB / /mnt/hadoop/store/PhEDEx_LoadTest07/LoadTest07_Debug_US_Nebraska/US_Caltech/3962/LoadTest07_Nebraska_AB_VBggLIppuOd19CPC_3962",
"/mnt/hadoop/user/uscms01/pnfs/unl.edu/data4/cms/store/phedex_monarctest/Nebraska/LoadTest07_Nebraska_E3 / /mnt/hadoop/store/PhEDEx_LoadTest07/LoadTest07_Debug_US_Nebraska/US_Caltech/3962/LoadTest07_Nebraska_E3_nxS6pVoM1of60fNW_3962",
"/mnt/hadoop/user/uscms01/pnfs/unl.edu/data4/cms/store/phedex_monarctest/Nebraska/LoadTest07_Nebraska_0B / /mnt/hadoop/store/PhEDEx_LoadTest07/LoadTest07_Debug_US_Nebraska/US_Caltech/3962/LoadTest07_Nebraska_0B_QIG6aG9hpal8XTJD_3962",
"/mnt/hadoop/user/uscms01/pnfs/unl.edu/data4/cms/store/phedex_monarctest/Nebraska/LoadTest07_Nebraska_1B / /mnt/hadoop/store/PhEDEx_LoadTest07/LoadTest07_Debug_US_Nebraska/US_Caltech/3962/LoadTest07_Nebraska_1B_but3hjz0eT7bWHrr_3962",
"/mnt/hadoop/user/uscms01/pnfs/unl.edu/data4/cms/store/phedex_monarctest/Nebraska/LoadTest07_Nebraska_39 / /mnt/hadoop/store/PhEDEx_LoadTest07/LoadTest07_Debug_US_Nebraska/US_Caltech/3962/LoadTest07_Nebraska_39_mcBzPcNt1pJcaAkc_3962",
"/mnt/hadoop/user/uscms01/pnfs/unl.edu/data4/cms/store/phedex_monarctest/Nebraska/LoadTest07_Nebraska_81 / /mnt/hadoop/store/PhEDEx_LoadTest07/LoadTest07_Debug_US_Nebraska/US_Caltech/3962/LoadTest07_Nebraska_81_QueqvduaTBMjk3pu_3962",
"/mnt/hadoop/user/uscms01/pnfs/unl.edu/data4/cms/store/phedex_monarctest/Nebraska/LoadTest07_Nebraska_CB / /mnt/hadoop/store/PhEDEx_LoadTest07/LoadTest07_Debug_US_Nebraska/US_Caltech/3962/LoadTest07_Nebraska_CB_YbQNLdVZnUIEukVe_3962",
"/mnt/hadoop/user/uscms01/pnfs/unl.edu/data4/cms/store/phedex_monarctest/Nebraska/LoadTest07_Nebraska_1C / /mnt/hadoop/store/PhEDEx_LoadTest07/LoadTest07_Debug_US_Nebraska/US_Caltech/3962/LoadTest07_Nebraska_1C_3iyt9ymMVRpAVEWz_3962",
"/mnt/hadoop/user/uscms01/pnfs/unl.edu/data4/cms/store/phedex_monarctest/Nebraska/LoadTest07_Nebraska_62 / /mnt/hadoop/store/PhEDEx_LoadTest07/LoadTest07_Debug_US_Nebraska/US_Caltech/3962/LoadTest07_Nebraska_62_wboDVRwpaSU8jU0O_3962",
]
options = dict(port=54321, hostDest="gridftp05.ultralight.org", transferFiles=transferFiles, gridUserSrc="cmsphedex")
s = SendingClientAction("fdtcp-cithep249.ultralight.org-phedex-2011-04-08--06h:19m:21s:822724mics-epjiu", options)
# logger = Logger(name = "fdtcp", logFile = "/tmp/logfile")
logger = Logger(name="fdtcp")
uri = "some.machine.edu:333/PYRO"
logger.debug("Calling '%s' request: %s\n%s ..." % (uri, s.__class__.__name__, s))
logger.close()
logger.error("erroneous message")