def testLogIntoClosedLogFile():
"""
Test writing into a closed logger. The logger should handle.
"""
testFile = TEST_LOG_FILE
for met, level in (('warning', logging.WARNING), ('warn', logging.WARNING),
('fatal', logging.FATAL), ('error', logging.ERROR),
('debug', logging.DEBUG),
('critical', logging.CRITICAL), ('info', logging.INFO)):
if os.path.exists(testFile):
m = "Can't do logging into file test, file %s exists." % testFile
py.test.fail(m)
logger = Logger("test file logger", logFile=testFile, level=level)
testMessage = "testLogIntoClosedLogFile-%s" % met
c = getattr(logger, met)
logger.close()
# log after closing the logger
c(testMessage)
# message to search for
toSearchMsg = ("Attempt to log into already closed.*$\n.*%s" %
testMessage)
pattObj = re.compile(toSearchMsg, re.MULTILINE)
fd = open(testFile, 'r')
match = pattObj.search(fd.read())
if not match:
m = ("Log file '%s' should contain log message '%s' - does "
"not." % (testFile, toSearchMsg))
py.test.fail(m)
os.remove(testFile)
def testLogger():
logger = Logger("test logger", level = logging.DEBUG)
logger.info("info message")
logger.warn("warn message")
logger.error("error message")
logger.fatal("fatal message")
logger.close()
def testLogIntoClosedLogFile():
"""
Test writing into a closed logger. The logger should handle.
"""
testFile = TEST_LOG_FILE
for met, level in (('warning', logging.WARNING),
('warn', logging.WARNING),
('fatal', logging.FATAL),
('error', logging.ERROR),
('debug', logging.DEBUG),
('critical', logging.CRITICAL),
('info', logging.INFO)):
if os.path.exists(testFile):
m = "Can't do logging into file test, file %s exists." % testFile
py.test.fail(m)
logger = Logger("test file logger", logFile=testFile, level=level)
testMessage = "testLogIntoClosedLogFile-%s" % met
c = getattr(logger, met)
logger.close()
# log after closing the logger
c(testMessage)
# message to search for
toSearchMsg = ("Attempt to log into already closed.*$\n.*%s" %
testMessage)
pattObj = re.compile(toSearchMsg, re.MULTILINE)
fd = open(testFile, 'r')
match = pattObj.search(fd.read())
if not match:
m = ("Log file '%s' should contain log message '%s' - does "
"not." % (testFile, toSearchMsg))
py.test.fail(m)
os.remove(testFile)
def testTracebackComplex():
logger = Logger("test logger", level=logging.DEBUG)
try:
old_div(1, 0)
except Exception:
logger.getTracebackComplex()
logger.close()
def testTracebackComplex():
logger = Logger("test logger", level=logging.DEBUG)
try:
old_div(1, 0)
except Exception:
logger.getTracebackComplex()
logger.close()
def testAddressAlreadyInUseRoundRobinPortReservation():
"""
#38 - Address already in use FDT Java
https://trac.hep.caltech.edu/trac/fdtcp/ticket/38
Address already in use problem was seen during #5:comment:20
https://trac.hep.caltech.edu/trac/fdtcp/ticket/5#comment:20
2 times out of 338 transfer (attempts). Probably, when there is
traffic the port can't be bound immediately even if it was
released very short ago by the previous process.
This test could not reproduce the problem (when reusing immediately
the same port number for the next request), so FDTD.getFreePort()
was reimplemented to reserver ports on round-robin basis.
"""
hostName = os.uname()[1]
f = getTempFile(functionalFDTDConfiguration)
inputOption = "--config=%s" % f.name
conf = ConfigFDTD(inputOption.split())
conf.sanitize()
testName = inspect.stack()[0][3]
logger = Logger(name=testName, logFile="/tmp/fdtdtest-%s.log" % testName, level=logging.DEBUG)
apMon = None
fdtd = FDTD(conf, apMon, logger)
# launch two subsequent ReceivingServerAction, second will likely fail
# to bind the same, just very short ago, released port
serverId = "%s" % testName
testAction = TestAction(hostName, hostName)
testAction.id = serverId
# do TestAction
r = fdtd.service.service(testAction)
assert r.status == 0
options = dict(gridUserDest="someuserDest", clientIP=os.uname()[1], destFiles=[])
recvServerAction = ReceivingServerAction(testAction.id, options)
# do ReceivingServerAction - start FDT Java server
r = fdtd.service.service(recvServerAction)
print(r.msg)
assert r.status == 0
assert r.serverPort == 54321
cleanupAction = CleanupProcessesAction(serverId, timeout=0, waitTimeout=False)
# do CleanupProcessesAction - shut FDT Java server, port shall be
# released
r = fdtd.service.service(cleanupAction)
print(r.msg)
assert r.status == 0
# do another ReceivingServerAction - start FDT Java server
r = fdtd.service.service(recvServerAction)
print(r.msg)
assert r.status == 0
# will not get the same port, but the next one in the range
assert r.serverPort == 54322
# in fact, if separate log files are enabled, after this last
# ReceivingServerAction, there is a separate log file open.
# taking the the service down, it should also closed it's related
# to open files #41 problem
fdtd.shutdown()
fdtd.pyroDaemon.closedown()
logger.close()
def testFDTDServiceOpenFiles():
"""
#41 - Too many open files (fdtd side)
"""
hostName = os.uname()[1]
f = getTempFile(functionalFDTDConfiguration)
inputOption = "--config=%s" % f.name
conf = ConfigFDTD(inputOption.split())
conf.sanitize()
testName = inspect.stack()[0][3]
logger = Logger(name=testName,
logFile="/tmp/fdtdtest-%s.log" % testName,
level=logging.DEBUG)
apMon = None
fdtd = FDTD(conf, apMon, logger)
proc = Process(os.getpid())
initStateNumOpenFiles = len(proc.get_open_files())
for testAction in [TestAction("fakeSrc", "fakeDst") for i in range(3)]:
r = fdtd.service.service(testAction)
logger.debug("Result: %s" % r)
assert r.status == 0
# after TestAction, there should not be left behind any open files
numOpenFilesNow = len(proc.get_open_files())
assert initStateNumOpenFiles == numOpenFilesNow
# test on ReceivingServerAction - it's action after which the
# separate logger is not closed, test the number of open files went +1,
# send CleanupProcessesAction and shall again remain
# initStateNumOpenFiles send appropriate TestAction first (like in real)
serverId = "server-id"
testAction = TestAction(hostName, hostName)
testAction.id = serverId
r = fdtd.service.service(testAction)
assert r.status == 0
options = dict(gridUserDest="someuserDest",
clientIP=os.uname()[1],
destFiles=[])
recvServerAction = ReceivingServerAction(testAction.id, options)
r = fdtd.service.service(recvServerAction)
print r.msg
assert r.status == 0
numOpenFilesNow = len(proc.get_open_files())
# there should be only 1 extra opened file now
assert initStateNumOpenFiles == numOpenFilesNow - 1
cleanupAction = CleanupProcessesAction(serverId, timeout=2)
r = fdtd.service.service(cleanupAction)
print r.msg
assert r.status == 0
numOpenFilesNow = len(proc.get_open_files())
assert initStateNumOpenFiles == numOpenFilesNow
fdtd.shutdown()
fdtd.pyroDaemon.closedown()
logger.close()
def testTracebackSimple():
logger = Logger("test logger", level=logging.DEBUG)
try:
old_div(1, 0)
except Exception:
logger.getTracebackSimple()
logger.error("exception", traceBack=True)
logger.fatal("exception", traceBack=True)
logger.close()
def testTracebackSimple():
logger = Logger("test logger", level=logging.DEBUG)
try:
old_div(1, 0)
except Exception:
logger.getTracebackSimple()
logger.error("exception", traceBack=True)
logger.fatal("exception", traceBack=True)
logger.close()
def testLogger():
logger = Logger("test logger", level=logging.DEBUG)
logger.info("info message")
logger.warn("warn message")
logger.warning("warning message")
logger.error("error message")
logger.critical("critical message")
logger.fatal("fatal message")
logger.debug("debug message")
logger.close()
def testFDTDServiceOpenFiles():
"""
#41 - Too many open files (fdtd side)
"""
hostName = os.uname()[1]
f = getTempFile(functionalFDTDConfiguration)
inputOption = "--config=%s" % f.name
conf = ConfigFDTD(inputOption.split())
conf.sanitize()
testName = inspect.stack()[0][3]
logger = Logger(name=testName, logFile="/tmp/fdtdtest-%s.log" % testName, level=logging.DEBUG)
apMon = None
fdtd = FDTD(conf, apMon, logger)
proc = Process(os.getpid())
initStateNumOpenFiles = len(proc.get_open_files())
for testAction in [TestAction("fakeSrc", "fakeDst") for i in range(3)]:
r = fdtd.service.service(testAction)
logger.debug("Result: %s" % r)
assert r.status == 0
# after TestAction, there should not be left behind any open files
numOpenFilesNow = len(proc.get_open_files())
assert initStateNumOpenFiles == numOpenFilesNow
# test on ReceivingServerAction - it's action after which the
# separate logger is not closed, test the number of open files went +1,
# send CleanupProcessesAction and shall again remain
# initStateNumOpenFiles send appropriate TestAction first (like in real)
serverId = "server-id"
testAction = TestAction(hostName, hostName)
testAction.id = serverId
r = fdtd.service.service(testAction)
assert r.status == 0
options = dict(gridUserDest="someuserDest", clientIP=os.uname()[1], destFiles=[])
recvServerAction = ReceivingServerAction(testAction.id, options)
r = fdtd.service.service(recvServerAction)
print(r.msg)
assert r.status == 0
numOpenFilesNow = len(proc.get_open_files())
# there should be only 1 extra opened file now
assert initStateNumOpenFiles == numOpenFilesNow - 1
cleanupAction = CleanupProcessesAction(serverId, timeout=2)
r = fdtd.service.service(cleanupAction)
print(r.msg)
assert r.status == 0
numOpenFilesNow = len(proc.get_open_files())
assert initStateNumOpenFiles == numOpenFilesNow
fdtd.shutdown()
fdtd.pyroDaemon.closedown()
logger.close()
def testLoggerFile():
testFile = "testlogfile.log"
if not os.path.exists(testFile):
logger = Logger("test file logger", logFile = testFile,
level = logging.INFO)
testMessage = "testmessage"
logger.info(testMessage)
line = open(testFile, 'r').readlines()
os.remove(testFile)
last = line[-1].split()[-1]
logger.close()
assert testMessage == last
else:
m = "Can't do logging into file test, file %s exists." % testFile
py.test.fail(m)
def testLoggerFile():
testFile = TEST_LOG_FILE
if not os.path.exists(testFile):
logger = Logger("test file logger",
logFile=testFile,
level=logging.INFO)
testMessage = "testmessage"
logger.info(testMessage)
logger.close()
line = open(testFile, 'r').readlines()
os.remove(testFile)
# not the last line which the logger closing plus 3 x \n , but
# line before that
last = line[-5].split()[-1]
assert testMessage == last
else:
m = "Can't do logging into file test, file %s exists." % testFile
py.test.fail(m)
def testLoggerFile():
testFile = TEST_LOG_FILE
if not os.path.exists(testFile):
logger = Logger("test file logger",
logFile=testFile,
level=logging.INFO)
testMessage = "testmessage"
logger.info(testMessage)
logger.close()
line = open(testFile, 'r').readlines()
os.remove(testFile)
# not the last line which the logger closing plus 3 x \n , but
# line before that
last = line[-5].split()[-1]
assert testMessage == last
else:
m = "Can't do logging into file test, file %s exists." % testFile
py.test.fail(m)
def testFDTDServiceOpenFilesFullTransfer():
"""
#41:comment:8 - Too many open files (fdtd side)
SendingClient actually removed itself from the executors container
once it finishes so subsequent CleanupProcessesAction doesn't know
about this process, nor about its open separate log file, which
doesn't get closed.
Simulate a simple successful transfer, send all actions and
check number of open files - does all as it happens in fdtd.service()
"""
hostName = os.uname()[1]
testName = inspect.stack()[0][3]
initStateNumOpenFilesTestStart, filesStr = getOpenFilesList()
print("%s: test 0: open files: %s items:\n%s" % (testName, initStateNumOpenFilesTestStart, filesStr))
# there should not be any open files now
assert initStateNumOpenFilesTestStart == 0
f = getTempFile(functionalFDTDConfiguration)
inputOption = "--config=%s --port=10001" % f.name
confServer = ConfigFDTD(inputOption.split())
confServer.sanitize()
loggerServer = Logger(name=testName, logFile="/tmp/fdtdtest-%s-writer.log" % testName, level=logging.DEBUG)
apMon = None
fdtdServer = FDTD(confServer, apMon, loggerServer)
inputOption = "--config=%s --port=10002" % f.name
confReader = ConfigFDTD(inputOption.split())
confReader.sanitize()
loggerReader = Logger(name=testName, logFile="/tmp/fdtdtest-%s-reader.log" % testName, level=logging.DEBUG)
apMon = None
fdtdReader = FDTD(confReader, apMon, loggerReader)
# -2 open log files, additional -1 is the temp config file
initStateNumOpenFiles, filesStr = getOpenFilesList()
print("%s: test 1: open files: %s items:\n%s" % (testName, initStateNumOpenFiles, filesStr))
assert initStateNumOpenFilesTestStart == initStateNumOpenFiles - 2 - 1
testActionServer = TestAction(hostName, hostName)
testActionServer.id = testActionServer.id + "-writer"
r = fdtdServer.service.service(testActionServer)
assert r.status == 0
options = dict(gridUserDest="someuserDest", clientIP=os.uname()[1], destFiles=["/dev/null"])
recvServerAction = ReceivingServerAction(testActionServer.id, options)
r = fdtdServer.service.service(recvServerAction)
print(r.msg)
assert r.status == 0
serverFDTPort = r.serverPort
# there should be only 1 extra opened file now - ReceivingServerAction
# separate log
numOpenFilesNow, filesStr = getOpenFilesList()
print("%s: test 2: open files: %s items:\n%s" % (testName, numOpenFilesNow, filesStr))
assert initStateNumOpenFiles == numOpenFilesNow - 1
testActionReader = TestAction(hostName, hostName)
testActionReader.id = testActionReader.id + "-reader"
r = fdtdReader.service.service(testActionReader)
assert r.status == 0
files = [TransferFile("/etc/passwd", "/dev/null")] # list of TransferFile
options = dict(port=serverFDTPort, hostDest=os.uname()[1], transferFiles=files, gridUserSrc="soemuserSrc")
sndClientAction = SendingClientAction(testActionReader.id, options)
r = fdtdReader.service.service(sndClientAction)
assert r.status == 0
# there should be +2 extra - for separate both server and client
numOpenFilesNow, filesStr = getOpenFilesList()
print("%s: test 3: open files: %s items:\n%s" % (testName, numOpenFilesNow, filesStr))
# 2 extra files - separate transfer log at both ends
assert initStateNumOpenFiles == numOpenFilesNow - 2
# now the transfer is over, both server (writer) and sender (reader)
# parties kept their separate log files open, CleanupProcessesAction
# will close them
print("going to clean up")
cl = CleanupProcessesAction(testActionReader.id, waitTimeout=False)
r = fdtdReader.service.service(cl)
assert r.status == 0
# one shall be closed now
numOpenFilesNow, filesStr = getOpenFilesList()
print("%s: test 4: open files: %s items:\n%s" % (testName, numOpenFilesNow, filesStr))
assert initStateNumOpenFiles == numOpenFilesNow - 1
cl = CleanupProcessesAction(testActionServer.id, waitTimeout=False)
r = fdtdServer.service.service(cl)
assert r.status == 0
# both separate log files should be closed now
# problem #41:comment:8 was here - server behaved correctly, but
# reader kept its separate log file open
numOpenFilesNow, filesStr = getOpenFilesList()
print("%s: test 5: open files: %s items:\n%s" % (testName, numOpenFilesNow, filesStr))
assert initStateNumOpenFiles == numOpenFilesNow
fdtdServer.shutdown()
fdtdServer.pyroDaemon.closedown()
loggerServer.close()
fdtdReader.shutdown()
fdtdReader.pyroDaemon.closedown()
loggerReader.close()
# after even log files were closed, etc
numOpenFilesNow, filesStr = getOpenFilesList()
print("%s: test 6: open files: %s items:\n%s" % (testName, numOpenFilesNow, filesStr))
# -1: the temp configuration file is still open
assert initStateNumOpenFilesTestStart == numOpenFilesNow - 1
try:
transfers = Transfers(conf, apMon, logger)
except FDTCopyException, ex:
logger.fatal(ex)
# everything failed - all transfers
if conf.get("report"):
# TODO
# if necessary - may take just the input or the copyjobfile
# and generate something with everything assigned 1
logger.error("Report file was required, can't generate it with "
"failure on this stage ...")
if apMon:
apMon.free()
logger.warn("fdtcp finished.")
logger.close()
sys.exit(1)
# main result status of the fdtcp app run, any failed transfer
# from transfers.transfers causes whole app status set to 1
appExitStatus = 0
try:
for transfer in transfers.transfers.values():
try:
transfer.performTransfer()
transfer.result = 0
except FDTCopyException, ex:
logger.error("Transfer failed, reason: %s" % ex)
transfer.result = 1
transfer.log = ex
appExitStatus = 1
def testMainLogFileOpeningDuringDaemonisation():
py.test.skip(
"Test skipped, messes up with input/output streams for "
"other test, better to run it stand-alone: "
"py.test fdtcplib/test/test_fdtd.py -s -k testMainLogFileOpeningDuringDaemonisation"
)
"""
As described in the problem #41:comment:9, the main log file remains
open thrice (under different descriptor) after initial daemonisation.
In order to include this particular test into whole test suite running
(issue with file descriptors being redirected, etc in fdtd.daemonize()
function), perhaps after running this test, the file descriptors
can be put back from backup sys.__stdout__, etc?
"""
c = """
[general]
logFile = /tmp/fdtd.log
pidFile = /tmp/fdtd.pid
"""
testName = inspect.stack()[0][3]
f = getTempFile(c)
inputOption = "--config=%s" % f.name
conf = ConfigFDTD(inputOption.split())
f.close()
logFile = "%s-%s" % (conf.get("logFile"), testName)
pidFile = conf.get("pidFile")
if os.path.exists(logFile):
print("test: %s: file '%s' exists, removing ..." % (testName, logFile))
os.remove(logFile)
if os.path.exists(pidFile):
print("test: %s: file '%s' exists, removing ..." % (testName, pidFile))
os.remove(pidFile)
logger = Logger(name=testName, logFile=logFile, level=logging.DEBUG)
logger.debug("Before daemonization ...")
numFiles, filesStr = getOpenFilesList()
logger.debug("Logging open files: %s items:\n%s" % (numFiles, filesStr))
try:
daemonize(conf, logger)
except SystemExit:
# previous leading process continues here
return
# here continues the newly created background daemon
f = open(pidFile, "r")
rc = f.read()
f.close()
rc = rc.strip()
pid = int(rc)
numFiles, filesStr = getOpenFilesList()
logger.debug("Logging open files: %s items:\n%s" % (numFiles, filesStr))
logger.debug("Before finishing ... ")
logger.close()
# the log file may be open more times due to streams descriptor
# duplication as done in fdtd.daemonization() but now, once is
# closed, there should not any other outstanding open file
assert numFiles == 0
def testAddressAlreadyInUseRoundRobinPortReservation():
"""
#38 - Address already in use FDT Java
https://trac.hep.caltech.edu/trac/fdtcp/ticket/38
Address already in use problem was seen during #5:comment:20
https://trac.hep.caltech.edu/trac/fdtcp/ticket/5#comment:20
2 times out of 338 transfer (attempts). Probably, when there is
traffic the port can't be bound immediately even if it was
released very short ago by the previous process.
This test could not reproduce the problem (when reusing immediately
the same port number for the next request), so FDTD.getFreePort()
was reimplemented to reserver ports on round-robin basis.
"""
hostName = os.uname()[1]
f = getTempFile(functionalFDTDConfiguration)
inputOption = "--config=%s" % f.name
conf = ConfigFDTD(inputOption.split())
conf.sanitize()
testName = inspect.stack()[0][3]
logger = Logger(name=testName,
logFile="/tmp/fdtdtest-%s.log" % testName,
level=logging.DEBUG)
apMon = None
fdtd = FDTD(conf, apMon, logger)
# launch two subsequent ReceivingServerAction, second will likely fail
# to bind the same, just very short ago, released port
serverId = "%s" % testName
testAction = TestAction(hostName, hostName)
testAction.id = serverId
# do TestAction
r = fdtd.service.service(testAction)
assert r.status == 0
options = dict(gridUserDest="someuserDest",
clientIP=os.uname()[1],
destFiles=[])
recvServerAction = ReceivingServerAction(testAction.id, options)
# do ReceivingServerAction - start FDT Java server
r = fdtd.service.service(recvServerAction)
print r.msg
assert r.status == 0
assert r.serverPort == 54321
cleanupAction = CleanupProcessesAction(serverId,
timeout=0,
waitTimeout=False)
# do CleanupProcessesAction - shut FDT Java server, port shall be
# released
r = fdtd.service.service(cleanupAction)
print r.msg
assert r.status == 0
# do another ReceivingServerAction - start FDT Java server
r = fdtd.service.service(recvServerAction)
print r.msg
assert r.status == 0
# will not get the same port, but the next one in the range
assert r.serverPort == 54322
# in fact, if separate log files are enabled, after this last
# ReceivingServerAction, there is a separate log file open.
# taking the the service down, it should also closed it's related
# to open files #41 problem
fdtd.shutdown()
fdtd.pyroDaemon.closedown()
logger.close()
def testFDTDServiceOpenFilesFullTransfer():
"""
#41:comment:8 - Too many open files (fdtd side)
SendingClient actually removed itself from the executors container
once it finishes so subsequent CleanupProcessesAction doesn't know
about this process, nor about its open separate log file, which
doesn't get closed.
Simulate a simple successful transfer, send all actions and
check number of open files - does all as it happens in fdtd.service()
"""
hostName = os.uname()[1]
testName = inspect.stack()[0][3]
initStateNumOpenFilesTestStart, filesStr = getOpenFilesList()
print("%s: test 0: open files: %s items:\n%s" %
(testName, initStateNumOpenFilesTestStart, filesStr))
# there should not be any open files now
assert initStateNumOpenFilesTestStart == 0
f = getTempFile(functionalFDTDConfiguration)
inputOption = "--config=%s --port=10001" % f.name
confServer = ConfigFDTD(inputOption.split())
confServer.sanitize()
loggerServer = Logger(name=testName,
logFile="/tmp/fdtdtest-%s-writer.log" % testName,
level=logging.DEBUG)
apMon = None
fdtdServer = FDTD(confServer, apMon, loggerServer)
inputOption = "--config=%s --port=10002" % f.name
confReader = ConfigFDTD(inputOption.split())
confReader.sanitize()
loggerReader = Logger(name=testName,
logFile="/tmp/fdtdtest-%s-reader.log" % testName,
level=logging.DEBUG)
apMon = None
fdtdReader = FDTD(confReader, apMon, loggerReader)
# -2 open log files, additional -1 is the temp config file
initStateNumOpenFiles, filesStr = getOpenFilesList()
print("%s: test 1: open files: %s items:\n%s" %
(testName, initStateNumOpenFiles, filesStr))
assert initStateNumOpenFilesTestStart == initStateNumOpenFiles - 2 - 1
testActionServer = TestAction(hostName, hostName)
testActionServer.id = testActionServer.id + "-writer"
r = fdtdServer.service.service(testActionServer)
assert r.status == 0
options = dict(gridUserDest="someuserDest",
clientIP=os.uname()[1],
destFiles=["/dev/null"])
recvServerAction = ReceivingServerAction(testActionServer.id, options)
r = fdtdServer.service.service(recvServerAction)
print r.msg
assert r.status == 0
serverFDTPort = r.serverPort
# there should be only 1 extra opened file now - ReceivingServerAction
# separate log
numOpenFilesNow, filesStr = getOpenFilesList()
print("%s: test 2: open files: %s items:\n%s" %
(testName, numOpenFilesNow, filesStr))
assert initStateNumOpenFiles == numOpenFilesNow - 1
testActionReader = TestAction(hostName, hostName)
testActionReader.id = testActionReader.id + "-reader"
r = fdtdReader.service.service(testActionReader)
assert r.status == 0
files = [TransferFile("/etc/passwd", "/dev/null")] # list of TransferFile
options = dict(port=serverFDTPort,
hostDest=os.uname()[1],
transferFiles=files,
gridUserSrc="soemuserSrc")
sndClientAction = SendingClientAction(testActionReader.id, options)
r = fdtdReader.service.service(sndClientAction)
assert r.status == 0
# there should be +2 extra - for separate both server and client
numOpenFilesNow, filesStr = getOpenFilesList()
print("%s: test 3: open files: %s items:\n%s" %
(testName, numOpenFilesNow, filesStr))
# 2 extra files - separate transfer log at both ends
assert initStateNumOpenFiles == numOpenFilesNow - 2
# now the transfer is over, both server (writer) and sender (reader)
# parties kept their separate log files open, CleanupProcessesAction
# will close them
print "going to clean up"
cl = CleanupProcessesAction(testActionReader.id, waitTimeout=False)
r = fdtdReader.service.service(cl)
assert r.status == 0
# one shall be closed now
numOpenFilesNow, filesStr = getOpenFilesList()
print("%s: test 4: open files: %s items:\n%s" %
(testName, numOpenFilesNow, filesStr))
assert initStateNumOpenFiles == numOpenFilesNow - 1
cl = CleanupProcessesAction(testActionServer.id, waitTimeout=False)
r = fdtdServer.service.service(cl)
assert r.status == 0
# both separate log files should be closed now
# problem #41:comment:8 was here - server behaved correctly, but
# reader kept its separate log file open
numOpenFilesNow, filesStr = getOpenFilesList()
print("%s: test 5: open files: %s items:\n%s" %
(testName, numOpenFilesNow, filesStr))
assert initStateNumOpenFiles == numOpenFilesNow
fdtdServer.shutdown()
fdtdServer.pyroDaemon.closedown()
loggerServer.close()
fdtdReader.shutdown()
fdtdReader.pyroDaemon.closedown()
loggerReader.close()
# after even log files were closed, etc
numOpenFilesNow, filesStr = getOpenFilesList()
print("%s: test 6: open files: %s items:\n%s" %
(testName, numOpenFilesNow, filesStr))
# -1: the temp configuration file is still open
assert initStateNumOpenFilesTestStart == numOpenFilesNow - 1
def testMainLogFileOpeningDuringDaemonisation():
py.test.skip("Test skipped, messes up with input/output streams for "
"other test, better to run it stand-alone: "
"py.test fdtcplib/test/test_fdtd.py -s -k testMainLogFileOpeningDuringDaemonisation")
"""
As described in the problem #41:comment:9, the main log file remains
open thrice (under different descriptor) after initial daemonisation.
In order to include this particular test into whole test suite running
(issue with file descriptors being redirected, etc in fdtd.daemonize()
function), perhaps after running this test, the file descriptors
can be put back from backup sys.__stdout__, etc?
"""
c = """
[general]
logFile = /tmp/fdtd.log
pidFile = /tmp/fdtd.pid
"""
testName = inspect.stack()[0][3]
f = getTempFile(c)
inputOption = "--config=%s" % f.name
conf = ConfigFDTD(inputOption.split())
f.close()
logFile = "%s-%s" % (conf.get("logFile"), testName)
pidFile = conf.get("pidFile")
if os.path.exists(logFile):
print("test: %s: file '%s' exists, removing ..." %
(testName, logFile))
os.remove(logFile)
if os.path.exists(pidFile):
print("test: %s: file '%s' exists, removing ..." %
(testName, pidFile))
os.remove(pidFile)
logger = Logger(name=testName, logFile=logFile, level=logging.DEBUG)
logger.debug("Before daemonization ...")
numFiles, filesStr = getOpenFilesList()
logger.debug("Logging open files: %s items:\n%s" % (numFiles, filesStr))
try:
daemonize(conf, logger)
except SystemExit:
# previous leading process continues here
return
# here continues the newly created background daemon
f = open(pidFile, 'r')
rc = f.read()
f.close()
rc = rc.strip()
pid = int(rc)
numFiles, filesStr = getOpenFilesList()
logger.debug("Logging open files: %s items:\n%s" % (numFiles, filesStr))
logger.debug("Before finishing ... ")
logger.close()
# the log file may be open more times due to streams descriptor
# duplication as done in fdtd.daemonization() but now, once is
# closed, there should not any other outstanding open file
assert numFiles == 0
try:
transfers = Transfers(conf, apMon, logger)
except FDTCopyException, ex:
logger.fatal(ex)
# everything failed - all transfers
if conf.get("report"):
# TODO
# if necessary - may take just the input or the copyjobfile
# and generate something with everything assigned 1
logger.error("Report file was required, can't generate it with "
"failure on this stage ...")
apMon.free()
logger.warn("fdtcp finished.")
logger.close()
sys.exit(1)
try:
for transfer in transfers.transfers.values():
try:
transfer.performTransfer()
transfer.result = 0
except FDTCopyException, ex:
logger.error("Transfer failed, reason: %s" % ex)
transfer.result = 1
transfer.log = ex
except ExecutorException, ex:
# TODO
# this type of exception is too low-level, should be wrapper on an upper level
