def testAppend(self):
log = logfile.LogFile(self.name, self.dir)
log.write("0123456789")
log.close()
log = logfile.LogFile(self.name, self.dir)
self.assertEquals(log.size, 10)
self.assertEquals(log._file.tell(), log.size)
log.write("abc")
self.assertEquals(log.size, 13)
self.assertEquals(log._file.tell(), log.size)
f = log._file
f.seek(0, 0)
self.assertEquals(f.read(), "0123456789abc")
log.close()
def test_persistence(self):
"""
L{LogFile} objects can be pickled and unpickled, which preserves all
the various attributes of the log file.
"""
rotateLength = 12345
defaultMode = 0o642
maxRotatedFiles = 42
log = logfile.LogFile(self.name, self.dir, rotateLength, defaultMode,
maxRotatedFiles)
log.write("123")
log.close()
copy = pickle.loads(pickle.dumps(log))
self.addCleanup(copy.close)
# Check that the unpickled log is the same as the original one.
self.assertEqual(self.name, copy.name)
self.assertEqual(self.dir, copy.directory)
self.assertEqual(self.path, copy.path)
self.assertEqual(rotateLength, copy.rotateLength)
self.assertEqual(defaultMode, copy.defaultMode)
self.assertEqual(maxRotatedFiles, copy.maxRotatedFiles)
self.assertEqual(log.size, copy.size)
def start_twisted_logging(sub_path):
def custom_emit(self, event_dict):
"""Custom emit for FileLogObserver"""
text = log.textFromEventDict(event_dict)
if text is None:
return
self.timeFormat = '[%Y-%m-%d %H:%M:%S %f]'
time_str = self.formatTime(event_dict['time'])
fmt_dict = {'text': text.replace("\n", "\n\t")}
msg_str = log._safeFormat("%(text)s\n", fmt_dict)
util.untilConcludes(self.write, time_str + " " + msg_str)
util.untilConcludes(self.flush)
total_path = os.path.join(logging_file_root_path, sub_path)
# reload(sys)
# sys.setdefaultencoding('utf8')
# if not os.path.exists(total_path):
# os.makedirs(total_path)
file_path = os.path.join(total_path, TWISTED_LOG_NAME)
file_path = file_path.replace("\\", "/")
if not os.path.exists(file_path):
print("文件不存在")
f = logfile.LogFile(file_path, "./", rotateLength=1024 * 1024, maxRotatedFiles=3)
log.FileLogObserver.emit = custom_emit
log.startLogging(f, setStdout=False)
log.startLogging(sys.stdout)
def start_logger(spider_name):
# TODO: FIX read for files like spidername.log.1
filename = datetime.now().strftime("%Y-%m-%d." + spider_name + ".log")
logfile_ = logfile.LogFile(filename, GLOBAL_PATH + '/logs')
logger = ScrapyFileLogObserver(logfile_, logging.DEBUG)
tlog.addObserver(logger.emit)
def test_noPermission(self):
"""
Check it keeps working when permission on dir changes.
"""
log = logfile.LogFile(self.name, self.dir)
self.addCleanup(log.close)
log.write("abc")
# change permissions so rotation would fail
os.chmod(self.dir, 0o555)
# if this succeeds, chmod doesn't restrict us, so we can't
# do the test
try:
f = open(os.path.join(self.dir, "xxx"), "w")
except (OSError, IOError):
pass
else:
f.close()
return
log.rotate() # this should not fail
log.write("def")
log.flush()
f = log._file
self.assertEqual(f.tell(), 6)
f.seek(0, 0)
self.assertEqual(f.read(), b"abcdef")
def test_maxNumberOfLog(self):
"""
Test it respect the limit on the number of files when maxRotatedFiles
is not None.
"""
log = logfile.LogFile(self.name,
self.dir,
rotateLength=10,
maxRotatedFiles=3)
self.addCleanup(log.close)
log.write("1" * 11)
log.write("2" * 11)
self.assertTrue(os.path.exists("{0}.1".format(self.path)))
log.write("3" * 11)
self.assertTrue(os.path.exists("{0}.2".format(self.path)))
log.write("4" * 11)
self.assertTrue(os.path.exists("{0}.3".format(self.path)))
with open("{0}.3".format(self.path)) as fp:
self.assertEqual(fp.read(), "1" * 11)
log.write("5" * 11)
with open("{0}.3".format(self.path)) as fp:
self.assertEqual(fp.read(), "2" * 11)
self.assertFalse(os.path.exists("{0}.4".format(self.path)))
def test_rotation(self):
"""
Rotating log files autorotate after a period of time, and can also be
manually rotated.
"""
# this logfile should rotate every 10 bytes
with contextlib.closing(
logfile.LogFile(self.name, self.dir, rotateLength=10)) as log:
# test automatic rotation
log.write("123")
log.write("4567890")
log.write("1" * 11)
self.assertTrue(os.path.exists("{0}.1".format(self.path)))
self.assertFalse(os.path.exists("{0}.2".format(self.path)))
log.write('')
self.assertTrue(os.path.exists("{0}.1".format(self.path)))
self.assertTrue(os.path.exists("{0}.2".format(self.path)))
self.assertFalse(os.path.exists("{0}.3".format(self.path)))
log.write("3")
self.assertFalse(os.path.exists("{0}.3".format(self.path)))
# test manual rotation
log.rotate()
self.assertTrue(os.path.exists("{0}.3".format(self.path)))
self.assertFalse(os.path.exists("{0}.4".format(self.path)))
self.assertEqual(log.listLogs(), [1, 2, 3])
def testNoPermission(self):
"logfile: check it keeps working when permission on dir changes."
log = logfile.LogFile(self.name, self.dir)
log.write("abc")
# change permissions so rotation would fail
os.chmod(self.dir, 0444)
try:
# if this succeeds, chmod doesn't restrict us, so we can't
# do the test
try:
f = open(os.path.join(self.dir, "xxx"), "w")
except (OSError, IOError):
pass
else:
f.close()
return
log.rotate() # this should not fail
log.write("def")
log.flush()
f = log._file
self.assertEquals(f.tell(), 6)
f.seek(0, 0)
self.assertEquals(f.read(), "abcdef")
log.close()
finally:
# reset permission so tearDown won't fail, regardless of if an
# assertion was raised.
os.chmod(self.dir, 0777)
def get_json_log_observer():
f = logfile.LogFile("carbon_forwarder.log", log_dir, rotateLength=log_rotate_length, maxRotatedFiles=max_rotated_log_files)
observer = jsonFileLogObserver(f)
filterer = FilteringLogObserver(observer,
[LogLevelFilterPredicate(
LogLevel.levelWithName(log_level))])
return filterer
def openLogFile(logfile, max_file_size, rotated_log_files):
name = os.path.basename(logfile)
directory = os.path.dirname(logfile)
return txlogfile.LogFile(name,
directory,
rotateLength=max_file_size,
maxRotatedFiles=rotated_log_files)
def startService(self):
# logfile is a file-like object that supports rotation
self.logFile = logfile.LogFile(self.logName,
self.logDir,
rotateLength=self.maxLogSize)
self.logFile.rotate() # force rotation each time restarted
self.errlog = ErrorLog(self.logFile)
self.errlog.start()
def _configureTwistedLog():
theLog = logfile.LogFile("web.log", base.getConfig("logDir"))
log.startLogging(theLog, setStdout=False)
def rotator():
theLog.shouldRotate()
reactor.callLater(86400, rotator)
rotator()
def test_fromFullPath(self):
"""
Test the fromFullPath method.
"""
log1 = logfile.LogFile(self.name, self.dir, 10, defaultMode=0777)
log2 = logfile.LogFile.fromFullPath(self.path, 10, defaultMode=0777)
self.assertEquals(log1.name, log2.name)
self.assertEquals(os.path.abspath(log1.path), log2.path)
self.assertEquals(log1.rotateLength, log2.rotateLength)
self.assertEquals(log1.defaultMode, log2.defaultMode)
def testModePreservation(self):
"logfile: check rotated files have same permissions as original."
if not hasattr(os, "chmod"): return
f = open(self.path, "w").close()
os.chmod(self.path, 0707)
mode = os.stat(self.path)[0]
log = logfile.LogFile(self.name, self.dir)
log.write("abc")
log.rotate()
self.assertEquals(mode, os.stat(self.path)[0])
def test_defaultPermissions(self):
"""
Test the default permission of the log file: if the file exist, it
should keep the permission.
"""
with open(self.path, "wb"):
os.chmod(self.path, 0o707)
currentMode = stat.S_IMODE(os.stat(self.path)[stat.ST_MODE])
log1 = logfile.LogFile(self.name, self.dir)
self.assertEqual(stat.S_IMODE(os.stat(self.path)[stat.ST_MODE]), currentMode)
self.addCleanup(log1.close)
def test_append(self):
"""
Log files can be written to, closed. Their size is the number of
bytes written to them. Everything that was written to them can
be read, even if the writing happened on separate occasions,
and even if the log file was closed in between.
"""
with contextlib.closing(logfile.LogFile(self.name, self.dir)) as log:
log.write("0123456789")
log = logfile.LogFile(self.name, self.dir)
self.addCleanup(log.close)
self.assertEqual(log.size, 10)
self.assertEqual(log._file.tell(), log.size)
log.write("abc")
self.assertEqual(log.size, 13)
self.assertEqual(log._file.tell(), log.size)
f = log._file
f.seek(0, 0)
self.assertEqual(f.read(), b"0123456789abc")
def testWriting(self):
log = logfile.LogFile(self.name, self.dir)
log.write("123")
log.write("456")
log.flush()
log.write("7890")
log.close()
f = open(self.path, "r")
self.assertEquals(f.read(), "1234567890")
f.close()
def test_specifiedPermissions(self):
"""
Test specifying the permissions used on the log file.
"""
log1 = logfile.LogFile(self.name, self.dir, defaultMode=0066)
mode = stat.S_IMODE(os.stat(self.path)[stat.ST_MODE])
if runtime.platform.isWindows():
# The only thing we can get here is global read-only
self.assertEquals(mode, 0444)
else:
self.assertEquals(mode, 0066)
def testModePreservation(self):
"""
Check rotated files have same permissions as original.
"""
f = open(self.path, "w").close()
os.chmod(self.path, 0707)
mode = os.stat(self.path)[stat.ST_MODE]
log = logfile.LogFile(self.name, self.dir)
log.write("abc")
log.rotate()
self.assertEquals(mode, os.stat(self.path)[stat.ST_MODE])
def test_listLogsIgnoresZeroSuffixedFiles(self):
"""
L{LogFile.listLogs} ignores log files which rotated suffix is 0.
"""
log = logfile.LogFile(self.name, self.dir)
self.addCleanup(log.close)
for i in range(0, 3):
with open("{0}.{1}".format(log.path, i), "w") as fp:
fp.write("123")
self.assertEqual([1, 2], log.listLogs())
def test_log_file_data_type_text(self):
"""
Test a LogFile that uses text instead of binary for data writing
"""
with contextlib.closing(logfile.LogFile(self.name, self.dir, data_type_text=True)) as log:
log.write("123")
log.write("456")
log.flush()
log.write(b"7890")
with open(self.path) as f:
self.assertEqual(f.read(), "1234567890")
def test_log_file_data_type_binary_line_endings(self):
"""
Test a LogFile that uses text binary data type doesn't get windows line endings converted
"""
with contextlib.closing(logfile.LogFile(self.name, self.dir, data_type_text=False)) as log:
log.write(b"123\n")
log.write(b"456\n")
log.flush()
log.write(b"7890\n")
with open(self.path, 'rb') as f:
self.assertEqual(f.read(), b"123\n456\n7890\n")
def start_logger(debug):
"""
Logger will log for file if debug set to True else will print to cmdline.
The logfiles will rotate after exceeding since of 1M and 100 count.
"""
if debug:
tlog.startLogging(sys.stdout)
else:
filename = datetime.now().strftime("%Y-%m-%d.scrapy.log")
logfile_ = logfile.LogFile(filename, 'logs/', maxRotatedFiles=100)
logger = ScrapyFileLogObserver(logfile_, logging.INFO)
tlog.addObserver(logger.emit)
def test_defaultPermissions(self):
"""
Test the default permission of the log file: if the file exist, it
should keep the permission.
"""
f = file(self.path, "w")
os.chmod(self.path, 0707)
currentMode = stat.S_IMODE(os.stat(self.path)[stat.ST_MODE])
f.close()
log1 = logfile.LogFile(self.name, self.dir)
self.assertEquals(stat.S_IMODE(os.stat(self.path)[stat.ST_MODE]),
currentMode)
def start_logging(self):
"""
If configured enables logserver
"""
twlog.startLogging(sys.stdout)
if GLSettings.logfile:
name = os.path.basename(GLSettings.logfile)
directory = os.path.dirname(GLSettings.logfile)
logfile = twlogfile.LogFile(name, directory,
rotateLength=GLSettings.log_file_size,
maxRotatedFiles=GLSettings.maximum_rotated_log_files)
twlog.addObserver(GLLogObserver(logfile).emit)
def test_writing(self):
"""
Log files can be written to, flushed and closed. Closing a log file
also flushes it.
"""
with contextlib.closing(logfile.LogFile(self.name, self.dir)) as log:
log.write("123")
log.write("456")
log.flush()
log.write("7890")
with open(self.path) as f:
self.assertEqual(f.read(), "1234567890")
def test_listLogsWithBadlyNamedFiles(self):
"""
L{LogFile.listLogs} doesn't choke if it encounters a file with an
unexpected name.
"""
log = logfile.LogFile(self.name, self.dir)
self.addCleanup(log.close)
with open("{0}.1".format(log.path), "w") as fp:
fp.write("123")
with open("{0}.bad-file".format(log.path), "w") as fp:
fp.write("123")
self.assertEqual([1], log.listLogs())
def __init__(self, logger, ftpEndpoint, mtrimSerial):
''' Initializes our custom protocol
:param logger: The local file to store results
:param ftpEndpoint: The endpoint to send results to and read recipes from
'''
SerialClientProtocol.__init__(self)
self.logger = logger
self.ftpEndpoint = ftpEndpoint
self.mtrimSerial = mtrimSerial
self.logFile = logger.getFileName()
self.ENQCount = 0
self.lcOEE = LoopingCall(self.startOEEData)
self.lcAlarms = LoopingCall(self.startAlarmsData)
self.lcFTP = LoopingCall(self.startFTPTransfer)
self._reconnecting = False
self.config = utilities.optionReader()
fLogFile = logfile.LogFile('df1comms.log', '/home/pi/projects/newSLC/logs', maxRotatedFiles=2)
fLogObserver = log.FileLogObserver(fLogFile)
log.startLogging(logfile.LogFile('df1comms.log', '/home/pi/projects/newSLC/logs', maxRotatedFiles=2))
#log.startLogging(sys.stdout)
self.notified = False
self.transferred = False
self.loaded = False
def __init__(self,
filelog=None,
dirlog="../log/",
rotateLengthMB=1,
maxRotatedFiles=100):
f = sys.stdout
if filelog:
if not os.path.isdir(dirlog):
os.makedirs(dirlog)
f = logfile.LogFile(filelog,
dirlog,
rotateLength=rotateLengthMB * 1000000,
maxRotatedFiles=maxRotatedFiles)
self.logger = LevelFileLogObserver(f, logging.DEBUG)
def test_writing(self):
"""
Log files can be written to, flushed and closed. Closing a log file
also flushes it.
"""
log = logfile.LogFile(self.name, self.dir)
log.write("123")
log.write("456")
log.flush()
log.write("7890")
log.close()
f = open(self.path)
self.assertEqual(f.read(), "1234567890")
f.close()
