def log(message, filename=None, mode=None):
''' Log message that denova.python.log can't. '''
if filename is None:
try:
user = whoami()
# we want to catch any type of exception and Exception won't always do that
except:
user = '******'
filename = os.path.join(gettempdir(), f'_log.{user}.log')
if mode is None:
mode = '0666'
with open(filename, 'a') as logfile:
current_timestamp = timestamp()
try:
# logwriter dies sometimes and stops regular logging
# but logit itself logs to this alternate log
# this print should goto the systemd journal
# journalctl --unit logit
# error is e.g.:
# 2020-04-17 19:21:33,555 too many values to unpack (expected 3)
logfile.write(f'{current_timestamp} {format_exc()}\n') # DEBUG
logfile.write(f'{current_timestamp} {message}\n')
except UnicodeDecodeError:
from denova.python.utils import is_string
try:
logfile.write(f'unable to write message because it is a type: {type(message)}')
if not is_string(message):
decoded_message = message.decode(errors='replace')
logfile.write(f'{current_timestamp} {decoded_message}\n')
except: # 'bare except' because it catches more than "except Exception"
print(format_exc())
def __init__(self, line):
''' Line is a line from an apache compatible web log. '''
self.line = line
m = LogLine.Line_Format.search(str(line))
if m:
self.ip_address = LogLine.get_string(m.group('ip_address'))
if not is_string(self.ip_address):
self.ip_address = self.ip_address.decode()
self.domain = LogLine.get_string(m.group('domain'))
self.method = LogLine.get_string(m.group('method'))
if self.method:
self.method = self.method.upper()
self.timestamp = LogLine.get_timestamp(m.group('timestamp'))
self.referer = LogLine.get_string(m.group('referer'))
if self.referer and self.referer == '-':
self.referer = ''
self.url = LogLine.get_string(m.group('url'))
self.query_string = LogLine.get_string(m.group('query_string'))
self.http_status_code = LogLine.get_int(
m.group('http_status_code'))
self.protocol = LogLine.get_string(m.group('protocol'))
self.bytes = LogLine.get_int(m.group('bytes'))
self.user = LogLine.get_string(m.group('user'))
if self.user and self.user == '-':
self.user = ''
self.agent = LogLine.get_string(m.group('agent'))
self.browser_name, self.browser_version, self.other = LogLine.get_browser_info(
self.agent)
else:
message = f'{LogLine.NOT_APACHE_FORMAT}: {line}'
if not is_string(line):
line = line.decode()
m = LogLine.Page_Not_Found_Format.search(line)
if not m:
log(message)
# but still raise the error so we don't process the entry
raise ValueError(message)
def write(filename, lines, append=False):
'''Write the lines to a text file.
Log any io errors and then raise another ioerrr.
'''
try:
if append:
method = 'at'
else:
method = 'wt'
outputFile = open(filename, method)
for line in lines:
if isinstance(line, list):
# line must be another list
# let's assume there aren't any more nested lists
for inner_line in line:
if is_string(l):
text = inner_line.decode()
else:
text = inner_line
outputFile.write(text)
else:
if is_string(line):
text = line
else:
text = line.decode()
outputFile.write(text)
outputFile.close()
except IOError:
log(f'Unable to write {filename}')
log(format_exc())
raise IOError
return lines
def strip_input(data):
'''Strip the leading and trailing spaces.
>>> data = " This is a line without end spaces. "
>>> strip_input(data)
'This is a line without end spaces.'
'''
try:
if data is not None:
if is_string(data) or isinstance(data, CharField):
data = data.strip()
elif isinstance(data, EmailField):
data = f'{data}'
data = data.strip()
elif isinstance(data, bytes):
data = data.decode().strip()
except Exception:
log(format_exc())
return data
def chmod(mode, path, recursive=False):
''' Change permissions of path.
mode is a string or octal integer for the chmod command.
Examples::
'o+rw,g+rw'
0660
'''
# arg order used to be chmod(path, mode, ...), so check types
# delete this assert if no assertion errors 2015-01-01
# after we remove this assert, mode can be a string
assert is_string(path)
if isinstance(mode, int):
# chmod wants an octal int, not decimal
# so if it's an int we convert to an octal string
oct_mode = oct(mode)
if 'o' in oct_mode:
i = oct_mode.find('o')
oct_mode = oct_mode[i+1:]
if len(oct_mode) > 4 and not oct_mode.startswith('0'):
mode = '0' + oct_mode
else:
mode = oct_mode
try:
if recursive:
run(*['chmod', '--recursive', mode, path])
else:
run(*['chmod', mode, path])
except CalledProcessError as cpe:
log.error(f'unable to chmod: path={path}, mode={mode}')
log.error(cpe)
raise
def edit_file_in_place(filename, replacements, regexp=False, lines=False):
''' Replace text in file.
'replacements' is a dict of {old: new, ...}.
Every occurence of each old string is replaced with the
matching new string.
If regexp=True, the old string is a regular expression.
If lines=True, each line is matched separately.
Perserves permissions.
>>> # note double backslashes because this is a string within a docstring
>>> text = (
... 'browser.search.defaultenginename=Startpage HTTPS\\n' +
... 'browser.search.selectedEngine=Startpage HTTPS\\n' +
... 'browser.startup.homepage=https://tails.boum.org/news/\\n' +
... 'spellchecker.dictionary=en_US')
>>> f = tempfile.NamedTemporaryFile(mode='w', delete=False)
>>> f.write(text)
178
>>> f.close()
>>> HOMEPAGE = 'http://127.0.0.1/'
>>> replacements = {
... 'browser.startup.homepage=.*':
... f'browser.startup.homepage={HOMEPAGE}',
... }
>>> edit_file_in_place(f.name, replacements, regexp=True, lines=True)
>>> with open(f.name) as textfile:
... newtext = textfile.read()
>>> assert HOMEPAGE in newtext
>>> os.remove(f.name)
'''
# import delayed to avoid infinite recursion
import denova.python.utils
log.debug(f'edit file in place: {filename}, replacements {replacements}')
# sometimes replace_strings() gets a type error
for old, new in replacements.items():
assert is_string(old), f'replacement old "{old}" should be string but is type {type(old)}'
assert is_string(new), f'replacement new "{new}" should be string but is type {type(new)}'
# read text
mode = os.stat(filename).st_mode
with open(filename) as textfile:
text = textfile.read()
if lines:
newtext = []
for line in text.split('\n'):
# sometimes replace_strings() gets a type error
assert is_string(line), f'line should be string but is {type(line)}'
newline = denova.python.utils.replace_strings(line, replacements, regexp)
newtext.append(newline)
text = '\n'.join(newtext)
else:
text = denova.python.utils.replace_strings(text, replacements, regexp)
# write text
with open(filename, 'w') as textfile:
textfile.write(text)
os.chmod(filename, mode)
assert mode == os.stat(filename).st_mode
def write(self, message):
''' Write message to log.
write() writes to a log as you would to a file.
This lets you redirect sys.stdout to the log and log print output
generated by third party libraries, even if the library uses
print statements without '>>' or print functions without 'stream='.
'''
def notify_webmaster(message):
''' Send the message to the webmaster.
We can't use reinhardt.dmail.send because it imports this module.
'''
if alert_from_address is not None and alert_to_address is not None:
addresses = f'From: {alert_from_address}\nTo: {alert_to_address}'
msg = f'{addresses}\nSubject: {subject}\n\n{message}\n'
server = smtplib.SMTP('localhost')
server.sendmail(alert_from_address, alert_to_address, msg)
# we don't use @synchronized here because of import errors
global _RAISE_LOGGIN_ERRORS
try:
from denova.python.utils import is_string
if is_string(message):
self._write(message)
elif isinstance(message, (bytes, bytearray)):
# send bytes to self._write()
self._write(message.decode(errors='replace'))
else:
self.write(
f'unable to write message because it is a {type(message)}')
if self.verbose:
print(message)
if _USE_MASTER_LOG and not self.is_master():
if self.user not in _MASTER_LOGS:
_MASTER_LOGS[self.user] = Log('master.log',
dirname=self.dirname)
try:
logline = f'{os.path.basename(self.pathname)} {message}'
_MASTER_LOGS[self.user]._write(logline)
except UnicodeDecodeError:
_MASTER_LOGS[self.user]._write(
f'{self.filename} - !! Unable to log message -- UnicodeDecodeError !!'
)
except UnicodeDecodeError:
try:
_debug(message.decode(errors='replace'))
except UnicodeDecodeError:
self.write(
f'unable to write message because it is a type: {type(message)}'
)
subject = '!! Unable to log message !!'
self._write(subject)
if _RAISE_LOGGIN_ERRORS:
_RAISE_LOGGIN_ERRORS = False
notify_webmaster(message)
except Exception:
if _DEBUGGING:
sys.exit('debug exit after exception') # DEBUG
"""
self._write(format_exc())
subject = '!! Unable to log message !!'
self._write(subject)
if _RAISE_LOGGIN_ERRORS:
_RAISE_LOGGIN_ERRORS = False
notify_webmaster(message)
"""
raise