def log(message, filename=None, mode=None): ''' Log message that denova.python.log can't. ''' if filename is None: try: user = whoami() # we want to catch any type of exception and Exception won't always do that except: user = '******' filename = os.path.join(gettempdir(), f'_log.{user}.log') if mode is None: mode = '0666' with open(filename, 'a') as logfile: current_timestamp = timestamp() try: # logwriter dies sometimes and stops regular logging # but logit itself logs to this alternate log # this print should goto the systemd journal # journalctl --unit logit # error is e.g.: # 2020-04-17 19:21:33,555 too many values to unpack (expected 3) logfile.write(f'{current_timestamp} {format_exc()}\n') # DEBUG logfile.write(f'{current_timestamp} {message}\n') except UnicodeDecodeError: from denova.python.utils import is_string try: logfile.write(f'unable to write message because it is a type: {type(message)}') if not is_string(message): decoded_message = message.decode(errors='replace') logfile.write(f'{current_timestamp} {decoded_message}\n') except: # 'bare except' because it catches more than "except Exception" print(format_exc())
def __init__(self, line): ''' Line is a line from an apache compatible web log. ''' self.line = line m = LogLine.Line_Format.search(str(line)) if m: self.ip_address = LogLine.get_string(m.group('ip_address')) if not is_string(self.ip_address): self.ip_address = self.ip_address.decode() self.domain = LogLine.get_string(m.group('domain')) self.method = LogLine.get_string(m.group('method')) if self.method: self.method = self.method.upper() self.timestamp = LogLine.get_timestamp(m.group('timestamp')) self.referer = LogLine.get_string(m.group('referer')) if self.referer and self.referer == '-': self.referer = '' self.url = LogLine.get_string(m.group('url')) self.query_string = LogLine.get_string(m.group('query_string')) self.http_status_code = LogLine.get_int( m.group('http_status_code')) self.protocol = LogLine.get_string(m.group('protocol')) self.bytes = LogLine.get_int(m.group('bytes')) self.user = LogLine.get_string(m.group('user')) if self.user and self.user == '-': self.user = '' self.agent = LogLine.get_string(m.group('agent')) self.browser_name, self.browser_version, self.other = LogLine.get_browser_info( self.agent) else: message = f'{LogLine.NOT_APACHE_FORMAT}: {line}' if not is_string(line): line = line.decode() m = LogLine.Page_Not_Found_Format.search(line) if not m: log(message) # but still raise the error so we don't process the entry raise ValueError(message)
def write(filename, lines, append=False): '''Write the lines to a text file. Log any io errors and then raise another ioerrr. ''' try: if append: method = 'at' else: method = 'wt' outputFile = open(filename, method) for line in lines: if isinstance(line, list): # line must be another list # let's assume there aren't any more nested lists for inner_line in line: if is_string(l): text = inner_line.decode() else: text = inner_line outputFile.write(text) else: if is_string(line): text = line else: text = line.decode() outputFile.write(text) outputFile.close() except IOError: log(f'Unable to write {filename}') log(format_exc()) raise IOError return lines
def strip_input(data): '''Strip the leading and trailing spaces. >>> data = " This is a line without end spaces. " >>> strip_input(data) 'This is a line without end spaces.' ''' try: if data is not None: if is_string(data) or isinstance(data, CharField): data = data.strip() elif isinstance(data, EmailField): data = f'{data}' data = data.strip() elif isinstance(data, bytes): data = data.decode().strip() except Exception: log(format_exc()) return data
def chmod(mode, path, recursive=False): ''' Change permissions of path. mode is a string or octal integer for the chmod command. Examples:: 'o+rw,g+rw' 0660 ''' # arg order used to be chmod(path, mode, ...), so check types # delete this assert if no assertion errors 2015-01-01 # after we remove this assert, mode can be a string assert is_string(path) if isinstance(mode, int): # chmod wants an octal int, not decimal # so if it's an int we convert to an octal string oct_mode = oct(mode) if 'o' in oct_mode: i = oct_mode.find('o') oct_mode = oct_mode[i+1:] if len(oct_mode) > 4 and not oct_mode.startswith('0'): mode = '0' + oct_mode else: mode = oct_mode try: if recursive: run(*['chmod', '--recursive', mode, path]) else: run(*['chmod', mode, path]) except CalledProcessError as cpe: log.error(f'unable to chmod: path={path}, mode={mode}') log.error(cpe) raise
def edit_file_in_place(filename, replacements, regexp=False, lines=False): ''' Replace text in file. 'replacements' is a dict of {old: new, ...}. Every occurence of each old string is replaced with the matching new string. If regexp=True, the old string is a regular expression. If lines=True, each line is matched separately. Perserves permissions. >>> # note double backslashes because this is a string within a docstring >>> text = ( ... 'browser.search.defaultenginename=Startpage HTTPS\\n' + ... 'browser.search.selectedEngine=Startpage HTTPS\\n' + ... 'browser.startup.homepage=https://tails.boum.org/news/\\n' + ... 'spellchecker.dictionary=en_US') >>> f = tempfile.NamedTemporaryFile(mode='w', delete=False) >>> f.write(text) 178 >>> f.close() >>> HOMEPAGE = 'http://127.0.0.1/' >>> replacements = { ... 'browser.startup.homepage=.*': ... f'browser.startup.homepage={HOMEPAGE}', ... } >>> edit_file_in_place(f.name, replacements, regexp=True, lines=True) >>> with open(f.name) as textfile: ... newtext = textfile.read() >>> assert HOMEPAGE in newtext >>> os.remove(f.name) ''' # import delayed to avoid infinite recursion import denova.python.utils log.debug(f'edit file in place: {filename}, replacements {replacements}') # sometimes replace_strings() gets a type error for old, new in replacements.items(): assert is_string(old), f'replacement old "{old}" should be string but is type {type(old)}' assert is_string(new), f'replacement new "{new}" should be string but is type {type(new)}' # read text mode = os.stat(filename).st_mode with open(filename) as textfile: text = textfile.read() if lines: newtext = [] for line in text.split('\n'): # sometimes replace_strings() gets a type error assert is_string(line), f'line should be string but is {type(line)}' newline = denova.python.utils.replace_strings(line, replacements, regexp) newtext.append(newline) text = '\n'.join(newtext) else: text = denova.python.utils.replace_strings(text, replacements, regexp) # write text with open(filename, 'w') as textfile: textfile.write(text) os.chmod(filename, mode) assert mode == os.stat(filename).st_mode
def write(self, message): ''' Write message to log. write() writes to a log as you would to a file. This lets you redirect sys.stdout to the log and log print output generated by third party libraries, even if the library uses print statements without '>>' or print functions without 'stream='. ''' def notify_webmaster(message): ''' Send the message to the webmaster. We can't use reinhardt.dmail.send because it imports this module. ''' if alert_from_address is not None and alert_to_address is not None: addresses = f'From: {alert_from_address}\nTo: {alert_to_address}' msg = f'{addresses}\nSubject: {subject}\n\n{message}\n' server = smtplib.SMTP('localhost') server.sendmail(alert_from_address, alert_to_address, msg) # we don't use @synchronized here because of import errors global _RAISE_LOGGIN_ERRORS try: from denova.python.utils import is_string if is_string(message): self._write(message) elif isinstance(message, (bytes, bytearray)): # send bytes to self._write() self._write(message.decode(errors='replace')) else: self.write( f'unable to write message because it is a {type(message)}') if self.verbose: print(message) if _USE_MASTER_LOG and not self.is_master(): if self.user not in _MASTER_LOGS: _MASTER_LOGS[self.user] = Log('master.log', dirname=self.dirname) try: logline = f'{os.path.basename(self.pathname)} {message}' _MASTER_LOGS[self.user]._write(logline) except UnicodeDecodeError: _MASTER_LOGS[self.user]._write( f'{self.filename} - !! Unable to log message -- UnicodeDecodeError !!' ) except UnicodeDecodeError: try: _debug(message.decode(errors='replace')) except UnicodeDecodeError: self.write( f'unable to write message because it is a type: {type(message)}' ) subject = '!! Unable to log message !!' self._write(subject) if _RAISE_LOGGIN_ERRORS: _RAISE_LOGGIN_ERRORS = False notify_webmaster(message) except Exception: if _DEBUGGING: sys.exit('debug exit after exception') # DEBUG """ self._write(format_exc()) subject = '!! Unable to log message !!' self._write(subject) if _RAISE_LOGGIN_ERRORS: _RAISE_LOGGIN_ERRORS = False notify_webmaster(message) """ raise