def save(self, filename=None, gpg_recipient=None, encryption_key=None): filename = filename or self.filename if filename: gpg_recipient = gpg_recipient or self.gpg_recipient() encryption_key = encryption_key or self.encryption_key() if encryption_key: from mailpile.crypto.streamer import EncryptingStreamer es = EncryptingStreamer(encryption_key, dir=os.path.dirname(filename)) es.write(self.as_vCard()) es.save(filename) else: fd = gpg_open(filename, gpg_recipient, 'wb') fd.write(self.as_vCard()) fd.close() return self else: raise ValueError('Save to what file?')
class EventLog(object): """ This is the Mailpile Event Log. The log is written encrypted to disk on an ongoing basis (rotated every N lines), but entries are kept in RAM as well. The event log allows for recording of incomplete events, to help different parts of the app "remember" tasks which have yet to complete or may need to be retried. """ def __init__(self, logdir, encryption_key_func, rollover=10240): self.logdir = logdir self.encryption_key_func = encryption_key_func self.rollover = rollover self.events = {} # Internals... self._lock = threading.Lock() self._log_fd = None def _save_filename(self): return os.path.join(self.logdir, self._log_start_id) def _open_log(self): if self._log_fd: self._log_fd.close() if not os.path.exists(self.logdir): os.mkdir(self.logdir) self._log_start_id = NewEventId() enc_key = self.encryption_key_func() if enc_key: self._log_fd = EncryptingStreamer(enc_key, dir=self.logdir) self._log_fd.save(self._save_filename(), finish=False) else: self._log_fd = open(self._save_filename(), 'w', 0) # Write any incomplete events to the new file for e in self.incomplete(): self._log_fd.write('%s\n' % e) # We're starting over, incomplete events don't count self._logged = 0 def _maybe_rotate_log(self): if self._logged > self.rollover: self._log_fd.close() kept_events = {} for e in self.incomplete(): kept_events[e.event_id] = e self.events = kept_events self.open_log() def _match(self, event, filters): return True def incomplete(self, **filters): """Return all the incomplete events, in order.""" for ek in sorted(self.events.keys()): e = self.events.get(ek, None) if (e is not None and e.state != Event.COMPLETE and self._match(e, filters)): yield e def since(self, ts, **filters): """Return all events since a given time, in order.""" for ek in sorted(self.events.keys()): e = self.events.get(ek, None) if (e is not None and e.ts >= ts and self._match(e, filters)): yield e def log_event(self, event): """Log an Event object.""" self._lock.acquire() try: if not self._log_fd: self._open_log() self._log_fd.write('%s\n' % event) self.events[event.event_id] = event self._logged += 1 self._maybe_rotate_log() finally: self._lock.release() return event def log(self, *args, **kwargs): """Log a new event.""" return self.log_event(Event(*args, **kwargs)) def close(self): self._lock.acquire() try: self._log_fd.close() self._log_fd = None finally: self._lock.release()
class EventLog(object): """ This is the Mailpile Event Log. The log is written encrypted to disk on an ongoing basis (rotated every N lines), but entries are kept in RAM as well. The event log allows for recording of incomplete events, to help different parts of the app "remember" tasks which have yet to complete or may need to be retried. """ KEEP_LOGS = 2 def __init__(self, logdir, encryption_key_func, rollover=10240): self.logdir = logdir self.encryption_key_func = encryption_key_func self.rollover = rollover self._events = {} # Internals... self._waiter = threading.Condition() self._lock = threading.Lock() self._log_fd = None def _notify_waiters(self): self._waiter.acquire() self._waiter.notifyAll() self._waiter.release() def wait(self, timeout=None): self._waiter.acquire() self._waiter.wait(timeout) self._waiter.release() def _save_filename(self): return os.path.join(self.logdir, self._log_start_id) def _open_log(self): if self._log_fd: self._log_fd.close() if not os.path.exists(self.logdir): os.mkdir(self.logdir) self._log_start_id = NewEventId() enc_key = self.encryption_key_func() if enc_key: self._log_fd = EncryptingStreamer(enc_key, dir=self.logdir) self._log_fd.save(self._save_filename(), finish=False) else: self._log_fd = open(self._save_filename(), 'w', 0) # Write any incomplete events to the new file for e in self.incomplete(): self._log_fd.write('%s\n' % e) # We're starting over, incomplete events don't count self._logged = 0 def _maybe_rotate_log(self): if self._logged > self.rollover: self._log_fd.close() kept_events = {} for e in self.incomplete(): kept_events[e.event_id] = e self._events = kept_events self._open_log() self.purge_old_logfiles() def _list_logfiles(self): return sorted([l for l in os.listdir(self.logdir) if not l.startswith('.')]) def _save_events(self, events): if not self._log_fd: self._open_log() events.sort(key=lambda ev: ev.ts) for event in events: self._log_fd.write('%s\n' % event) self._events[event.event_id] = event def _load_logfile(self, lfn): enc_key = self.encryption_key_func() with open(os.path.join(self.logdir, lfn)) as fd: if enc_key: lines = fd.read() else: with DecryptingStreamer(enc_key, fd) as streamer: lines = streamer.read() if lines: for line in lines.splitlines(): event = Event.Parse(line) self._events[event.event_id] = event def _match(self, event, filters): for kw, rule in filters.iteritems(): if kw.endswith('!'): truth, okw, kw = False, kw, kw[:-1] else: truth, okw = True, kw if kw == 'source': if truth != (event.source == _ClassName(rule)): return False elif kw == 'flag': if truth != (rule in event.flags): return False elif kw == 'flags': if truth != (event.flags == rule): return False elif kw == 'since': when = float(rule) if when < 0: when += time.time() if truth != (event.ts > when): return False elif kw.startswith('data_'): if truth != (str(event.data.get(kw[5:])) == str(rule)): return False elif kw.startswith('private_data_'): if truth != (str(event.data.get(kw[13:])) == str(rule)): return False else: # Unknown keywords match nothing... print 'Unknown keyword: `%s=%s`' % (okw, rule) return False return True def incomplete(self, **filters): """Return all the incomplete events, in order.""" for ek in sorted(self._events.keys()): e = self._events.get(ek, None) if (e is not None and Event.COMPLETE not in e.flags and self._match(e, filters)): yield e def since(self, ts, **filters): """Return all events since a given time, in order.""" if ts < 0: ts += time.time() for ek in sorted(self._events.keys()): e = self._events.get(ek, None) if (e is not None and e.ts >= ts and self._match(e, filters)): yield e def events(self, **filters): return self.since(0, **filters) def get(self, event_id, default=None): return self._events.get(event_id, default) def log_event(self, event): """Log an Event object.""" self._lock.acquire() try: self._save_events([event]) self._logged += 1 self._maybe_rotate_log() self._notify_waiters() finally: self._lock.release() return event def log(self, *args, **kwargs): """Log a new event.""" return self.log_event(Event(*args, **kwargs)) def close(self): self._lock.acquire() try: self._log_fd.close() self._log_fd = None finally: self._lock.release() def _prune_completed(self): for event_id in self._events.keys(): if Event.COMPLETE in self._events[event_id].flags: del self._events[event_id] def load(self): self._lock.acquire() try: self._open_log() for lf in self._list_logfiles()[-4:]: try: self._load_logfile(lf) except (OSError, IOError): import traceback traceback.print_exc() self._prune_completed() self._save_events(self._events.values()) return self finally: self._lock.release() def purge_old_logfiles(self, keep=None): keep = keep or self.KEEP_LOGS for lf in self._list_logfiles()[:-keep]: try: os.remove(os.path.join(self.logdir, lf)) except OSError: pass
class EventLog(object): """ This is the Mailpile Event Log. The log is written encrypted to disk on an ongoing basis (rotated every N lines), but entries are kept in RAM as well. The event log allows for recording of incomplete events, to help different parts of the app "remember" tasks which have yet to complete or may need to be retried. """ KEEP_LOGS = 2 def __init__(self, logdir, decryption_key_func, encryption_key_func, rollover=1024): self.logdir = logdir self.decryption_key_func = decryption_key_func or (lambda: None) self.encryption_key_func = encryption_key_func or (lambda: None) self.rollover = rollover self._events = {} # Internals... self._waiter = threading.Condition(EventRLock()) self._lock = EventLock() self._log_fd = None def _notify_waiters(self): with self._waiter: self._waiter.notifyAll() def wait(self, timeout=None): with self._waiter: self._waiter.wait(timeout) def _save_filename(self): return os.path.join(self.logdir, self._log_start_id) def _open_log(self): if self._log_fd: self._log_fd.close() if not os.path.exists(self.logdir): os.mkdir(self.logdir) self._log_start_id = NewEventId() enc_key = self.encryption_key_func() if enc_key: self._log_fd = EncryptingStreamer(enc_key, dir=self.logdir, name='EventLog/ES', long_running=True) self._log_fd.save(self._save_filename(), finish=False) else: self._log_fd = open(self._save_filename(), 'wb', 0) # Write any incomplete events to the new file for e in self.incomplete(): self._log_fd.write('%s\n' % e) # We're starting over, incomplete events don't count self._logged = 0 def _maybe_rotate_log(self): if self._logged > self.rollover: self._log_fd.close() kept_events = {} for e in self.incomplete(): kept_events[e.event_id] = e self._events = kept_events self._open_log() self.purge_old_logfiles() def _list_logfiles(self): return sorted( [l for l in os.listdir(self.logdir) if not l.startswith('.')]) def _save_events(self, events, recursed=False): if not self._log_fd: self._open_log() events.sort(key=lambda ev: ev.ts) try: for event in events: self._log_fd.write('%s\n' % event) self._events[event.event_id] = event except IOError: if recursed: raise else: self._unlocked_close() return self._save_events(events, recursed=True) def _load_logfile(self, lfn): enc_key = self.decryption_key_func() with open(os.path.join(self.logdir, lfn)) as fd: if enc_key: with DecryptingStreamer(fd, mep_key=enc_key, name='EventLog/DS') as streamer: lines = streamer.read() streamer.verify(_raise=IOError) else: lines = fd.read() if lines: for line in lines.splitlines(): event = Event.Parse(line) self._events[event.event_id] = event def _match(self, event, filters): for kw, rule in filters.iteritems(): if kw.endswith('!'): truth, okw, kw = False, kw, kw[:-1] else: truth, okw = True, kw if kw == 'source': if truth != (event.source == _ClassName(rule)): return False elif kw == 'flag': if truth != (rule in event.flags): return False elif kw == 'flags': if truth != (event.flags == rule): return False elif kw == 'event_id': if truth != (event.event_id == rule): return False elif kw == 'since': when = float(rule) if when < 0: when += time.time() if truth != (event.ts > when): return False elif kw.startswith('data_'): if truth != (str(event.data.get(kw[5:])) == str(rule)): return False elif kw.startswith('private_data_'): if truth != (str(event.data.get(kw[13:])) == str(rule)): return False else: # Unknown keywords match nothing... print 'Unknown keyword: `%s=%s`' % (okw, rule) return False return True def incomplete(self, **filters): """Return all the incomplete events, in order.""" if 'event_id' in filters: ids = [filters['event_id']] else: ids = sorted(self._events.keys()) for ek in ids: e = self._events.get(ek, None) if (e is not None and Event.COMPLETE not in e.flags and self._match(e, filters)): yield e def since(self, ts, **filters): """Return all events since a given time, in order.""" if ts < 0: ts += time.time() if 'event_id' in filters and filters['event_id'][:1] != '!': ids = [filters['event_id']] else: ids = sorted(self._events.keys()) for ek in ids: e = self._events.get(ek, None) if (e is not None and e.ts >= ts and self._match(e, filters)): yield e def events(self, **filters): return self.since(0, **filters) def get(self, event_id, default=None): return self._events.get(event_id, default) def log_event(self, event): """Log an Event object.""" with self._lock: self._save_events([event]) self._logged += 1 self._maybe_rotate_log() self._notify_waiters() return event def log(self, *args, **kwargs): """Log a new event.""" return self.log_event(Event(*args, **kwargs)) def close(self): with self._lock: return self._unlocked_close() def _unlocked_close(self): try: self._log_fd.close() self._log_fd = None except (OSError, IOError): pass def _prune_completed(self): for event_id in self._events.keys(): if Event.COMPLETE in self._events[event_id].flags: del self._events[event_id] def load(self): with self._lock: self._open_log() for lf in self._list_logfiles()[-4:]: try: self._load_logfile(lf) except (OSError, IOError): # Nothing we can do, no point complaining... pass self._prune_completed() self._save_events(self._events.values()) return self def purge_old_logfiles(self, keep=None): keep = keep or self.KEEP_LOGS for lf in self._list_logfiles()[:-keep]: try: os.remove(os.path.join(self.logdir, lf)) except OSError: pass
class EventLog(object): """ This is the Mailpile Event Log. The log is written encrypted to disk on an ongoing basis (rotated every N lines), but entries are kept in RAM as well. The event log allows for recording of incomplete events, to help different parts of the app "remember" tasks which have yet to complete or may need to be retried. """ KEEP_LOGS = 2 def __init__(self, logdir, decryption_key_func, encryption_key_func, rollover=1024): self.logdir = logdir self.decryption_key_func = decryption_key_func or (lambda: None) self.encryption_key_func = encryption_key_func or (lambda: None) self.rollover = rollover self._events = {} # Internals... self._watching_uis = [] self._waiter = threading.Condition(EventRLock()) self._lock = EventLock() self._log_fd = None def _notify_waiters(self): with self._waiter: self._waiter.notifyAll() def wait(self, timeout=None): with self._waiter: self._waiter.wait(timeout) def _save_filename(self): return os.path.join(self.logdir, self._log_start_id) def _open_log(self): if self._log_fd: self._log_fd.close() if not os.path.exists(self.logdir): os.mkdir(self.logdir) self._log_start_id = NewEventId() enc_key = self.encryption_key_func() if enc_key: self._log_fd = EncryptingStreamer(enc_key, dir=self.logdir, name='EventLog/ES', long_running=True) self._log_fd.save(self._save_filename(), finish=False) else: self._log_fd = open(self._save_filename(), 'wb', 0) # Write any incomplete events to the new file for e in self.incomplete(): self._log_fd.write('%s\n' % e) # We're starting over, incomplete events don't count self._logged = 0 def _maybe_rotate_log(self): if self._logged > self.rollover: self._log_fd.close() kept_events = {} for e in self.incomplete(): kept_events[e.event_id] = e self._events = kept_events self._open_log() self.purge_old_logfiles() def _list_logfiles(self): return sorted([l for l in os.listdir(self.logdir) if not l.startswith('.')]) def _save_events(self, events, recursed=False): if not self._log_fd: self._open_log() events.sort(key=lambda ev: ev.ts) try: for event in events: self._log_fd.write('%s\n' % event) self._events[event.event_id] = event except IOError: if recursed: raise else: self._unlocked_close() return self._save_events(events, recursed=True) def _load_logfile(self, lfn): enc_key = self.decryption_key_func() with open(os.path.join(self.logdir, lfn)) as fd: if enc_key: with DecryptingStreamer(fd, mep_key=enc_key, name='EventLog/DS') as streamer: lines = streamer.read() streamer.verify(_raise=IOError) else: lines = fd.read() if lines: for line in lines.splitlines(): event = Event.Parse(line) self._events[event.event_id] = event def _match(self, event, filters): def compare(val, rule): if isinstance(rule, (str, unicode)): return unicode(val) == unicode(rule) else: return rule.match(unicode(val)) is not None for kw, rule in filters.iteritems(): if kw.endswith('!'): truth, okw, kw = False, kw, kw[:-1] else: truth, okw = True, kw if kw == 'source': if truth != compare(event.source, _ClassName(rule, ignore_regexps=True)): return False elif kw == 'flag': if truth != (rule in event.flags): return False elif kw == 'flags': if truth != compare(event.flags, rule): return False elif kw == 'event_id': if truth != compare(event.event_id, rule): return False elif kw == 'since': when = float(rule) if when < 0: when += time.time() if truth != (event.ts > when): return False elif kw.startswith('data_'): if truth != compare(event.data.get(kw[5:]), rule): return False elif kw.startswith('private_data_'): if truth != compare(event.data.get(kw[13:]), rule): return False else: # Unknown keywords match nothing... print 'Unknown keyword: `%s=%s`' % (okw, rule) return False return True def incomplete(self, **filters): """Return all the incomplete events, in order.""" if 'event_id' in filters: ids = [filters['event_id']] else: ids = sorted(self._events.keys()) for ek in ids: e = self._events.get(ek, None) if (e is not None and Event.COMPLETE not in e.flags and self._match(e, filters)): yield e def since(self, ts, **filters): """Return all events since a given time, in order.""" if ts < 0: ts += time.time() if 'event_id' in filters and filters['event_id'][:1] != '!': ids = [filters['event_id']] else: ids = sorted(self._events.keys()) for ek in ids: e = self._events.get(ek, None) if (e is not None and e.ts >= ts and self._match(e, filters)): yield e def events(self, **filters): return self.since(0, **filters) def get(self, event_id, default=None): return self._events.get(event_id, default) def log_event(self, event): """Log an Event object.""" with self._lock: self._save_events([event]) self._logged += 1 self._maybe_rotate_log() self._notify_waiters() for ui in self._watching_uis: ui.notify(event.as_text(compact=True)) return event def log(self, *args, **kwargs): """Log a new event.""" return self.log_event(Event(*args, **kwargs)) def close(self): with self._lock: return self._unlocked_close() def _unlocked_close(self): try: self._log_fd.close() self._log_fd = None except (OSError, IOError): pass def _prune_completed(self): for event_id in self._events.keys(): if Event.COMPLETE in self._events[event_id].flags: del self._events[event_id] def ui_watch(self, ui): while ui.log_parent is not None: ui = ui.log_parent if ui not in self._watching_uis: self._watching_uis.append(ui) return True else: return False def ui_unwatch(self, ui): while ui.log_parent is not None: ui = ui.log_parent try: self._watching_uis.remove(ui) except ValueError: pass def load(self): with self._lock: self._open_log() for lf in self._list_logfiles()[-4:]: try: self._load_logfile(lf) except (OSError, IOError): # Nothing we can do, no point complaining... pass self._prune_completed() self._save_events(self._events.values()) return self def purge_old_logfiles(self, keep=None): keep = keep or self.KEEP_LOGS for lf in self._list_logfiles()[:-keep]: try: safe_remove(os.path.join(self.logdir, lf)) except OSError: pass
class EventLog(object): """ This is the Mailpile Event Log. The log is written encrypted to disk on an ongoing basis (rotated every N lines), but entries are kept in RAM as well. The event log allows for recording of incomplete events, to help different parts of the app "remember" tasks which have yet to complete or may need to be retried. """ KEEP_LOGS = 2 def __init__(self, logdir, encryption_key_func, rollover=10240): self.logdir = logdir self.encryption_key_func = encryption_key_func self.rollover = rollover self.events = {} # Internals... self._lock = threading.Lock() self._log_fd = None def _save_filename(self): return os.path.join(self.logdir, self._log_start_id) def _open_log(self): if self._log_fd: self._log_fd.close() if not os.path.exists(self.logdir): os.mkdir(self.logdir) self._log_start_id = NewEventId() enc_key = self.encryption_key_func() if enc_key: self._log_fd = EncryptingStreamer(enc_key, dir=self.logdir) self._log_fd.save(self._save_filename(), finish=False) else: self._log_fd = open(self._save_filename(), 'w', 0) # Write any incomplete events to the new file for e in self.incomplete(): self._log_fd.write('%s\n' % e) # We're starting over, incomplete events don't count self._logged = 0 def _maybe_rotate_log(self): if self._logged > self.rollover: self._log_fd.close() kept_events = {} for e in self.incomplete(): kept_events[e.event_id] = e self.events = kept_events self._open_log() self.purge_old_logfiles() def _match(self, event, filters): return True def _list_logfiles(self): return sorted([l for l in os.listdir(self.logdir) if not l.startswith('.')]) def _save_events(self, events): if not self._log_fd: self._open_log() events.sort(key=lambda ev: ev.ts) for event in events: self._log_fd.write('%s\n' % event) self.events[event.event_id] = event def _load_logfile(self, lfn): enc_key = self.encryption_key_func() with open(os.path.join(self.logdir, lfn)) as fd: if enc_key: lines = fd.read() else: with DecryptingStreamer(enc_key, fd) as streamer: lines = fd.read() if lines: for line in lines.splitlines(): event = Event.Parse(line) if Event.COMPLETE in event.flags: if event.event_id in self.events: del self.events[event.event_id] else: self.events[event.event_id] = event self._save_events(self.events.values()) def incomplete(self, **filters): """Return all the incomplete events, in order.""" for ek in sorted(self.events.keys()): e = self.events.get(ek, None) if (e is not None and Event.COMPLETE not in e.flags and self._match(e, filters)): yield e def since(self, ts, **filters): """Return all events since a given time, in order.""" for ek in sorted(self.events.keys()): e = self.events.get(ek, None) if (e is not None and e.ts >= ts and self._match(e, filters)): yield e def log_event(self, event): """Log an Event object.""" self._lock.acquire() try: self._save_events([event]) self._logged += 1 self._maybe_rotate_log() finally: self._lock.release() return event def log(self, *args, **kwargs): """Log a new event.""" return self.log_event(Event(*args, **kwargs)) def close(self): self._lock.acquire() try: self._log_fd.close() self._log_fd = None finally: self._lock.release() def load(self): self._lock.acquire() try: self._open_log() for lf in self._list_logfiles()[-1:]: try: self._load_logfile(lf) except (OSError, IOError): pass return self finally: self._lock.release() def purge_old_logfiles(self, keep=None): keep = keep or self.KEEP_LOGS for lf in self._list_logfiles()[:-keep]: try: os.remove(os.path.join(self.logdir, lf)) except OSError: pass