def prepare_user_updates(csv_filepath: str) -> List[UserUpdate]: with start_action(action_type="read_user_csv", csv_filepath=csv_filepath) as action: with open(csv_filepath) as f: reader = csv.DictReader(f, quotechar=settings.csv_quotechar, strict=True, delimiter=settings.csv_delimiter) user_rows = list(reader) action.add_success_fields(len_user_rows=len(user_rows)) users = [] with start_action(action_type="prepare_updates"): for ll, row in enumerate(user_rows): try: sync_id = row[settings.field_sync_id] user = UserUpdate(email=row[settings.field_email], sync_id=sync_id) if settings.field_verified: user.verified = row[settings.field_verified] == "1" if settings.field_eligible: user.eligible = row[settings.field_eligible] == "1" if settings.field_department: user.department = row[settings.field_department] users.append(user) except Exception as e: log_message(message_type="prepare_user_failure", line=ll, exception=e) return users
def processEnded(self, reason): with self._action: log_message(message_type=u"process-ended") if self.magic_seen is not None: d, self.magic_seen = self.magic_seen, None d.errback(Exception("Service failed.")) self.exited.callback(None)
def make_wsgi_app(settings_filepath=None, testing=False): with start_action(action_type='morepath_scan'): morepath.autoscan() morepath.scan(ekklesia_portal) if testing: log_message( message_type="testing", msg="running in testing mode, not loading any config from file") else: with start_action(action_type='settings'): settings = get_app_settings(settings_filepath) App._loaded_settings = settings App.init_settings(settings) with start_action(action_type='make_app'): App.commit() app = App() database.configure_sqlalchemy(app.settings.database, testing) app.babel_init() app.babel.localeselector(get_locale) log_message(message_type="environment", env=dict(os.environ), encoding=locale.getpreferredencoding(), default_locale=locale.getdefaultlocale()) return app
def create_department_group(keycloak_admin: keycloak.KeycloakAdmin, department: Department, parent_id=None, path="/"): with start_action(action_type="create_department_group", department=department.internal_name, path=path): # Create or update group keycloak_admin.create_group({"name": department.internal_name, "attributes": {"display_name": [department.name]}}, parent=parent_id, skip_exists=True) # Extend path path += department.internal_name # Get group id try: created_group = keycloak_admin.get_group_by_path(path, search_in_subgroups=True) except keycloak.KeycloakGetError as e: log_message(message_type="get_group_failed", exception=e) return # Add slash to group path path += "/" # Create groups for sub departments subgroup_names = [] for sub_department in department.children: create_department_group(keycloak_admin, sub_department, parent_id=created_group["id"], path=path) subgroup_names.append(sub_department.internal_name) # Delete old subgroups for old_group in created_group["subGroups"]: if old_group["name"] not in subgroup_names: keycloak_admin.delete_group(old_group["id"])
def log(self, sql, timestamp, duration, curs): notices = [notice.strip() for notice in self.notices] duration_ms = (duration * 1000) duration_formatted = f"{duration_ms:.2f}ms" log_entry = dict(duration=duration_formatted, duration_ms=duration, sql=sql) if notices: log_entry['notices'] = notices log_message('sql-statement', **log_entry) self._history.append(sql, timestamp, duration, notices)
def eliot_garbage_received(self, data): """ Called when non-JSON lines are received on FD 3. Since FD 3 is suppposed to only have eliot-logs, log them as malformed. """ with self._action.context(): log_message(message_type=u"malformed-eliot-log", data=data.decode("utf8"))
def emit(self, record): eliot.log_message( message_type=record.name, log_level=record.levelname, # logger=record.name, message=record.getMessage(), ) if record.exc_info: write_traceback(exc_info=record.exc_info)
def err_received(self, data): """ Called when non-JSON lines are received on stderr. On Windows we use stderr for eliot logs from magic-folder. But neither magic-folder nor tahoe guarantee that there is no other output there, so we treat it as expected. """ with self._action.context(): log_message(message_type=u"err-received", data=data.decode("utf8")) sys.stdout.write(data.decode("utf8"))
def periodic_scan(node, folder_name, path): """ Wait for the given magic folder to run a periodic scan. This should cause the given path to be snapshotted. :param MagicFolderEnabledNode node: The node on which to do the scan. """ from twisted.internet import reactor log_message(message_type="integration:wait_for_scan", node=node.name, folder=folder_name) # XXX need a better way than "wait 3 seconds" to know if the scan is completed... return twisted_sleep(reactor, 3.0)
def out_received(self, data): """ Called with output from stdout. """ with self._action.context(): log_message(message_type=u"out-received", data=data.decode("utf8")) sys.stdout.write(data.decode("utf8")) self._output.write(data.decode("utf8")) if self.magic_seen is not None and self._magic_text in self._output.getvalue( ): print("Saw '{}' in the logs".format(self._magic_text)) d, self.magic_seen = self.magic_seen, None d.callback(self)
def start_magic_folder(self): if self.magic_folder is not None: return # We log a notice that we are starting the service in the context of the test # but the logs of the service are in the context of the fixture. log_message(message_type=u"integation:magic-folder:start", node=self.name) with self.action.context(): self.magic_folder = yield _run_magic_folder( self.reactor, self.request, self.base_dir, self.name, )
def change_language(self, request): lang = request.POST.get('lang') if lang not in request.app.settings.app.languages: raise HTTPBadRequest("unsupported language") back_url = request.POST.get('back_url') parsed_app_url = urlparse(request.application_url) parsed_back_url = urlparse(back_url) if parsed_app_url.netloc != parsed_back_url.netloc: log_message(message_type="invalid_redirect", url=back_url) raise HTTPBadRequest("redirect not allowed") request.browser_session['lang'] = lang return redirect(back_url)
def stop_magic_folder(self): log_message(message_type=u"integation:magic-folder:stop", node=self.name) if self.magic_folder is None: return try: log_message( message_type=u"integation:magic-folder:stop", node=self.name, signal="TERM", ) self.magic_folder.signalProcess('TERM') yield self.magic_folder.proto.exited self.magic_folder = None except ProcessExitedAlready: pass
def main(reactor): print("Logging to example-eliot.log...") logWriter = ThreadedWriter( FileDestination(file=open("example-eliot.log", "ab")), reactor) # Manually start the service, which will add it as a # destination. Normally we'd register ThreadedWriter with the usual # Twisted Service/Application infrastructure. logWriter.startService() # Log a message: log_message(message_type="test", value="hello", another=1) # Manually stop the service. done = logWriter.stopService() return done
def await_file_contents(path, contents, timeout=15): """ Return a deferred that fires when the file at `path` (any path-like object) has the exact content `contents`. :raises ExpectedFileMismatchException: if the path doesn't have the expected content after the timeout. :raises ExpectedFileUnfoundException: if the path doesn't exist after the the timeout. """ assert isinstance(contents, bytes), "file-contents must be bytes" from twisted.internet import reactor start_time = reactor.seconds() while reactor.seconds() - start_time < timeout: print(" waiting for '{}'".format(path)) if exists(path): try: with open(path, 'rb') as f: current = f.read() except IOError: print("IOError; trying again") else: if current == contents: return print(" file contents still mismatched") # annoying if we dump huge files to console if len(contents) < 80: print(" wanted: {}".format( contents.decode("utf8").replace('\n', ' '))) print(" got: {}".format( current.decode("utf8").replace('\n', ' '))) log_message( message_type=u"integration:await-file-contents:mismatched", got=current.decode("utf8"), ) else: log_message( message_type=u"integration:await-file-contents:missing", ) yield twisted_sleep(reactor, 1) if exists(path): raise ExpectedFileMismatchException(path, timeout) raise ExpectedFileUnfoundException(path, timeout)
def report(m): log_message(message_type="integration:cleanup", message=m) print(m)
""" Write some logs to journald. """ from __future__ import print_function from eliot import log_message, start_action, add_destinations from eliot.journald import JournaldDestination add_destinations(JournaldDestination()) def divide(a, b): with start_action(action_type="divide", a=a, b=b): return a / b print(divide(10, 2)) log_message(message_type="inbetween") print(divide(10, 0))
def main(): log_message(message_type="test", value="hello", another=1) time.sleep(0.2) log_message(message_type="test", value="goodbye", another=2)
def errReceived(self, data): print("ERR: {}".format(data.decode(sys.getfilesystemencoding()))) with self._action.context(): log_message(message_type=u"err-received", data=data.decode(sys.getfilesystemencoding())) self.output.write(data.decode(sys.getfilesystemencoding()))
def resume_tahoe(self): log_message(message_type=u"integation:tahoe-node:resume", node=self.name) print("resume tahoe: {}".format(self.name)) self.tahoe.resume()
def pause_tahoe(self): log_message(message_type=u"integation:tahoe-node:pause", node=self.name) print("suspend tahoe: {}".format(self.name)) self.tahoe.suspend()
def update_keycloak_users(user_updates: List[UserUpdate]): keycloak_admin = create_keycloak_admin_client() with start_action(action_type="get_keycloak_users") as action: keycloak_users = keycloak_admin.get_users({'search': '@'}) action.add_success_fields(keycloak_users=len(keycloak_users)) with start_action(action_type="get_keycloak_groups") as action: parent_group = keycloak_admin.get_group_by_path( settings.parent_group_path) group_ids_by_name = { g['name']: g['id'] for g in find_all_groups(parent_group) } action.add_success_fields(parent_group_name=parent_group["name"], parent_group_id=parent_group["id"], num_groups=len(group_ids_by_name)) updates_by_sync_id = {u.sync_id: u for u in user_updates} updates_by_email = {u.email: u for u in user_updates} used_sync_ids, duplicate_sync_ids = get_used_and_dup_sync_ids( keycloak_users) keycloak_users_no_verified_email = [ user for user in keycloak_users if not user.get("emailVerified") ] with start_action(action_type="non_verified_users"): for user in keycloak_users_no_verified_email: remove_user_attributes_and_groups(keycloak_admin, user) logout_everywhere(keycloak_admin, user) keycloak_users_verified_email = [ user for user in keycloak_users if user.get("emailVerified") ] with start_action(action_type="update_keycloak"): for user in keycloak_users_verified_email: try: with start_action(action_type="update_keycloak_user", user_id=user["id"]): user_update = get_user_update(user, updates_by_email, updates_by_sync_id, used_sync_ids, duplicate_sync_ids) update_keycloak_user_attrs(keycloak_admin, user, user_update) update_keycloak_user_group(keycloak_admin, parent_group["id"], user, user_update, group_ids_by_name) # Disable if syncing failed for a user (e.g. user no longer a member) except SyncCheckFailed as e: with start_action(action_type="sync_check_failed", user_id=user["id"], problem=str(e)): disable_keycloak_user(keycloak_admin, user, str(e)) logout_everywhere(keycloak_admin, user) # Ignore other exceptions except Exception: log_message(message_type="user_update_exception", user_id=user["id"], exception=e)
def parse_department(department, departments, cur_i, name_field_num=0) -> (int, List[Department]): """ Computes all sub-departments for the given department. :param department: The current department (may be a dummy department when a certain layer doesn't exist) :param departments: The list of all departments :param cur_i: The next department id in the department list :param name_field_num: The current id of the name field to use :return: Returns the current department id """ name_field_num += 1 unassigned_departments = [] while cur_i < len(departments): row = departments[cur_i] internal_name = row[settings.internal_name] with start_action(action_type="parse_department", row=cur_i, name_field=settings.name_fields[name_field_num], internal_name=internal_name) as action: try: higher_level = False for i in range(name_field_num): # Department on higher level if get_department_name(row, i): higher_level = True break if higher_level: action.add_success_fields(state="higher_level") break # Department doesn't exist anymore ignore_after_date = row[settings.ignore_after_name] if ignore_after_date and datetime.strptime( ignore_after_date, settings.ignore_after_format) <= datetime.now(): cur_i += 1 action.add_success_fields(state="disbanded") continue name = get_department_name(row, name_field_num) child_department = Department(internal_name=internal_name, name=name, children=[]) # Valid department on this level, sub departments start on next line if name: action.add_success_fields(name=name) next_line = cur_i + 1 # Invalid department on this level, this may be a sub department else: action.add_success_fields(name="Empty") next_line = cur_i # Parse children if not already at lowest department level if len(settings.name_fields) > name_field_num + 1: (cur_i, unassigned_sub_departments) = parse_department( child_department, departments, next_line, name_field_num) unassigned_departments += unassigned_sub_departments action.add_success_fields(unassigned_sub_departments=len( unassigned_sub_departments)) # Valid department on this level if name: action.add_success_fields(state="valid") department.children.append(child_department) # No valid department, but valid on lower level elif len(child_department.children) > 0: action.add_success_fields(state="valid_lower_level") department.children += child_department.children # No valid department on any level (no name given) else: action.add_success_fields(state="unassigned") # Add as child to default layer department (we just don't know if the level is right) unassigned_departments.append(child_department) # Add all unassigned departments (having no valid name) from lower levels if this is the default level if len( unassigned_departments ) > 0 and name_field_num == settings.name_default and name: child_department.children += unassigned_departments action.add_success_fields( unassigned_taken=len(unassigned_departments)) unassigned_departments.clear() action.add_success_fields( children=len(child_department.children)) except Exception as e: log_message(message_type="parse_department_failure", line=cur_i, exception=e) cur_i += 1 return cur_i - 1, unassigned_departments