def prepare_user_updates(csv_filepath: str) -> List[UserUpdate]:
with start_action(action_type="read_user_csv",
csv_filepath=csv_filepath) as action:
with open(csv_filepath) as f:
reader = csv.DictReader(f,
quotechar=settings.csv_quotechar,
strict=True,
delimiter=settings.csv_delimiter)
user_rows = list(reader)
action.add_success_fields(len_user_rows=len(user_rows))
users = []
with start_action(action_type="prepare_updates"):
for ll, row in enumerate(user_rows):
try:
sync_id = row[settings.field_sync_id]
user = UserUpdate(email=row[settings.field_email],
sync_id=sync_id)
if settings.field_verified:
user.verified = row[settings.field_verified] == "1"
if settings.field_eligible:
user.eligible = row[settings.field_eligible] == "1"
if settings.field_department:
user.department = row[settings.field_department]
users.append(user)
except Exception as e:
log_message(message_type="prepare_user_failure",
line=ll,
exception=e)
return users
def processEnded(self, reason):
with self._action:
log_message(message_type=u"process-ended")
if self.magic_seen is not None:
d, self.magic_seen = self.magic_seen, None
d.errback(Exception("Service failed."))
self.exited.callback(None)
def make_wsgi_app(settings_filepath=None, testing=False):
with start_action(action_type='morepath_scan'):
morepath.autoscan()
morepath.scan(ekklesia_portal)
if testing:
log_message(
message_type="testing",
msg="running in testing mode, not loading any config from file")
else:
with start_action(action_type='settings'):
settings = get_app_settings(settings_filepath)
App._loaded_settings = settings
App.init_settings(settings)
with start_action(action_type='make_app'):
App.commit()
app = App()
database.configure_sqlalchemy(app.settings.database, testing)
app.babel_init()
app.babel.localeselector(get_locale)
log_message(message_type="environment",
env=dict(os.environ),
encoding=locale.getpreferredencoding(),
default_locale=locale.getdefaultlocale())
return app
def create_department_group(keycloak_admin: keycloak.KeycloakAdmin, department: Department, parent_id=None, path="/"):
with start_action(action_type="create_department_group", department=department.internal_name, path=path):
# Create or update group
keycloak_admin.create_group({"name": department.internal_name, "attributes": {"display_name": [department.name]}},
parent=parent_id, skip_exists=True)
# Extend path
path += department.internal_name
# Get group id
try:
created_group = keycloak_admin.get_group_by_path(path, search_in_subgroups=True)
except keycloak.KeycloakGetError as e:
log_message(message_type="get_group_failed", exception=e)
return
# Add slash to group path
path += "/"
# Create groups for sub departments
subgroup_names = []
for sub_department in department.children:
create_department_group(keycloak_admin, sub_department, parent_id=created_group["id"], path=path)
subgroup_names.append(sub_department.internal_name)
# Delete old subgroups
for old_group in created_group["subGroups"]:
if old_group["name"] not in subgroup_names:
keycloak_admin.delete_group(old_group["id"])
def log(self, sql, timestamp, duration, curs):
notices = [notice.strip() for notice in self.notices]
duration_ms = (duration * 1000)
duration_formatted = f"{duration_ms:.2f}ms"
log_entry = dict(duration=duration_formatted, duration_ms=duration, sql=sql)
if notices:
log_entry['notices'] = notices
log_message('sql-statement', **log_entry)
self._history.append(sql, timestamp, duration, notices)
def eliot_garbage_received(self, data):
"""
Called when non-JSON lines are received on FD 3.
Since FD 3 is suppposed to only have eliot-logs, log them as malformed.
"""
with self._action.context():
log_message(message_type=u"malformed-eliot-log",
data=data.decode("utf8"))
def emit(self, record):
eliot.log_message(
message_type=record.name,
log_level=record.levelname,
# logger=record.name,
message=record.getMessage(),
)
if record.exc_info:
write_traceback(exc_info=record.exc_info)
def err_received(self, data):
"""
Called when non-JSON lines are received on stderr.
On Windows we use stderr for eliot logs from magic-folder.
But neither magic-folder nor tahoe guarantee that there is
no other output there, so we treat it as expected.
"""
with self._action.context():
log_message(message_type=u"err-received", data=data.decode("utf8"))
sys.stdout.write(data.decode("utf8"))
def periodic_scan(node, folder_name, path):
"""
Wait for the given magic folder to run a periodic scan. This
should cause the given path to be snapshotted.
:param MagicFolderEnabledNode node: The node on which to do the scan.
"""
from twisted.internet import reactor
log_message(message_type="integration:wait_for_scan",
node=node.name,
folder=folder_name)
# XXX need a better way than "wait 3 seconds" to know if the scan is completed...
return twisted_sleep(reactor, 3.0)
def out_received(self, data):
"""
Called with output from stdout.
"""
with self._action.context():
log_message(message_type=u"out-received", data=data.decode("utf8"))
sys.stdout.write(data.decode("utf8"))
self._output.write(data.decode("utf8"))
if self.magic_seen is not None and self._magic_text in self._output.getvalue(
):
print("Saw '{}' in the logs".format(self._magic_text))
d, self.magic_seen = self.magic_seen, None
d.callback(self)
def start_magic_folder(self):
if self.magic_folder is not None:
return
# We log a notice that we are starting the service in the context of the test
# but the logs of the service are in the context of the fixture.
log_message(message_type=u"integation:magic-folder:start",
node=self.name)
with self.action.context():
self.magic_folder = yield _run_magic_folder(
self.reactor,
self.request,
self.base_dir,
self.name,
)
def change_language(self, request):
lang = request.POST.get('lang')
if lang not in request.app.settings.app.languages:
raise HTTPBadRequest("unsupported language")
back_url = request.POST.get('back_url')
parsed_app_url = urlparse(request.application_url)
parsed_back_url = urlparse(back_url)
if parsed_app_url.netloc != parsed_back_url.netloc:
log_message(message_type="invalid_redirect", url=back_url)
raise HTTPBadRequest("redirect not allowed")
request.browser_session['lang'] = lang
return redirect(back_url)
def stop_magic_folder(self):
log_message(message_type=u"integation:magic-folder:stop",
node=self.name)
if self.magic_folder is None:
return
try:
log_message(
message_type=u"integation:magic-folder:stop",
node=self.name,
signal="TERM",
)
self.magic_folder.signalProcess('TERM')
yield self.magic_folder.proto.exited
self.magic_folder = None
except ProcessExitedAlready:
pass
def main(reactor):
print("Logging to example-eliot.log...")
logWriter = ThreadedWriter(
FileDestination(file=open("example-eliot.log", "ab")), reactor)
# Manually start the service, which will add it as a
# destination. Normally we'd register ThreadedWriter with the usual
# Twisted Service/Application infrastructure.
logWriter.startService()
# Log a message:
log_message(message_type="test", value="hello", another=1)
# Manually stop the service.
done = logWriter.stopService()
return done
def await_file_contents(path, contents, timeout=15):
"""
Return a deferred that fires when the file at `path` (any path-like
object) has the exact content `contents`.
:raises ExpectedFileMismatchException: if the path doesn't have the
expected content after the timeout.
:raises ExpectedFileUnfoundException: if the path doesn't exist after the
the timeout.
"""
assert isinstance(contents, bytes), "file-contents must be bytes"
from twisted.internet import reactor
start_time = reactor.seconds()
while reactor.seconds() - start_time < timeout:
print(" waiting for '{}'".format(path))
if exists(path):
try:
with open(path, 'rb') as f:
current = f.read()
except IOError:
print("IOError; trying again")
else:
if current == contents:
return
print(" file contents still mismatched")
# annoying if we dump huge files to console
if len(contents) < 80:
print(" wanted: {}".format(
contents.decode("utf8").replace('\n', ' ')))
print(" got: {}".format(
current.decode("utf8").replace('\n', ' ')))
log_message(
message_type=u"integration:await-file-contents:mismatched",
got=current.decode("utf8"),
)
else:
log_message(
message_type=u"integration:await-file-contents:missing", )
yield twisted_sleep(reactor, 1)
if exists(path):
raise ExpectedFileMismatchException(path, timeout)
raise ExpectedFileUnfoundException(path, timeout)
def report(m):
log_message(message_type="integration:cleanup", message=m)
print(m)
"""
Write some logs to journald.
"""
from __future__ import print_function
from eliot import log_message, start_action, add_destinations
from eliot.journald import JournaldDestination
add_destinations(JournaldDestination())
def divide(a, b):
with start_action(action_type="divide", a=a, b=b):
return a / b
print(divide(10, 2))
log_message(message_type="inbetween")
print(divide(10, 0))
def main():
log_message(message_type="test", value="hello", another=1)
time.sleep(0.2)
log_message(message_type="test", value="goodbye", another=2)
def errReceived(self, data):
print("ERR: {}".format(data.decode(sys.getfilesystemencoding())))
with self._action.context():
log_message(message_type=u"err-received",
data=data.decode(sys.getfilesystemencoding()))
self.output.write(data.decode(sys.getfilesystemencoding()))
def resume_tahoe(self):
log_message(message_type=u"integation:tahoe-node:resume",
node=self.name)
print("resume tahoe: {}".format(self.name))
self.tahoe.resume()
def pause_tahoe(self):
log_message(message_type=u"integation:tahoe-node:pause",
node=self.name)
print("suspend tahoe: {}".format(self.name))
self.tahoe.suspend()
def update_keycloak_users(user_updates: List[UserUpdate]):
keycloak_admin = create_keycloak_admin_client()
with start_action(action_type="get_keycloak_users") as action:
keycloak_users = keycloak_admin.get_users({'search': '@'})
action.add_success_fields(keycloak_users=len(keycloak_users))
with start_action(action_type="get_keycloak_groups") as action:
parent_group = keycloak_admin.get_group_by_path(
settings.parent_group_path)
group_ids_by_name = {
g['name']: g['id']
for g in find_all_groups(parent_group)
}
action.add_success_fields(parent_group_name=parent_group["name"],
parent_group_id=parent_group["id"],
num_groups=len(group_ids_by_name))
updates_by_sync_id = {u.sync_id: u for u in user_updates}
updates_by_email = {u.email: u for u in user_updates}
used_sync_ids, duplicate_sync_ids = get_used_and_dup_sync_ids(
keycloak_users)
keycloak_users_no_verified_email = [
user for user in keycloak_users if not user.get("emailVerified")
]
with start_action(action_type="non_verified_users"):
for user in keycloak_users_no_verified_email:
remove_user_attributes_and_groups(keycloak_admin, user)
logout_everywhere(keycloak_admin, user)
keycloak_users_verified_email = [
user for user in keycloak_users if user.get("emailVerified")
]
with start_action(action_type="update_keycloak"):
for user in keycloak_users_verified_email:
try:
with start_action(action_type="update_keycloak_user",
user_id=user["id"]):
user_update = get_user_update(user, updates_by_email,
updates_by_sync_id,
used_sync_ids,
duplicate_sync_ids)
update_keycloak_user_attrs(keycloak_admin, user,
user_update)
update_keycloak_user_group(keycloak_admin,
parent_group["id"], user,
user_update, group_ids_by_name)
# Disable if syncing failed for a user (e.g. user no longer a member)
except SyncCheckFailed as e:
with start_action(action_type="sync_check_failed",
user_id=user["id"],
problem=str(e)):
disable_keycloak_user(keycloak_admin, user, str(e))
logout_everywhere(keycloak_admin, user)
# Ignore other exceptions
except Exception:
log_message(message_type="user_update_exception",
user_id=user["id"],
exception=e)
def parse_department(department,
departments,
cur_i,
name_field_num=0) -> (int, List[Department]):
"""
Computes all sub-departments for the given department.
:param department: The current department (may be a dummy department when a certain layer doesn't exist)
:param departments: The list of all departments
:param cur_i: The next department id in the department list
:param name_field_num: The current id of the name field to use
:return: Returns the current department id
"""
name_field_num += 1
unassigned_departments = []
while cur_i < len(departments):
row = departments[cur_i]
internal_name = row[settings.internal_name]
with start_action(action_type="parse_department",
row=cur_i,
name_field=settings.name_fields[name_field_num],
internal_name=internal_name) as action:
try:
higher_level = False
for i in range(name_field_num):
# Department on higher level
if get_department_name(row, i):
higher_level = True
break
if higher_level:
action.add_success_fields(state="higher_level")
break
# Department doesn't exist anymore
ignore_after_date = row[settings.ignore_after_name]
if ignore_after_date and datetime.strptime(
ignore_after_date,
settings.ignore_after_format) <= datetime.now():
cur_i += 1
action.add_success_fields(state="disbanded")
continue
name = get_department_name(row, name_field_num)
child_department = Department(internal_name=internal_name,
name=name,
children=[])
# Valid department on this level, sub departments start on next line
if name:
action.add_success_fields(name=name)
next_line = cur_i + 1
# Invalid department on this level, this may be a sub department
else:
action.add_success_fields(name="Empty")
next_line = cur_i
# Parse children if not already at lowest department level
if len(settings.name_fields) > name_field_num + 1:
(cur_i, unassigned_sub_departments) = parse_department(
child_department, departments, next_line,
name_field_num)
unassigned_departments += unassigned_sub_departments
action.add_success_fields(unassigned_sub_departments=len(
unassigned_sub_departments))
# Valid department on this level
if name:
action.add_success_fields(state="valid")
department.children.append(child_department)
# No valid department, but valid on lower level
elif len(child_department.children) > 0:
action.add_success_fields(state="valid_lower_level")
department.children += child_department.children
# No valid department on any level (no name given)
else:
action.add_success_fields(state="unassigned")
# Add as child to default layer department (we just don't know if the level is right)
unassigned_departments.append(child_department)
# Add all unassigned departments (having no valid name) from lower levels if this is the default level
if len(
unassigned_departments
) > 0 and name_field_num == settings.name_default and name:
child_department.children += unassigned_departments
action.add_success_fields(
unassigned_taken=len(unassigned_departments))
unassigned_departments.clear()
action.add_success_fields(
children=len(child_department.children))
except Exception as e:
log_message(message_type="parse_department_failure",
line=cur_i,
exception=e)
cur_i += 1
return cur_i - 1, unassigned_departments
