def get_history(self, room_id: str, limit: int = 100):
if limit is None:
limit = -1
messages = self.redis.lrange(RedisKeys.room_history(room_id), 0, limit)
cleaned_messages = list()
for message_entry in messages:
message_entry = str(message_entry, 'utf-8')
msg_id, published, user_id, user_name, target_name, channel_id, channel_name, msg = \
message_entry.split(',', 7)
cleaned_messages.append({
'message_id': msg_id,
'from_user_id': user_id,
'from_user_name': b64d(user_name),
'target_id': room_id,
'target_name': b64d(target_name),
'body': b64d(msg),
'domain': 'room',
'channel_id': channel_id,
'channel_name': b64d(channel_name),
'timestamp': published,
'deleted': False
})
return cleaned_messages
def store_message(self, activity: Activity, deleted=False) -> None:
message = b64d(activity.object.content)
actor_name = b64d(activity.actor.display_name)
self.driver.msg_insert(msg_id=activity.id,
from_user_id=activity.actor.id,
from_user_name=actor_name,
target_id=activity.target.id,
target_name=activity.target.display_name,
body=message,
domain=activity.target.object_type,
sent_time=activity.published,
channel_id=activity.object.url,
channel_name=activity.object.display_name,
deleted=deleted)
def create_room(arg: tuple) -> None:
data, activity = arg
room_name = activity.target.display_name
room_id = activity.target.id
channel_id = activity.object.url
user_id = activity.actor.id
user_name = activity.actor.display_name
object_type = 'unknown'
if hasattr(activity.target, 'object_type'):
object_type = activity.target.object_type
is_ephemeral = object_type != 'private'
owners = OnCreateHooks._get_owners(activity)
if utils.is_base64(room_name):
room_name = utils.b64d(room_name)
environ.env.db.create_room(room_name,
room_id,
channel_id,
user_id,
user_name,
ephemeral=is_ephemeral)
logger.debug('settings "{}" as owners of room {}'.format(
','.join(owners), room_id))
for owner_id in owners:
environ.env.db.set_owner(room_id, owner_id)
def ban_user():
form = request.get_json()
target = form['target']
target_uuid = form['target_uuid']
user_uuid = form['user_uuid']
duration = form['duration']
try:
user_manager.ban_user(user_uuid, target_uuid, duration, target)
except ValidationException as e:
return api_response(400, message='invalid duration: %s' % str(e))
except UnknownBanTypeException as e:
return api_response(400, message='could not ban user: %s' % str(e))
except Exception as e:
logger.exception(traceback.format_exc())
return api_response(400, message=str(e))
try:
user = user_manager.get_user(user_uuid)
user['name'] = utils.b64d(user['name'])
user['duration'] = duration
except NoSuchUserException:
return api_response(400, message="No such user.")
if target == 'channel':
user['channel'] = {
'uuid': target_uuid,
'name': channel_manager.name_for_uuid(target_uuid)
}
elif target == 'room':
user['room'] = {
'uuid': target_uuid,
'name': room_manager.name_for_uuid(target_uuid)
}
return api_response(200, user)
def publish_activity() -> None:
user_name = activity.actor.display_name
if utils.is_base64(user_name):
user_name = utils.b64d(user_name)
activity_json = utils.activity_for_message(user_id, user_name)
environ.env.publish(activity_json, external=True)
def test_join_returns_activity_with_correct_owner(self):
self.set_owner()
act = self.activity_for_join()
response = api.on_join(act, as_parser(act))
attachments = response[1]['object']['attachments']
owners = self.get_attachment_for_key(attachments, 'owner')
user_id, user_name = owners[0]['id'], owners[0]['displayName']
self.assertEqual(ApiJoinTest.USER_ID, user_id)
self.assertEqual(ApiJoinTest.USER_NAME, b64d(user_name))
def test_get_acls_channel(self):
self._create_channel()
self.manager.add_acl_channel(BaseDatabaseTest.CHANNEL_ID,
ApiActions.LIST, 'age', '25:45')
acls = self.manager.get_acls_channel(BaseDatabaseTest.CHANNEL_ID)
self.assertEqual(1, len(acls))
self.assertEqual('age', acls[0]['type'])
self.assertEqual('25:45', b64d(acls[0]['value']))
self.assertEqual(ApiActions.LIST, acls[0]['action'])
def decode_or_throw(self, b64_word: str) -> str:
if not utils.is_base64(b64_word):
logger.error('word is not base64 encoded: "%s"' % b64_word)
raise RuntimeError('word is not base64 encoded: "%s"' % b64_word)
try:
return utils.b64d(b64_word)
except Exception as e:
logger.error('could not decode base64 word "%s": %s' % (str(b64_word), str(e)))
raise RuntimeError('could not decode base64 word "%s": %s' % (str(b64_word), str(e)))
def test_get_acls_room(self):
self._create_channel()
self._create_room()
self.manager.add_acl_room(BaseDatabaseTest.ROOM_ID, ApiActions.JOIN,
'age', '25:45')
acls = self.manager.get_acls_room(BaseDatabaseTest.ROOM_ID)
self.assertEqual(1, len(acls))
self.assertEqual('age', acls[0]['type'])
self.assertEqual('25:45', b64d(acls[0]['value']))
self.assertEqual(ApiActions.JOIN, acls[0]['action'])
def test_list_rooms_correct_name(self):
self.assert_in_room(False)
self.create_and_join_room()
self.assert_in_room(True)
act = self.activity_for_list_rooms()
response_data = api.on_list_rooms(act, as_parser(act))
self.assertEqual(
ApiListRoomsTest.ROOM_NAME,
b64d(response_data[1]['object']['attachments'][0]['displayName']))
def test_history_contains_correct_user_name(self):
self.join_room()
self.remove_owner()
self.remove_owner_channel()
self.send_message('my message')
act = self.activity_for_history()
response_data = api.on_history(act, as_parser(act))
activity = as_parser(response_data[1])
self.assertEqual(BaseTest.USER_NAME, b64d(activity.object.attachments[0].author.display_name))
def test_users_in_room_is_correct_name(self):
self.assert_in_room(False)
act = self.activity_for_join()
api.on_join(act, as_parser(act))
self.assert_in_room(True)
act = self.activity_for_users_in_room()
response_data = api.on_users_in_room(act, as_parser(act))
self.assertEqual(
ApiUsersInRoomTest.USER_NAME,
b64d(response_data[1]['object']['attachments'][0]['displayName']))
def test_join_returns_correct_history(self):
msg = 'this is a test message'
self.set_owner()
self.assert_join_succeeds()
msg_response = self.send_message(msg)[1]
self.leave_room()
act = self.activity_for_join()
response = api.on_join(act, as_parser(act))
attachments = response[1]['object']['attachments']
from pprint import pprint
pprint(self.get_attachment_for_key(attachments, 'history'))
all_history = self.get_attachment_for_key(attachments, 'history')
self.assertEqual(1, len(all_history))
history_obj = all_history[0]
self.assertEqual(msg_response['id'], history_obj['id'])
self.assertEqual(msg, b64d(history_obj['content']))
self.assertEqual(msg_response['published'], history_obj['published'])
self.assertEqual(ApiJoinTest.USER_NAME, b64d(history_obj['author']['displayName']))
def banned():
ban_form = BanForm(request.form)
if request.method == 'POST' and ban_form.validate():
try:
user_manager.ban_user(ban_form.uuid.data, ban_form.target_id.data,
ban_form.duration.data,
ban_form.target_type.data)
return redirect('/banned')
except ValidationException as e:
ban_form.target_type.errors.append('Ban not valid: "%s"' % e.msg)
except UnknownBanTypeException as e:
ban_form.target_type.errors.append('Unkonwn ban type "%s"' %
e.ban_type)
bans = user_manager.get_banned_users()
channel_bans = bans['channels']
for channel_id in channel_bans:
channel_bans[channel_id]['name'] = utils.b64d(
channel_bans[channel_id]['name'])
for user_id in channel_bans[channel_id]['users']:
channel_bans[channel_id]['users'][user_id]['name'] = \
utils.b64d(channel_bans[channel_id]['users'][user_id]['name'])
room_bans = bans['rooms']
for room_id in room_bans:
room_bans[room_id]['name'] = utils.b64d(room_bans[room_id]['name'])
for user_id in room_bans[room_id]['users']:
room_bans[room_id]['users'][user_id]['name'] = \
utils.b64d(room_bans[room_id]['users'][user_id]['name'])
global_bans = bans['global']
for user_id in global_bans:
global_bans[user_id]['name'] = \
utils.b64d(global_bans[user_id]['name'])
return render_template('banned.html',
form=ban_form,
globally=global_bans,
channels=channel_bans,
rooms=room_bans)
def test_history_contains_correct_sent_message(self):
self.join_room()
self.remove_owner()
self.remove_owner_channel()
message = 'my message'
self.send_message(message)
act = self.activity_for_history()
response_data = api.on_history(act, as_parser(act))
activity = as_parser(response_data[1])
self.assertEqual(message, b64d(activity.object.attachments[0].content))
def add_room_moderator(room_uuid: str):
form = request.get_json()
user_uuid = form['moderator']
if is_blank(user_uuid):
return api_response(400, message='Blank user id is not allowed.')
try:
user = user_manager.get_user(user_uuid)
user['name'] = utils.b64d(user['name'])
except NoSuchUserException:
return api_response(400, message='No Such User.')
user_manager.add_room_moderator(room_uuid, user_uuid)
return api_response(200, user)
def get_channel(channel_uuid: str):
""" Get channel owners/admins/acls """
acls = acl_manager.get_acls_channel(channel_uuid)
acls_decoded = list()
for acl in acls:
acl['value'] = utils.b64d(acl['value'])
acls_decoded.append(acl)
return api_response(
200, {
'owners': channel_manager.get_owners(channel_uuid),
'admins': channel_manager.get_admins(channel_uuid),
'acls': acls_decoded,
})
def banned_users():
bans = user_manager.get_banned_users()
result = {'global': list(), 'channel': list(), 'room': list()}
channel_bans = bans['channels']
for channel_id in channel_bans:
channel = {
'name': utils.b64d(channel_bans[channel_id]['name']),
'uuid': channel_id
}
for user_id in channel_bans[channel_id]['users']:
user = channel_bans[channel_id]['users'][user_id]
user['uuid'] = user_id
user['name'] = utils.b64d(user['name'])
user['channel'] = channel
result['channel'].append(user)
room_bans = bans['rooms']
for room_id in room_bans:
room = {
'name': utils.b64d(room_bans[room_id]['name']),
'uuid': room_id
}
for user_id in room_bans[room_id]['users']:
user = room_bans[room_id]['users'][user_id]
user['uuid'] = user_id
user['name'] = utils.b64d(user['name'])
user['room'] = room
result['room'].append(user)
global_bans = bans['global']
for user_id in global_bans:
user = global_bans[user_id]
user['uuid'] = user_id
user['name'] = utils.b64d(user['name'])
result['global'].append(user)
return api_response(200, result)
def on_create(self, activity: Activity) -> (bool, int, str):
if not hasattr(activity, 'object') or not hasattr(
activity.object, 'url'):
return False, ECodes.MISSING_OBJECT_URL, 'no channel id set'
if not hasattr(activity.target, 'display_name'):
return False, ECodes.MISSING_TARGET_DISPLAY_NAME, 'no room name set'
room_name = activity.target.display_name
channel_id = activity.object.url
if not hasattr(activity, 'actor') or not hasattr(activity.actor, 'id'):
return False, ECodes.MISSING_ACTOR_ID, 'need actor.id (user uuid)'
try:
activity.object.display_name = utils.get_channel_name(channel_id)
except NoSuchChannelException:
return False, ECodes.NO_SUCH_CHANNEL, 'channel does not exist'
if room_name is None or room_name.strip() == '':
return False, ECodes.MISSING_TARGET_DISPLAY_NAME, 'got blank room name, can not create'
if not utils.is_base64(room_name):
return False, ECodes.NOT_BASE64, 'invalid room name, not base64 encoded'
room_name = utils.b64d(room_name)
if not environ.env.db.channel_exists(channel_id):
return False, ECodes.NO_SUCH_CHANNEL, 'channel does not exist'
if utils.room_name_restricted(room_name):
return False, ECodes.ROOM_NAME_RESTRICTED, 'restricted room name'
if environ.env.db.room_name_exists(channel_id, room_name):
return False, ECodes.ROOM_ALREADY_EXISTS, 'a room with that name already exists'
if not hasattr(activity.target, 'object_type') or \
activity.target.object_type is None or \
len(str(activity.target.object_type).strip()) == 0:
# for acl validation to know we're trying to create a room
activity.target.object_type = 'room'
channel_acls = utils.get_acls_in_channel_for_action(
channel_id, ApiActions.CREATE)
is_valid, msg = validation.acl.validate_acl_for_action(
activity, ApiTargets.CHANNEL, ApiActions.CREATE, channel_acls)
if not is_valid:
return False, ECodes.NOT_ALLOWED, msg
return True, None, None
def _process(self, data: dict, activity: Activity):
message = activity.object.content
if message is None or len(message.strip()) == 0:
return True, None, None
if not utils.is_base64(message):
return False, ErrorCodes.NOT_BASE64, \
'invalid message content, not base64 encoded'
message = utils.b64d(message)
if len(message) > self.max_length:
return False, ErrorCodes.MSG_TOO_LONG, \
'message content needs to be shorter than %s characters' % self.max_length
return True, None, None
def update_session_and_join_private_room(arg: tuple) -> None:
data, activity = arg
user_id = activity.actor.id
user_name = utils.b64d(activity.actor.display_name)
environ.env.session[SessionKeys.user_id.value] = user_id
environ.env.session[SessionKeys.user_name.value] = user_name
try:
user_agent_string = environ.env.request.user_agent.string
user_agent_platform = environ.env.request.user_agent.platform
user_agent_browser = environ.env.request.user_agent.browser
user_agent_version = environ.env.request.user_agent.version
user_agent_language = environ.env.request.user_agent.language
except Exception as e:
logger.error('could not get user agent for user "{}": {}'.format(
user_id, str(e)))
logger.exception(traceback.format_exc())
environ.env.capture_exception(sys.exc_info())
user_agent_string = ''
user_agent_platform = ''
user_agent_browser = ''
user_agent_version = ''
user_agent_language = ''
environ.env.session[
SessionKeys.user_agent.value] = user_agent_string or ''
environ.env.session[
SessionKeys.user_agent_browser.value] = user_agent_browser or ''
environ.env.session[
SessionKeys.user_agent_version.value] = user_agent_version or ''
environ.env.session[
SessionKeys.user_agent_platform.value] = user_agent_platform or ''
environ.env.session[
SessionKeys.user_agent_language.value] = user_agent_language or ''
if activity.actor.image is None:
environ.env.session['image_url'] = ''
environ.env.session[SessionKeys.image.value] = 'n'
else:
environ.env.session['image_url'] = activity.actor.image.url
environ.env.session[SessionKeys.image.value] = 'y'
sid = environ.env.request.sid
utils.create_or_update_user(user_id, user_name)
utils.add_sid_for_user_id(user_id, sid)
environ.env.join_room(user_id)
environ.env.join_room(environ.env.request.sid)
def get_room(channel_uuid: str, room_uuid: str):
acls = acl_manager.get_acls_room(room_uuid)
acls_decoded = list()
for acl in acls:
acl['value'] = utils.b64d(acl['value'])
acls_decoded.append(acl)
return api_response(
200, {
'channel': {
'uuid': channel_uuid,
'name': channel_manager.name_for_uuid(channel_uuid)
},
'acls': acls_decoded,
'owners': room_manager.get_owners(room_uuid),
'moderators': room_manager.get_moderators(room_uuid)
})
def async_post(self, json):
logger.debug('POST request: %s' % str(json))
if 'content' not in json:
raise RuntimeError('no key [content] in json message')
msg_content = json.get('content')
if msg_content is None or len(msg_content.strip()) == 0:
raise RuntimeError('content may not be blank')
if not utils.is_base64(msg_content):
raise RuntimeError('content in json message must be base64')
user_id = str(json.get('user_id', 0))
user_name = utils.b64d(json.get('user_name', utils.b64e('admin')))
object_type = json.get('object_type')
target_id = str(json.get('target_id'))
namespace = json.get('namespace', '/ws')
target_name = json.get('target_name')
data = utils.activity_for_message(user_id, user_name)
data['target'] = {
'objectType': object_type,
'id': target_id,
'displayName': target_name,
'url': namespace
}
data['object'] = {
'content': msg_content
}
if not environ.env.cache.user_is_in_multicast(target_id):
logger.info('user {} is offline, dropping message: {}'.format(target_id, str(json)))
return
try:
environ.env.out_of_scope_emit('message', data, room=target_id, json=True, namespace='/ws', broadcast=True)
except Exception as e:
logger.error('could not /send message to target {}: {}'.format(target_id, str(e)))
logger.exception(traceback.format_exc())
environ.env.capture_exception(sys.exc_info())
def _process(self, data: dict, activity: Activity):
room_name = activity.target.display_name
if room_name is None or room_name.strip() == '':
return False, ErrorCodes.MISSING_TARGET_DISPLAY_NAME, \
'got blank room name, can not create'
if not utils.is_base64(room_name):
return False, ErrorCodes.NOT_BASE64, \
'invalid room name, not base64 encoded'
room_name = utils.b64d(room_name)
if len(room_name) < self.min_length:
return False, ErrorCodes.ROOM_NAME_TOO_SHORT, \
'room name needs to be longer than %s characters' % self.min_length
if len(room_name) > self.max_length:
return False, ErrorCodes.ROOM_NAME_TOO_LONG, \
'room name needs to be shorter than %s characters' % self.max_length
return True, None, None
def ban_user(self, user_id: str, ban_info: dict):
target_type = ban_info.get('type', '')
target_id = ban_info.get('target', '')
duration = ban_info.get('duration', '')
reason = ban_info.get('reason', '')
banner_id = ban_info.get('admin_id', '')
try:
user_name = ban_info['name']
user_name = utils.b64d(user_name)
except KeyError:
logger.warning(
'no name specified in ban info, if we have to create the user it will get the ID as name'
)
user_name = user_id
try:
self.user_manager.ban_user(user_id,
target_id,
duration,
target_type,
reason=reason,
banner_id=banner_id,
user_name=user_name)
except ValueError as e:
logger.error('invalid ban duration "%s" for user %s: %s' %
(duration, user_id, str(e)))
self.env.capture_exception(sys.exc_info())
except NoSuchUserException as e:
logger.error('no such user %s: %s' % (user_id, str(e)))
self.env.capture_exception(sys.exc_info())
except UnknownBanTypeException as e:
logger.error('unknown ban type "%s" for user %s: %s' %
(target_type, user_id, str(e)))
self.env.capture_exception(sys.exc_info())
except Exception as e:
logger.error('could not ban user %s: %s' % (user_id, str(e)))
logger.error(traceback.format_exc())
self.env.capture_exception(sys.exc_info())
def _contains_blacklisted_word(self, activity: Activity):
message = activity.object.content
blacklist = self._get_black_list()
if blacklist is None or len(blacklist) == 0:
return None
if message is not None and len(message) > 0:
message = utils.b64d(message).lower()
contains_forbidden_word = any(word in message for word in blacklist)
if not contains_forbidden_word:
return None
for word in blacklist:
if word not in message:
continue
logger.warning(
'message from user %s used a blacklisted word "%s"' %
(activity.actor.id, word))
return word
return None
def rooms_for_channel(channel_uuid):
form = CreateRoomForm(request.form)
acl_form = CreateChannelAclForm(request.form)
owner_form = AddOwnerForm(request.form)
admin_form = AddAdminForm(request.form)
acls = acl_manager.get_acls_channel(channel_uuid)
acls_decoded = list()
for acl in acls:
acl['value'] = utils.b64d(acl['value'])
acls_decoded.append(acl)
return render_template(
'rooms_in_channel.html',
form=form,
owner_form=owner_form,
admin_form=admin_form,
acl_form=acl_form,
owners=channel_manager.get_owners(channel_uuid),
admins=channel_manager.get_admins(channel_uuid),
acls=acls_decoded,
channel_uuid=channel_uuid,
channel_name=channel_manager.name_for_uuid(channel_uuid),
rooms=room_manager.get_rooms(channel_uuid))
def users_for_room(channel_uuid, room_uuid):
owner_form = AddOwnerForm(request.form)
mod_form = AddModeratorForm(request.form)
acl_form = CreateRoomAclForm(request.form)
acls = acl_manager.get_acls_room(room_uuid)
acls_decoded = list()
for acl in acls:
acl['value'] = utils.b64d(acl['value'])
acls_decoded.append(acl)
return render_template(
'users_in_room.html',
channel_uuid=channel_uuid,
room_uuid=room_uuid,
owner_form=owner_form,
mod_form=mod_form,
acl_form=acl_form,
acls=acls_decoded,
channel_name=channel_manager.name_for_uuid(channel_uuid),
room_name=room_manager.name_for_uuid(room_uuid),
owners=room_manager.get_owners(room_uuid),
moderators=room_manager.get_moderators(room_uuid),
users=user_manager.get_users_for_room(room_uuid))
def test_b64d_invalid(self):
self.assertEqual('', utils.b64d('åäåö'))
def check_spam():
def remove_emojis(text):
return ''.join([
character for character in text
if character not in emoji.UNICODE_EMOJI
])
def remove_custom_emojis(text):
return re.sub(r':[a-z0-9]*:', '', text)
def remove_multiple_consecutive_chars(text):
return re.sub(r'(.)\1+', r'\1', text)
def remove_numbers(text):
return re.sub(r'\d', r'', text)
def remove_special_chars(text):
text = text.strip()
text = text.replace('*', '')
text = text.replace('+', '')
text = text.replace('"', '')
text = text.replace('_', '')
text = text.replace('\'', '')
text = text.replace('!', '')
text = text.replace('-', '')
text = text.replace('/', '')
text = text.replace(';', '')
text = text.replace('@', '')
text = text.replace('$', '')
text = text.replace('%', '')
text = text.replace('&', '')
text = text.replace(':', '')
text = text.replace('<', '')
text = text.replace('>', '')
text = text.replace('(', '')
text = text.replace(')', '')
return text
def replace_umlauts(text):
text = text.replace('å', 'a')
text = text.replace('ä', 'a')
text = text.replace('ö', 'o')
text = text.replace('ß', 's')
text = text.replace('ü', 'u')
return text
_is_spam = False
_spam_id = None
_message = None
spam_enabled = environ.env.config.get(ConfigKeys.SPAM_CLASSIFIER,
False)
if not spam_enabled:
return False, None
try:
_message = utils.b64d(activity.object.content)
try:
json_body = json.loads(_message)
_message = json_body.get('text')
except Exception:
pass # ignore, use original
except Exception as e:
logger.error('could not decode message: {}'.format(str(e)))
logger.exception(e)
environ.env.capture_exception(sys.exc_info())
return False, None
if environ.env.service_config.ignore_emoji():
try:
_message = remove_emojis(_message)
_message = remove_custom_emojis(_message)
except Exception as e:
logger.error(
'could not check if text has emojis: {}'.format(
str(e)))
logger.exception(e)
environ.env.capture_exception(sys.exc_info())
try:
_message = remove_multiple_consecutive_chars(_message)
_message = remove_special_chars(_message)
_message = remove_numbers(_message)
_message = replace_umlauts(_message)
_is_spam, _y_hats = environ.env.spam.is_spam(_message)
if _is_spam and environ.env.service_config.should_save_spam():
_spam_id = environ.env.db.save_spam_prediction(
activity, _message, _y_hats)
except Exception as e:
logger.error('could not predict spam: {}'.format(str(e)))
logger.exception(e)
environ.env.capture_exception(sys.exc_info())
return False, None
return _is_spam, _spam_id
