def db_update_enabled_languages(session, tid, languages, default_language):
"""
Transaction for updating the enabled languages for a tenant
:param session: An ORM session
:param tid: A tenant id
:param languages_enabled: The list of enabled languages
:param default_language: The language to be set as default
"""
cur_enabled_langs = db_get_languages(session, tid)
if len(languages) < 1:
raise errors.InputValidationError("No languages enabled!")
# get sure that the default language is included in the enabled languages
languages = set(languages + [default_language])
appdata = None
for lang_code in languages:
if lang_code not in LANGUAGES_SUPPORTED_CODES:
raise errors.InputValidationError("Invalid lang code: %s" % lang_code)
if lang_code not in cur_enabled_langs:
if appdata is None:
appdata = load_appdata()
log.debug("Adding a new lang %s" % lang_code)
models.config.add_new_lang(session, tid, lang_code, appdata)
to_remove = list(set(cur_enabled_langs) - set(languages))
if to_remove:
session.query(models.User).filter(models.User.tid == tid, models.User.language.in_(to_remove)).update({'language': default_language}, synchronize_session=False)
db_del(session, models.EnabledLanguage, (models.EnabledLanguage.tid == tid, models.EnabledLanguage.name.in_(to_remove)))
def db_update_enabled_languages(session, tid, languages_enabled,
default_language):
cur_enabled_langs = models.EnabledLanguage.list(session, tid)
new_enabled_langs = [text_type(y) for y in languages_enabled]
if len(new_enabled_langs) < 1:
raise errors.InputValidationError("No languages enabled!")
if default_language not in new_enabled_langs:
raise errors.InputValidationError(
"Invalid lang code for chosen default_language")
appdata = None
for lang_code in new_enabled_langs:
if lang_code not in LANGUAGES_SUPPORTED_CODES:
raise errors.InputValidationError("Invalid lang code: %s" %
lang_code)
if lang_code not in cur_enabled_langs:
if appdata is None:
appdata = load_appdata()
log.debug("Adding a new lang %s" % lang_code)
models.config.add_new_lang(session, tid, lang_code, appdata)
to_remove = list(set(cur_enabled_langs) - set(new_enabled_langs))
if to_remove:
session.query(
models.User).filter(models.User.tid == tid,
models.User.language.in_(to_remove)).update(
{'language': default_language},
synchronize_session='fetch')
session.query(models.EnabledLanguage).filter(
models.EnabledLanguage.tid == tid,
models.EnabledLanguage.name.in_(to_remove)).delete(
synchronize_session='fetch')
def check_hostname(session, tid, input_hostname):
"""
Ensure the hostname does not collide across tenants or include an origin
that it shouldn't.
"""
if input_hostname == '':
raise errors.InputValidationError('Hostname cannot be empty')
root_hostname = ConfigFactory(session, 1, 'node').get_val(u'hostname')
forbidden_endings = ['onion', 'localhost']
if tid != 1 and root_hostname != '':
forbidden_endings.append(root_hostname)
for v in forbidden_endings:
if input_hostname.endswith(v):
raise errors.InputValidationError(
'Hostname contains a forbidden origin')
existing_hostnames = {h.get_v() for h in session.query(Config) \
.filter(Config.tid != tid,
Config.var_name == u'hostname')}
if input_hostname in existing_hostnames:
raise errors.InputValidationError('Hostname already reserved')
def check_field_association(session, tid, field_dict):
if field_dict.get('fieldgroup_id', '') and session.query(
models.Field).filter(
models.Field.id == field_dict['fieldgroup_id'],
models.Field.tid != tid).count():
raise errors.InputValidationError()
if field_dict.get('template_id',
'') and session.query(models.Field).filter(
models.Field.id == field_dict['template_id'],
not_(models.Field.tid.in_(set([1, tid])))).count():
raise errors.InputValidationError()
if field_dict.get('step_id', '') and session.query(models.Field).filter(
models.Step.id == field_dict['step_id'], models.Questionnaire.id
== models.Step.questionnaire_id,
not_(models.Questionnaire.tid.in_(set([1, tid])))).count():
raise errors.InputValidationError()
if field_dict.get('fieldgroup_id', ''):
ancestors = set(
fieldtree_ancestors(session, field_dict['fieldgroup_id']))
if field_dict['id'] == field_dict['fieldgroup_id'] or field_dict[
'id'] in ancestors:
raise errors.InputValidationError(
"Provided field association would cause recursion loop")
def check_hostname(session, tid, hostname):
"""
Ensure the hostname does not collide across tenants or include an origin that it shouldn't.
:param session: An ORM session
:param tid: A tenant id
:param hostname: The hostname to be evaluated
"""
if hostname == '':
return
forbidden_endings = ['onion', 'localhost']
for v in forbidden_endings:
if hostname.endswith(v):
raise errors.InputValidationError(
'Hostname contains a forbidden origin')
existing_hostnames = {
h.value
for h in session.query(Config).filter(Config.tid != tid,
Config.var_name == 'hostname')
}
if hostname in existing_hostnames:
raise errors.InputValidationError('Hostname already reserved')
def check_field_association(session, tid, request):
"""
Transaction to check consistency of field association
:param session: The ORM session
:param tid: The tenant ID
:param request: The request data to be verified
"""
if request.get('fieldgroup_id', '') and session.query(models.Field).filter(
models.Field.id == request['fieldgroup_id'],
models.Field.tid != tid).count():
raise errors.InputValidationError()
if request.get('template_id', '') and session.query(models.Field).filter(
models.Field.id == request['template_id'],
not_(models.Field.tid.in_(set([1, tid])))).count():
raise errors.InputValidationError()
if request.get('step_id', '') and session.query(models.Field).filter(
models.Step.id == request['step_id'], models.Questionnaire.id
== models.Step.questionnaire_id,
not_(models.Questionnaire.tid.in_(set([1, tid])))).count():
raise errors.InputValidationError()
if request.get('fieldgroup_id', ''):
ancestors = set(fieldtree_ancestors(session, request['fieldgroup_id']))
if request['id'] == request['fieldgroup_id'] or request[
'id'] in ancestors:
raise errors.InputValidationError(
"Provided field association would cause recursion loop")
def validate_message(message, message_template):
try:
jmessage = json.loads(message)
except ValueError:
raise errors.InputValidationError("Invalid JSON format")
if BaseHandler.validate_jmessage(jmessage, message_template):
return jmessage
raise errors.InputValidationError("Unexpected condition!?")
def db_create_field(session, tid, field_dict, language):
"""
Create and add a new field to the session, then return the new serialized object.
:param session: the session on which perform queries.
:param field_dict: the field definition dict
:param language: the language of the field definition dict
:return: a serialization of the object
"""
field_dict['tid'] = tid
fill_localized_keys(field_dict, models.Field.localized_keys, language)
check_field_association(session, tid, field_dict)
field = models.db_forge_obj(session, models.Field, field_dict)
if field.template_id is not None:
# special handling of the whistleblower_identity field
if field.template_id == 'whistleblower_identity':
field_attrs = read_json_file(Settings.field_attrs_file)
attrs = field_attrs.get(field.template_id, {})
db_add_field_attrs(session, field.id, attrs)
if field.step_id is not None:
questionnaire = session.query(models.Questionnaire) \
.filter(models.Field.id == field.id,
models.Field.step_id == models.Step.id,
models.Step.questionnaire_id == models.Questionnaire.id,
models.Questionnaire.tid == tid).one()
if questionnaire.enable_whistleblower_identity is False:
questionnaire.enable_whistleblower_identity = True
else:
raise errors.InputValidationError(
"Whistleblower identity field already present")
else:
raise errors.InputValidationError(
"Cannot associate whistleblower identity field to a fieldgroup"
)
else:
attrs = field_dict.get('attrs', [])
options = field_dict.get('options', [])
db_update_fieldattrs(session, tid, field.id, attrs, language)
db_update_fieldoptions(session, tid, field.id, options, language)
if field.instance != 'reference':
for c in field_dict.get('children', []):
c['tid'] = field.tid
c['fieldgroup_id'] = field.id
db_create_field(session, tid, c, language)
return field
def validate_message(message, message_template):
try:
if isinstance(message, binary_type):
message = message.decode('utf-8')
jmessage = json.loads(message)
except ValueError:
raise errors.InputValidationError("Invalid JSON format")
if BaseHandler.validate_jmessage(jmessage, message_template):
return jmessage
raise errors.InputValidationError("Unexpected condition!?")
def delete_field(session, tid, field_id):
"""
Delete the field object corresponding to field_id
If the field has children, remove them as well.
If the field is immediately attached to a step object, remove it as well.
:param session: the session on which perform queries.
:param field_id: the id corresponding to the field.
"""
field = models.db_get(session, models.Field, models.Field.tid == tid,
models.Field.id == field_id)
if not field.editable:
raise errors.ForbiddenOperation
if field.instance == 'template' and session.query(models.Field).filter(
models.Field.tid == tid, models.Field.template_id
== field.id).count():
raise errors.InputValidationError(
"Cannot remove the field template as it is used by one or more questionnaires"
)
if field.template_id == 'whistleblower_identity' and field.step_id is not None:
session.query(models.Questionnaire) \
.filter(models.Questionnaire.tid == tid,
models.Step.id == field.step_id,
models.Questionnaire.id == models.Step.questionnaire_id).set(enable_whistleblower_identity = False)
session.delete(field)
def db_admin_update_user(session, state, tid, user_id, request, language):
"""
Updates the specified user.
"""
fill_localized_keys(request, models.User.localized_keys, language)
user = db_get_user(session, tid, user_id)
if user.username != request['username']:
check = session.query(models.User).filter(
models.User.username == text_type(request['username']),
models.UserTenant.user_id == models.User.id,
models.UserTenant.tenant_id == tid).one_or_none()
if check is not None:
raise errors.InputValidationError('Username already in use')
user.update(request)
password = request['password']
if password:
user.password = security.hash_password(password, user.salt)
user.password_change_date = datetime_now()
# The various options related in manage PGP keys are used here.
parse_pgp_options(state, user, request)
if user.role == 'admin':
db_refresh_memory_variables(session, [tid])
return user
def db_associate_context_receivers(session, context, receiver_ids):
"""
Transaction for associating receivers to a context
:param session: An ORM session
:param context: The context on which associate the specified receivers
:param receiver_ids: A list of receivers ids to be associated to the context
"""
db_del(session, models.ReceiverContext,
models.ReceiverContext.context_id == context.id)
if not receiver_ids:
return
if not session.query(models.Context).filter(
models.Context.id == context.id, models.Context.tid
== models.User.tid, models.User.id.in_(receiver_ids)).count():
raise errors.InputValidationError()
for i, receiver_id in enumerate(receiver_ids):
session.add(
models.ReceiverContext({
'context_id': context.id,
'receiver_id': receiver_id,
'order': i
}))
def parse_csv_ip_ranges_to_ip_networks(ip_str):
"""Takes a list of IP addresses and/or CIDRs, and converts them to a list
of python objects"""
ip_str = text_type(ip_str)
ip_network_list = []
for ip_network_str in ip_str.split(','):
# We want to normalize to IPvXNetwork, so we can run in comparsions on
# IP ranges for authentications. However, we may get IP addresses, CIDR
# ranges, or garbage. Python does provide strict=True with the ipaddress
# methods; however, it will accept any integer is which *not* what we want
# so we need to handle this carefully.
# If it has a /, we'll assume it's a CIDR address, otherwise, a raw IP
try:
if "/" in ip_network_str:
ip_net_obj = ipaddress.ip_network(ip_network_str, strict=True)
ip_network_list.append(ip_net_obj)
else:
# Let's try and see if we can work with this
ip_addr_obj = ipaddress.ip_address(ip_network_str)
# If we got here, it is, convert it to a proper /32 (or /128)
cidr_len = ip_addr_obj.max_prefixlen
ip_network = ipaddress.ip_network(ip_network_str + '/' + str(cidr_len))
ip_network_list.append(ip_network)
except ValueError:
raise errors.InputValidationError("Unable to parse IP address: %s" % ip_network_str)
return ip_network_list
def perform_action(session, tid, csr_fields):
db_cfg = load_tls_dict(session, tid)
pkv = tls.PrivKeyValidator()
ok, _ = pkv.validate(db_cfg)
if not ok:
raise errors.InputValidationError()
key_pair = db_cfg['ssl_key']
try:
csr_txt = tls.gen_x509_csr_pem(key_pair, csr_fields, Settings.csr_sign_bits)
log.debug("Generated a new CSR")
return csr_txt
except Exception as e:
log.err(e)
raise errors.InputValidationError('CSR gen failed')
def db_admin_update_user(session, tid, user_id, request, language):
"""
Updates the specified user.
"""
fill_localized_keys(request, models.User.localized_keys, language)
user = db_get_user(session, tid, user_id)
if user.username != request['username']:
check = session.query(models.User).filter(
models.User.username == str(request['username']),
models.User.tid == tid).one_or_none()
if check is not None:
raise errors.InputValidationError('Username already in use')
user.update(request)
password = request['password']
if password and not user.crypto_pub_key:
user.hash_alg = 'ARGON2'
user.salt = GCE.generate_salt()
user.password = GCE.hash_password(password, user.salt)
user.password_change_date = datetime_now()
# The various options related in manage PGP keys are used here.
parse_pgp_options(user, request)
if user.role == 'admin':
db_refresh_memory_variables(session, [tid])
return user
def post(self, name):
req = self.validate_message(self.request.content.read(),
requests.AdminTLSCfgFileResourceDesc)
file_res_cls = self.get_file_res_or_raise(name)
ok = yield file_res_cls.create_file(self.request.tid, req['content'])
if not ok:
raise errors.InputValidationError()
def db_create_user(session, state, tid, request, language):
request['tid'] = tid
fill_localized_keys(request, models.User.localized_keys, language)
if request['username']:
user = session.query(models.User).filter(
models.User.tid == tid, models.User.username == text_type(
request['username'])).one_or_none()
if user is not None:
raise errors.InputValidationError('Username already in use')
user = models.User({
'tid':
tid,
'username':
request['username'],
'role':
request['role'],
'state':
u'enabled',
'name':
request['name'],
'description':
request['description'],
'name':
request['name'],
'language':
language,
'password_change_needed':
request['password_change_needed'],
'mail_address':
request['mail_address']
})
if request['password']:
password = request['password']
elif user.role == 'receiver':
# code necessary because the user.role for recipient is receiver
password = '******'
else:
password = user.role
user.salt = security.generateRandomSalt()
user.password = security.hash_password(password, user.salt)
# The various options related in manage PGP keys are used here.
parse_pgp_options(state, user, request)
session.add(user)
session.flush()
if not request['username']:
user.username = user.id
return user
def db_create_user(session, state, tid, request, language):
request['tid'] = tid
fill_localized_keys(request, models.User.localized_keys, language)
if request['username']:
user = session.query(models.User).filter(
models.User.username == text_type(request['username']),
models.UserTenant.user_id == models.User.id,
models.UserTenant.tenant_id == tid).one_or_none()
if user is not None:
raise errors.InputValidationError('Username already in use')
user = models.User({
'tid':
tid,
'username':
request['username'],
'role':
request['role'],
'state':
u'enabled',
'name':
request['name'],
'description':
request['description'],
'language':
language,
'password_change_needed':
request['password_change_needed'],
'mail_address':
request['mail_address'],
'can_edit_general_settings':
request['can_edit_general_settings']
})
if not request['username']:
user.username = user.id = uuid4()
if request['password']:
password = request['password']
else:
password = u'password'
user.salt = security.generateRandomSalt()
user.password = security.hash_password(password, user.salt)
# The various options related in manage PGP keys are used here.
parse_pgp_options(state, user, request)
session.add(user)
session.flush()
db_create_usertenant_association(session, user.id, tid)
return user
def db_admin_update_user(session, tid, user_session, user_id, request,
language):
"""
Transaction for updating an existing user
:param session: An ORM session
:param tid: A tenant ID
:param user_session: The current user session
:param user_id: The ID of the user to update
:param request: The request data
:param language: The language of the request
:return: The serialized descriptor of the updated object
"""
fill_localized_keys(request, models.User.localized_keys, language)
user = db_get_user(session, tid, user_id)
if user.username != request['username']:
check = session.query(models.User).filter(
models.User.username == request['username'],
models.User.tid == tid).one_or_none()
if check is not None:
raise errors.InputValidationError('Username already in use')
user.update(request)
password = request['password']
if password and (not user.crypto_pub_key or user_session.ek):
if user.crypto_pub_key and user_session.ek:
enc_key = GCE.derive_key(password.encode(), user.salt)
crypto_escrow_prv_key = GCE.asymmetric_decrypt(
user_session.cc, Base64Encoder.decode(user_session.ek))
if tid == 1:
user_cc = GCE.asymmetric_decrypt(
crypto_escrow_prv_key,
Base64Encoder.decode(user.crypto_escrow_bkp1_key))
else:
user_cc = GCE.asymmetric_decrypt(
crypto_escrow_prv_key,
Base64Encoder.decode(user.crypto_escrow_bkp2_key))
user.crypto_prv_key = Base64Encoder.encode(
GCE.symmetric_encrypt(enc_key, user_cc))
if user.hash_alg != 'ARGON2':
user.hash_alg = 'ARGON2'
user.salt = GCE.generate_salt()
user.password = GCE.hash_password(password, user.salt)
user.password_change_date = datetime_now()
user.password_change_needed = True
# The various options related in manage PGP keys are used here.
parse_pgp_options(user, request)
return user_serialize_user(session, user, language)
def order_elements(session, handler, req_args, *args, **kwargs):
ctxs = session.query(models.Context).filter(models.Context.tid == handler.request.tid)
id_dict = { ctx.id: ctx for ctx in ctxs }
ids = req_args['ids']
if len(ids) != len(id_dict) or set(ids) != set(id_dict):
raise errors.InputValidationError('list does not contain all context ids')
for i, ctx_id in enumerate(ids):
id_dict[ctx_id].presentation_order = i
def put(self, *args, **kwargs):
request = self.validate_message(self.request.content.read(), requests.OpsDesc)
op_desc = self.operation_descriptors().get(request['operation'], None)
if op_desc is None:
raise errors.InputValidationError('Invalid command')
func, obj_validator = op_desc
if obj_validator is not None:
self.validate_jmessage(request['args'], obj_validator)
return func(self, request['args'], *args, **kwargs)
def delete_field(session, tid, field_id):
"""
Delete the field object corresponding to field_id
"""
field = models.db_get(session, models.Field, models.Field.tid == tid, models.Field.id == field_id)
if not field.editable:
raise errors.ForbiddenOperation
if field.instance == 'template' and session.query(models.Field).filter(models.Field.tid == tid, models.Field.template_id == field.id).count():
raise errors.InputValidationError("Cannot remove the field template as it is used by one or more questionnaires")
session.delete(field)
def check_hostname(session, tid, input_hostname):
"""
Ensure the hostname does not collide across tenants or
include an origin that it shouldn't.
"""
root_hostname = ConfigFactory(session, 1).get_val('hostname')
forbidden_endings = ['onion', 'localhost']
for v in forbidden_endings:
if input_hostname.endswith(v):
raise errors.InputValidationError(
'Hostname contains a forbidden origin')
existing_hostnames = {
h.value
for h in session.query(Config).filter(Config.tid != tid,
Config.var_name == 'hostname')
}
if input_hostname in existing_hostnames:
raise errors.InputValidationError('Hostname already reserved')
def try_to_enable_https(session, tid):
config = ConfigFactory(session, tid)
cv = tls.ChainValidator()
db_cfg = load_tls_dict(session, tid)
db_cfg['https_enabled'] = False
ok, _ = cv.validate(db_cfg)
if not ok:
raise errors.InputValidationError()
config.set_val(u'https_enabled', True)
State.tenant_cache[tid].https_enabled = True
def order_elements(session, handler, req_args, *args, **kwargs):
steps = session.query(models.Step) \
.filter(models.Step.questionnaire_id == req_args['questionnaire_id'],
models.Questionnaire.id == req_args['questionnaire_id'],
models.Questionnaire.tid == handler.request.tid)
id_dict = {step.id: step for step in steps}
ids = req_args['ids']
if len(ids) != len(id_dict) and set(ids) != set(id_dict):
raise errors.InputValidationError('list does not contain all context ids')
for i, step_id in enumerate(ids):
id_dict[step_id].presentation_order = i
def db_associate_context_receivers(session, tid, context, receiver_ids):
session.query(models.ReceiverContext).filter(models.ReceiverContext.context_id == context.id).delete(synchronize_session='fetch')
if not receiver_ids:
return
if not session.query(models.Context).filter(models.Context.id == context.id,
models.Context.tid == models.User.tid,
models.User.id.in_(receiver_ids)).count:
raise errors.InputValidationError()
for i, receiver_id in enumerate(receiver_ids):
session.add(models.ReceiverContext({'context_id': context.id,
'receiver_id': receiver_id,
'presentation_order': i}))
def order_status_elements(session, handler, req_args, *args, **kwargs):
"""Sets the presentation order for status elements"""
# Presentation order is ignored for statuses
statuses = session.query(models.SubmissionStatus)\
.filter(models.SubmissionStatus.tid == handler.request.tid,
models.SubmissionStatus.system_defined == False)
id_dict = {status.id: status for status in statuses}
ids = req_args['ids']
if len(ids) != len(id_dict) or set(ids) != set(id_dict):
raise errors.InputValidationError('list does not contain all context ids')
for i, status_id in enumerate(ids):
id_dict[status_id].presentation_order = i
def order_substatus_elements(session, handler, req_args, *args, **kwargs):
"""Sets presentation order for substatuses"""
submission_status_id = args[0]
substatuses = session.query(models.SubmissionSubStatus)\
.filter(models.SubmissionSubStatus.submissionstatus_id == submission_status_id)
id_dict = {substatus.id: substatus for substatus in substatuses}
ids = req_args['ids']
if len(ids) != len(id_dict) or set(ids) != set(id_dict):
raise errors.InputValidationError('list does not contain all context ids')
for i, substatus_id in enumerate(ids):
id_dict[substatus_id].presentation_order = i
def order_elements(session, tid, ids, *args, **kwargs):
"""
Transaction for reodering context elements
:param session: An ORM session
:param tid: The tenant ID
:param req_args: The request arguments
"""
ctxs = session.query(models.Context).filter(models.Context.tid == tid)
id_dict = {ctx.id: ctx for ctx in ctxs}
if len(ids) != len(id_dict) or set(ids) != set(id_dict):
raise errors.InputValidationError('list does not contain all context ids')
for i, ctx_id in enumerate(ids):
id_dict[ctx_id].order = i
def delete_field(session, tid, field_id):
"""
Transaction to delete a field
:param session: An ORM session
:param tid: The tenant ID
:param field_id: The id of the field to be deleted
"""
field = db_get(session,
models.Field,
(models.Field.tid == tid,
models.Field.id == field_id))
if field.instance == 'template' and session.query(models.Field).filter(models.Field.tid == tid, models.Field.template_id == field.id).count():
raise errors.InputValidationError("Cannot remove the field template as it is used by one or more questionnaires")
session.delete(field)
