def db_load_default_fields(session):
ffiles = [
os.path.join(Settings.questions_path, path)
for path in os.listdir(Settings.questions_path)
]
questions = []
qids = []
for ffile in ffiles:
questions.append(read_json_file(ffile))
qids.append(questions[-1]['id'])
session.query(models.Field).filter(
models.Field.id.in_(qids)).delete(synchronize_session='fetch')
session.query(models.Field).filter(
models.Field.fieldgroup_id.in_(qids)).delete(
synchronize_session='fetch')
session.query(models.FieldAttr).filter(
models.FieldAttr.field_id.in_(qids)).delete(
synchronize_session='fetch')
session.query(models.FieldOption).filter(
models.FieldOption.field_id.in_(qids)).delete(
synchronize_session='fetch')
for question in questions:
db_create_field(session, 1, question, None)
def db_fix_fields_attrs(session):
"""
Ensures that the current store and the field_attrs.json file correspond.
The content of the field_attrs dict is used to add and remove all of the
excepted forms of field_attrs for FieldAttrs in the db.
"""
field_attrs = read_json_file(Settings.field_attrs_file)
std_lst = [u'inputbox', u'textarea', u'checkbox', u'tos', u'date']
for field_type, attrs_dict in field_attrs.items():
attrs_to_keep_for_type = attrs_dict.keys()
if field_type in std_lst:
# Ensure that the standard field attrs do not have extra attr rows
_filter = not_(models.FieldAttr.name.in_(attrs_to_keep_for_type)), \
models.FieldAttr.field_id == models.Field.id, \
models.Field.type == field_type, \
models.Field.template_id == None
else:
# Look for dropped attrs in non-standard field_groups like whistleblower_identity
_filter = not_(models.FieldAttr.name.in_(attrs_to_keep_for_type)), \
models.FieldAttr.field_id == models.Field.id, \
models.Field.template_id == field_type
for res in session.query(models.FieldAttr).filter(*_filter):
session.delete(res)
# Add keys to the db that have been added to field_attrs
for field in session.query(models.Field):
type = field.type if field.template_id is None else field.template_id
attrs = field_attrs.get(type, {})
db_update_fieldattrs(session, field.id, attrs, None)
def db_fix_fields_attrs(session):
"""
Ensures that the current store and the field_attrs.json file correspond.
The content of the field_attrs dict is used to add and remove all of the
excepted forms of field_attrs for FieldAttrs in the db.
"""
field_attrs = read_json_file(Settings.field_attrs_file)
std_lst = [u'inputbox', u'textarea', u'multichoice', u'checkbox', u'tos', u'date']
for field_type, attrs_dict in field_attrs.items():
attrs_to_keep_for_type = attrs_dict.keys()
if field_type in std_lst:
# Ensure that the standard field attrs do not have extra attr rows
_filter = not_(models.FieldAttr.name.in_(attrs_to_keep_for_type)), \
models.FieldAttr.field_id == models.Field.id, \
models.Field.type == field_type, \
models.Field.template_id == None
else:
# Look for dropped attrs in non-standard field_groups like whistleblower_identity
_filter = not_(models.FieldAttr.name.in_(attrs_to_keep_for_type)), \
models.FieldAttr.field_id == models.Field.id, \
models.Field.template_id == field_type
for res in session.query(models.FieldAttr).filter(*_filter):
session.delete(res)
# Add keys to the db that have been added to field_attrs
for field in session.query(models.Field):
type = field.type if field.template_id is None else field.template_id
attrs = field_attrs.get(type, {})
db_add_field_attrs(session, field.id, attrs)
def db_create_field(session, tid, field_dict, language):
"""
Create and add a new field to the session, then return the new serialized object.
"""
field_dict['tid'] = tid
fill_localized_keys(field_dict, models.Field.localized_keys, language)
check_field_association(session, tid, field_dict)
if field_dict.get('template_id', '') != '':
if field_dict['template_id'] == 'whistleblower_identity':
if field_dict.get('step_id', '') == '':
raise errors.InputValidationError("Cannot associate whistleblower identity field to a fieldgroup")
q_id = session.query(models.Questionnaire.id) \
.filter(models.Questionnaire.id == models.Step.questionnaire_id,
models.Step.id == field_dict['step_id'])
field = session.query(models.Field) \
.filter(models.Field.template_id == u'whistleblower_identity',
models.Field.step_id == models.Step.id,
models.Step.questionnaire_id.in_(q_id.subquery())).one_or_none()
if field is not None:
raise errors.InputValidationError("Whistleblower identity field already present")
field = models.db_forge_obj(session, models.Field, field_dict)
template = session.query(models.Field).filter(models.Field.id == field_dict['template_id']).one()
field.label = template.label
field.hint = template.hint
field.description = template.description
attrs = field_dict.get('attrs')
if not attrs:
field_attrs = read_json_file(Settings.field_attrs_file)
attrs = field_attrs.get(field.template_id, {})
db_update_fieldattrs(session, field.id, attrs, None)
else:
field = models.db_forge_obj(session, models.Field, field_dict)
attrs = field_dict.get('attrs')
options = field_dict.get('options')
db_update_fieldattrs(session, field.id, attrs, language)
db_update_fieldoptions(session, field.id, options, language)
for trigger in field_dict.get('triggered_by_options', []):
db_create_trigger(session, tid, trigger['option'], 'field', field.id, trigger.get('sufficient', True))
if field.instance != 'reference':
for c in field_dict.get('children', []):
c['tid'] = field.tid
c['fieldgroup_id'] = field.id
db_create_field(session, tid, c, language)
return field
def load_default_fields(store):
ffiles = [
os.path.join(Settings.questions_path, path)
for path in os.listdir(Settings.questions_path)
]
for ffile in ffiles:
question = read_json_file(ffile)
store.find(models.Field, id=question['id']).remove()
db_create_field(store, question, None)
def load_profile(store, name):
path = os.path.join(Settings.client_path, 'data/profiles',
'{}.json'.format(name))
prof = read_json_file(path)
if not set(prof['node'].keys()) <= NodeFactory.admin_node:
raise ValueError('profile configuration key not in admin_node')
NodeFactory(store).update(prof['node'])
def db_create_field(session, tid, field_dict, language):
"""
Create and add a new field to the session, then return the new serialized object.
:param session: the session on which perform queries.
:param field_dict: the field definition dict
:param language: the language of the field definition dict
:return: a serialization of the object
"""
field_dict['tid'] = tid
fill_localized_keys(field_dict, models.Field.localized_keys, language)
check_field_association(session, tid, field_dict)
field = models.db_forge_obj(session, models.Field, field_dict)
if field.template_id is not None:
# special handling of the whistleblower_identity field
if field.template_id == 'whistleblower_identity':
field_attrs = read_json_file(Settings.field_attrs_file)
attrs = field_attrs.get(field.template_id, {})
db_add_field_attrs(session, field.id, attrs)
if field.step_id is not None:
questionnaire = session.query(models.Questionnaire) \
.filter(models.Field.id == field.id,
models.Field.step_id == models.Step.id,
models.Step.questionnaire_id == models.Questionnaire.id,
models.Questionnaire.tid == tid).one()
if questionnaire.enable_whistleblower_identity is False:
questionnaire.enable_whistleblower_identity = True
else:
raise errors.InputValidationError(
"Whistleblower identity field already present")
else:
raise errors.InputValidationError(
"Cannot associate whistleblower identity field to a fieldgroup"
)
else:
attrs = field_dict.get('attrs', [])
options = field_dict.get('options', [])
db_update_fieldattrs(session, tid, field.id, attrs, language)
db_update_fieldoptions(session, tid, field.id, options, language)
if field.instance != 'reference':
for c in field_dict.get('children', []):
c['tid'] = field.tid
c['fieldgroup_id'] = field.id
db_create_field(session, tid, c, language)
return field
def get_l10n(store, lang):
path = langfile_path(lang)
directory_traversal_check(Settings.client_path, path)
texts = read_json_file(path)
custom_texts = store.find(models.CustomTexts, models.CustomTexts.lang == lang).one()
custom_texts = custom_texts.texts if custom_texts is not None else {}
texts.update(custom_texts)
return texts
def load_default_fields(session):
ffiles = [
os.path.join(Settings.questions_path, path)
for path in os.listdir(Settings.questions_path)
]
for ffile in ffiles:
question = read_json_file(ffile)
question['tid'] = 1
session.query(
models.Field).filter(models.Field.id == question['id']).delete(
synchronize_session='fetch')
db_create_field(session, 1, question, None)
def db_fix_fields_attrs(store):
"""
Ensures that the current store and the field_attrs.json file correspond.
The content of the field_attrs dict is used to add and remove all of the
excepted forms of field_attrs for FieldAttrs in the db.
"""
field_attrs = read_json_file(Settings.field_attrs_file)
special_lst = ['whistleblower_identity']
std_lst = [
'inputbox', 'textarea', 'multichoice', 'checkbox', 'tos', 'date'
]
for field_type, attrs_dict in field_attrs.items():
attrs_to_keep_for_type = attrs_dict.keys()
if field_type in std_lst:
# Ensure that the standard field attrs do not have extra attr rows
res = store.find(
models.FieldAttr,
Not(In(models.FieldAttr.name, attrs_to_keep_for_type)),
models.FieldAttr.field_id == models.Field.id,
models.Field.type == field_type,
Not(In(models.Field.id, special_lst)))
else:
# Look for dropped attrs in non-standard field_groups like whistleblower_identity
res = store.find(
models.FieldAttr,
Not(In(models.FieldAttr.name, attrs_to_keep_for_type)),
models.FieldAttr.field_id == models.Field.id,
models.Field.id == field_type)
count = res.count()
if count:
log.debug("Removing %d attributes from fields of type %s", count,
field_type)
for r in res:
store.remove(r)
# Add keys to the db that have been added to field_attrs
for field in store.find(models.Field):
typ = field.type if field.id not in special_lst else field.id
attrs = field_attrs.get(typ, {})
for attr_name, attr_dict in attrs.items():
if not store.find(
models.FieldAttr,
And(models.FieldAttr.field_id == field.id,
models.FieldAttr.name == attr_name)).one():
log.debug("Adding new field attr %s.%s", typ, attr_name)
attr_dict['name'] = attr_name
attr_dict['field_id'] = field.id
models.db_forge_obj(store, models.FieldAttr, attr_dict)
def load_default_questionnaires(session):
qfiles = [os.path.join(Settings.questionnaires_path, path) for path in os.listdir(Settings.questionnaires_path)]
questionnaires = []
qids = []
for qfile in qfiles:
questionnaires.append(read_json_file(qfile))
qids.append(questionnaires[-1]['id'])
session.query(models.Questionnaire).filter(models.Questionnaire.id.in_(qids)).delete(synchronize_session='fetch')
session.query(models.Step).filter(models.Step.questionnaire_id.in_(qids)).delete(synchronize_session='fetch')
for questionnaire in questionnaires:
db_create_questionnaire(session, State, 1, questionnaire, None)
def test_post_invalid_json(self):
self.test_data_dir = os.path.join(helpers.DATA_DIR, 'questionnaires')
invalid_test_cases = [('cyclic_groupid.json',
errors.InputValidationError),
('duplicate_ids.json', IntegrityError)]
for fname, err in invalid_test_cases:
new_q = read_json_file(os.path.join(self.test_data_dir, fname))
handler = self.request(new_q, role='admin')
handler.request.language = None
yield self.assertFailure(handler.post(), err)
def db_load_default_questionnaires(session):
qfiles = [os.path.join(Settings.questionnaires_path, path) for path in os.listdir(Settings.questionnaires_path)]
questionnaires = []
qids = []
for qfile in qfiles:
questionnaires.append(read_json_file(qfile))
qids.append(questionnaires[-1]['id'])
session.query(models.Questionnaire).filter(models.Questionnaire.id.in_(qids)).delete(synchronize_session='fetch')
session.query(models.Step).filter(models.Step.questionnaire_id.in_(qids)).delete(synchronize_session='fetch')
for questionnaire in questionnaires:
db_create_questionnaire(session, 1, questionnaire, None)
def test_post_invalid_json(self):
self.test_data_dir = os.path.join(helpers.DATA_DIR, 'questionnaires')
invalid_test_cases = [
('cyclic_groupid.json', errors.InputValidationError),
('duplicate_ids.json', IntegrityError)
]
for fname, err in invalid_test_cases:
new_q = read_json_file(os.path.join(self.test_data_dir, fname))
handler = self.request(new_q, role='admin')
handler.request.language = None
yield self.assertFailure(handler.post(), err)
def load_default_questionnaires(store):
qfiles = [os.path.join(GLSettings.questionnaires_path, path) for path in os.listdir(GLSettings.questionnaires_path)]
for qfile in qfiles:
questionnaire = read_json_file(qfile)
steps = questionnaire.pop('steps')
q = store.find(models.Questionnaire, id=questionnaire['id']).one()
if q is None:
q = models.db_forge_obj(store, models.Questionnaire, questionnaire)
else:
store.find(models.Step, questionnaire_id=q.id).remove()
for step in steps:
step['questionnaire_id'] = q.id
db_create_step(store, step, None)
def db_load_default_fields(session):
ffiles = [os.path.join(Settings.questions_path, path) for path in os.listdir(Settings.questions_path)]
questions = []
qids = []
for ffile in ffiles:
questions.append(read_json_file(ffile))
qids.append(questions[-1]['id'])
session.query(models.Field).filter(models.Field.id.in_(qids)).delete(synchronize_session='fetch')
session.query(models.Field).filter(models.Field.fieldgroup_id.in_(qids)).delete(synchronize_session='fetch')
session.query(models.FieldAttr).filter(models.FieldAttr.field_id.in_(qids)).delete(synchronize_session='fetch')
session.query(models.FieldOption).filter(models.FieldOption.field_id.in_(qids)).delete(synchronize_session='fetch')
for question in questions:
db_create_field(session, 1, question, None)
def get_l10n(session, tid, lang):
path = langfile_path(lang)
directory_traversal_check(Settings.client_path, path)
if not os.path.exists(path):
raise errors.ResourceNotFound()
texts = read_json_file(path)
custom_texts = session.query(models.CustomTexts).filter(
models.CustomTexts.lang == lang,
models.CustomTexts.tid == tid).one_or_none()
custom_texts = custom_texts.texts if custom_texts is not None else {}
texts.update(custom_texts)
return texts
def prologue(self):
default_questionnaire = read_json_file(os.path.join(GLSettings.questionnaires_path, 'default.json'))
steps = default_questionnaire.pop('steps')
questionnaire = db_forge_obj(self.store_new, self.model_to['Questionnaire'], default_questionnaire)
questionnaire.key = u'default'
for step in steps:
f_children = step.pop('children')
s = db_forge_obj(self.store_new, self.model_to['Step'], step)
for child in f_children:
child['step_id'] = s.id
db_create_field(self.store_new, child, None)
s.questionnaire_id = questionnaire.id
self.store_new.commit()
def get_l10n(session, tid, lang):
if tid != 1:
config = ConfigFactory(session, 1)
if config.get_val(u'mode') == u'whistleblowing.it':
tid = 1
path = langfile_path(lang)
directory_traversal_check(Settings.client_path, path)
if not os.path.exists(path):
raise errors.ResourceNotFound()
texts = read_json_file(path)
custom_texts = session.query(models.CustomTexts).filter(models.CustomTexts.lang == lang, models.CustomTexts.tid == tid).one_or_none()
custom_texts = custom_texts.texts if custom_texts is not None else {}
texts.update(custom_texts)
return texts
def load_default_questionnaires(session):
qfiles = [
os.path.join(Settings.questionnaires_path, path)
for path in os.listdir(Settings.questionnaires_path)
]
for qfile in qfiles:
questionnaire = read_json_file(qfile)
steps = questionnaire.pop('steps')
q = session.query(models.Questionnaire).filter(
models.Questionnaire.id == questionnaire['id']).one_or_none()
if q is None:
q = models.db_forge_obj(session, models.Questionnaire,
questionnaire)
else:
session.query(models.Step).filter(
models.Step.questionnaire_id == q.id).delete(
synchronize_session='fetch')
for step in steps:
step['questionnaire_id'] = q.id
db_create_step(session, 1, step, None)
def load_appdata():
return read_json_file(Settings.appdata_file)
def db_create_field(session, tid, field_dict, language):
"""
Create and add a new field to the session, then return the new serialized object.
:param session: the session on which perform queries.
:param field_dict: the field definition dict
:param language: the language of the field definition dict
:return: a serialization of the object
"""
field_dict['tid'] = tid
fill_localized_keys(field_dict, models.Field.localized_keys, language)
check_field_association(session, tid, field_dict)
if field_dict.get('template_id', '') != '':
if field_dict['template_id'] == 'whistleblower_identity':
if field_dict.get('step_id', '') == '':
raise errors.InputValidationError(
"Cannot associate whistleblower identity field to a fieldgroup"
)
q_id = session.query(models.Questionnaire.id) \
.filter(models.Questionnaire.id == models.Step.questionnaire_id,
models.Step.id == field_dict['step_id'])
field = session.query(models.Field) \
.filter(models.Field.template_id == u'whistleblower_identity',
models.Field.step_id == models.Step.id,
models.Step.questionnaire_id.in_(q_id.subquery())).one_or_none()
if field is not None:
raise errors.InputValidationError(
"Whistleblower identity field already present")
field = models.db_forge_obj(session, models.Field, field_dict)
template = session.query(models.Field).filter(
models.Field.id == field_dict['template_id']).one()
field.label = template.label
field.hint = template.hint
field.description = template.description
attrs = field_dict.get('attrs')
if not attrs:
field_attrs = read_json_file(Settings.field_attrs_file)
attrs = field_attrs.get(field.template_id, {})
db_add_field_attrs(session, field.id, attrs)
else:
field = models.db_forge_obj(session, models.Field, field_dict)
attrs = field_dict.get('attrs')
options = field_dict.get('options')
db_update_fieldattrs(session, tid, field.id, attrs, language)
db_update_fieldoptions(session, tid, field.id, options, language)
if field.instance != 'reference':
for c in field_dict.get('children', []):
c['tid'] = field.tid
c['fieldgroup_id'] = field.id
db_create_field(session, tid, c, language)
return field
def load_profile(session, tid, name):
path = os.path.join(Settings.client_path, 'data/profiles',
'{}.json'.format(name))
prof = read_json_file(path)
ConfigFactory(session, tid).update('node', prof['node'])
def load_profile(session, tid, name):
path = os.path.join(Settings.client_path, 'data/profiles', '{}.json'.format(name))
prof = read_json_file(path)
ConfigFactory(session, tid).update('node', prof['node'])
def db_create_field(session, tid, field_dict, language):
"""
Create and add a new field to the session, then return the new serialized object.
"""
field_dict['tid'] = tid
fill_localized_keys(field_dict, models.Field.localized_keys, language)
check_field_association(session, tid, field_dict)
if field_dict.get('template_id', '') != '':
if field_dict['template_id'] == 'whistleblower_identity':
if field_dict.get('step_id', '') == '':
raise errors.InputValidationError(
"Cannot associate whistleblower identity field to a fieldgroup"
)
q_id = session.query(models.Questionnaire.id) \
.filter(models.Questionnaire.id == models.Step.questionnaire_id,
models.Step.id == field_dict['step_id'])
field = session.query(models.Field) \
.filter(models.Field.template_id == u'whistleblower_identity',
models.Field.step_id == models.Step.id,
models.Step.questionnaire_id.in_(q_id.subquery())).one_or_none()
if field is not None:
raise errors.InputValidationError(
"Whistleblower identity field already present")
field = models.db_forge_obj(session, models.Field, field_dict)
template = session.query(models.Field).filter(
models.Field.id == field_dict['template_id']).one()
field.label = template.label
field.description = template.description
field.hint = template.hint
field.placeholder = template.placeholder
attrs = field_dict.get('attrs')
if not attrs:
field_attrs = read_json_file(Settings.field_attrs_file)
attrs = field_attrs.get(field.template_id, {})
db_update_fieldattrs(session, field.id, attrs, None)
else:
field = models.db_forge_obj(session, models.Field, field_dict)
attrs = field_dict.get('attrs')
options = field_dict.get('options')
db_update_fieldattrs(session, field.id, attrs, language)
db_update_fieldoptions(session, field.id, options, language)
for trigger in field_dict.get('triggered_by_options', []):
db_create_trigger(session, tid, trigger['option'], 'field',
field.id, trigger.get('sufficient', True))
if field.instance != 'reference':
for c in field_dict.get('children', []):
c['tid'] = field.tid
c['fieldgroup_id'] = field.id
db_create_field(session, tid, c, language)
return field
def load_appdata():
return read_json_file(Settings.appdata_file)