def db_import_fields(store, step, fieldgroup, fields):
for field in fields:
f_attrs = copy.deepcopy(field['attrs'])
f_options = copy.deepcopy(field['options'])
f_children = copy.deepcopy(field['children'])
del field['attrs'], field['options'], field['children']
f = models.db_forge_obj(store, models.Field, field)
for key, value in f_attrs.iteritems():
value['name'] = key
a = models.db_forge_obj(store, models.FieldAttr, value)
f.attrs.add(a)
for f_option in f_options:
o = models.db_forge_obj(store, models.FieldOption, f_option)
f.options.add(o)
if (step):
f.step_id = step.id
step.children.add(f)
else:
f.fieldgroup_id = fieldgroupid
fieldgroup.children.add(f)
if f_children:
db_import_fields(store, None, f, f_children)
def db_create_field(session, tid, field_dict, language):
"""
Create and add a new field to the session, then return the new serialized object.
"""
field_dict['tid'] = tid
fill_localized_keys(field_dict, models.Field.localized_keys, language)
check_field_association(session, tid, field_dict)
if field_dict.get('template_id', '') != '':
if field_dict['template_id'] == 'whistleblower_identity':
if field_dict.get('step_id', '') == '':
raise errors.InputValidationError("Cannot associate whistleblower identity field to a fieldgroup")
q_id = session.query(models.Questionnaire.id) \
.filter(models.Questionnaire.id == models.Step.questionnaire_id,
models.Step.id == field_dict['step_id'])
field = session.query(models.Field) \
.filter(models.Field.template_id == u'whistleblower_identity',
models.Field.step_id == models.Step.id,
models.Step.questionnaire_id.in_(q_id.subquery())).one_or_none()
if field is not None:
raise errors.InputValidationError("Whistleblower identity field already present")
field = models.db_forge_obj(session, models.Field, field_dict)
template = session.query(models.Field).filter(models.Field.id == field_dict['template_id']).one()
field.label = template.label
field.hint = template.hint
field.description = template.description
attrs = field_dict.get('attrs')
if not attrs:
field_attrs = read_json_file(Settings.field_attrs_file)
attrs = field_attrs.get(field.template_id, {})
db_update_fieldattrs(session, field.id, attrs, None)
else:
field = models.db_forge_obj(session, models.Field, field_dict)
attrs = field_dict.get('attrs')
options = field_dict.get('options')
db_update_fieldattrs(session, field.id, attrs, language)
db_update_fieldoptions(session, field.id, options, language)
for trigger in field_dict.get('triggered_by_options', []):
db_create_trigger(session, tid, trigger['option'], 'field', field.id, trigger.get('sufficient', True))
if field.instance != 'reference':
for c in field_dict.get('children', []):
c['tid'] = field.tid
c['fieldgroup_id'] = field.id
db_create_field(session, tid, c, language)
return field
def db_setup_default_steps(store, context):
appdata = store.find(models.ApplicationData).one()
steps = copy.deepcopy(appdata.fields)
n_s = 1
for step in steps:
for f_child in step['children']:
if not field_is_present(store, f_child):
f_child['is_template'] = False
for step in steps:
f_children = copy.deepcopy(step['children'])
del step['children']
s = models.db_forge_obj(store, models.Step, step)
for f_child in f_children:
o_children = copy.deepcopy(f_child['options'])
del f_child['options']
# FIXME currently default updata do not handle fieldgroups
# all this block must be redesigned in order to be called recursively
del f_child['children']
f = models.db_forge_obj(store, models.Field, f_child)
n_o = 1
for o_child in o_children:
o = models.db_forge_obj(store, models.FieldOption, o_child)
o.field_id = f.id
o.number = n_o
f.options.add(o)
n_o += 1
f.step_id = s.id
s.children.add(f)
s.context_id = context.id
s.number = n_s
context.steps.add(s)
n_s += 1
def db_import_fields(store, step, fieldgroup, fields):
for field in fields:
f_attrs = copy.deepcopy(field['attrs'])
f_options = copy.deepcopy(field['options'])
f_children = copy.deepcopy(field['children'])
del field['attrs'], field['options'], field['children']
f = models.db_forge_obj(store, models.Field, field)
for attr in f_attrs:
f_attrs[attr]['name'] = attr
f.attrs.add(
models.db_forge_obj(store, models.FieldAttr, f_attrs[attr]))
for option in f_options:
f.options.add(
models.db_forge_obj(store, models.FieldOption, option))
if step:
step.children.add(f)
else:
fieldgroup.children.add(f)
if f_children:
db_import_fields(store, None, f, f_children)
def db_import_fields(store, step, fieldgroup, fields):
for field in fields:
f_attrs = copy.deepcopy(field['attrs'])
f_options = copy.deepcopy(field['options'])
f_children = copy.deepcopy(field['children'])
del field['attrs'], field['options'], field['children']
f = models.db_forge_obj(store, models.Field, field)
for key, value in f_attrs.iteritems():
value['name'] = key
a = models.db_forge_obj(store, models.FieldAttr, value)
f.attrs.add(a)
for f_option in f_options:
o = models.db_forge_obj(store, models.FieldOption, f_option)
f.options.add(o)
if (step):
f.step_id = step.id
step.children.add(f)
else:
f.fieldgroup_id = fieldgroupid
fieldgroup.children.add(f)
if f_children:
db_import_fields(store, None, f, f_children)
def db_add_field_attrs(session, field_id, field_attrs):
for attr_name, attr_dict in field_attrs.items():
x = session.query(models.FieldAttr) \
.filter(models.FieldAttr.field_id == field_id,
models.FieldAttr.name == attr_name).one_or_none()
if x is None:
attr_dict['name'] = attr_name
attr_dict['field_id'] = field_id
models.db_forge_obj(session, models.FieldAttr, attr_dict)
def import_fixture(store, fixture):
"""
Import a valid json object holding all informations, and stores them in the database.
:return: The traditional deferred used for transaction in GlobaLeaks.
"""
with open(os.path.join(FIXTURES_PATH, fixture)) as f:
data = json.loads(f.read())
for mock in data:
mock_class = getattr(models, mock['class'])
db_forge_obj(store, mock_class, mock['fields'])
def import_fixture(store, fixture):
"""
Import a valid json object holding all informations, and stores them in the database.
:return: The traditional deferred used for transaction in GlobaLeaks.
"""
with open(os.path.join(FIXTURES_PATH, fixture)) as f:
data = json.loads(f.read())
for mock in data:
mock_class = getattr(models, mock['class'])
db_forge_obj(store, mock_class, mock['fields'])
def db_fix_fields_attrs(store):
"""
Ensures that the current store and the field_attrs.json file correspond.
The content of the field_attrs dict is used to add and remove all of the
excepted forms of field_attrs for FieldAttrs in the db.
"""
field_attrs = read_json_file(Settings.field_attrs_file)
special_lst = ['whistleblower_identity']
std_lst = [
'inputbox', 'textarea', 'multichoice', 'checkbox', 'tos', 'date'
]
for field_type, attrs_dict in field_attrs.items():
attrs_to_keep_for_type = attrs_dict.keys()
if field_type in std_lst:
# Ensure that the standard field attrs do not have extra attr rows
res = store.find(
models.FieldAttr,
Not(In(models.FieldAttr.name, attrs_to_keep_for_type)),
models.FieldAttr.field_id == models.Field.id,
models.Field.type == field_type,
Not(In(models.Field.id, special_lst)))
else:
# Look for dropped attrs in non-standard field_groups like whistleblower_identity
res = store.find(
models.FieldAttr,
Not(In(models.FieldAttr.name, attrs_to_keep_for_type)),
models.FieldAttr.field_id == models.Field.id,
models.Field.id == field_type)
count = res.count()
if count:
log.debug("Removing %d attributes from fields of type %s", count,
field_type)
for r in res:
store.remove(r)
# Add keys to the db that have been added to field_attrs
for field in store.find(models.Field):
typ = field.type if field.id not in special_lst else field.id
attrs = field_attrs.get(typ, {})
for attr_name, attr_dict in attrs.items():
if not store.find(
models.FieldAttr,
And(models.FieldAttr.field_id == field.id,
models.FieldAttr.name == attr_name)).one():
log.debug("Adding new field attr %s.%s", typ, attr_name)
attr_dict['name'] = attr_name
attr_dict['field_id'] = field.id
models.db_forge_obj(store, models.FieldAttr, attr_dict)
def import_fixture(store, fixture):
"""
Import a valid json object holding all informations, and stores them in the database.
:return: The traditional deferred used for transaction in GlobaLeaks.
"""
data = load_json_file(os.path.join(FIXTURES_PATH, fixture))
for mock in data:
for k in mock['fields']:
if k.endswidh('_id') and mock['fields'][k] == '':
del mock['fields'][k]
mock_class = getattr(models, mock['class'])
models.db_forge_obj(store, mock_class, mock['fields'])
store.commit()
def import_fixture(store, fixture):
"""
Import a valid json object holding all informations, and stores them in the database.
:return: The traditional deferred used for transaction in GlobaLeaks.
"""
data = load_json_file(os.path.join(FIXTURES_PATH, fixture))
for mock in data:
for k in mock['fields']:
if k.endswidh('_id') and mock['fields'][k] == '':
del mock['fields'][k]
mock_class = getattr(models, mock['class'])
models.db_forge_obj(store, mock_class, mock['fields'])
store.commit()
def db_preallocate(session, desc):
t = models.db_forge_obj(session, models.Tenant, desc)
# required to generate the tenant id
session.flush()
return t
def db_update_fieldattr(session, field_id, attr_name, attr_dict, language):
"""
Transaction for updating a fieldattr
:param session: An ORM session
:param field_id: The field on which the attribute is configured
:param attr_name: The attribute name
:param attr_dict: The attribute configuration
:param language: The language of the request
:return: The ID of the attribute
"""
attr_dict['name'] = attr_name
attr_dict['field_id'] = field_id
if attr_dict['type'] == 'localized' and language is not None:
fill_localized_keys(attr_dict, ['value'], language)
o = session.query(models.FieldAttr).filter(
models.FieldAttr.field_id == field_id,
models.FieldAttr.name == attr_name).one_or_none()
if o is None:
attr_dict['id'] = ''
o = models.db_forge_obj(session, models.FieldAttr, attr_dict)
else:
o.update(attr_dict)
return o.id
def db_update_fieldoption(session, field_id, fieldoption_id, option_dict,
language, idx):
"""
Transaction to update a field option
:param session: An ORM session
:param field_id: The field ID of the field on which the option is set
:param fieldoption_id: The option ID
:param option_dict: The option configuration
:param language: The language of the request
:param idx: The order index with reference to the other options set
:return: The serialized descriptor of the option
"""
option_dict['field_id'] = field_id
fill_localized_keys(option_dict, models.FieldOption.localized_keys,
language)
o = None
if fieldoption_id is not None:
o = session.query(models.FieldOption).filter(
models.FieldOption.id == fieldoption_id).one_or_none()
if o is None:
o = models.db_forge_obj(session, models.FieldOption, option_dict)
else:
o.update(option_dict)
o.order = idx
return o.id
def db_fix_fields_attrs(store):
# Load the field attributes descriptors
field_attrs = {}
with file(GLSettings.field_attrs_file, 'r') as f:
json_string = f.read()
field_attrs = json.loads(json_string)
# Get the list of keys
uptodate_keys = []
for k1 in field_attrs.keys():
for k2 in field_attrs[k1].keys():
field_attrs[k1][k2]['name'] = k2
uptodate_keys.append(k2)
# Remove keys that have been removed
store.find(models.FieldAttr, Not(In(models.FieldAttr.name,
uptodate_keys))).remove()
# Add keys that have been added
for field in store.find(models.Field):
attrs = field_attrs.get(field.type, {})
for attr_name in attrs.keys():
if not store.find(
models.FieldAttr,
And(models.FieldAttr.field_id == field.id,
models.FieldAttr.name == attr_name)).one():
field.attrs.add(
models.db_forge_obj(store, models.FieldAttr,
attrs[attr_name]))
def db_preallocate(session, desc):
t = models.db_forge_obj(session, models.Tenant, desc)
# required to generate the tenant id
session.flush()
return t
def db_schedule_email(session, tid, address, subject, body):
return models.db_forge_obj(session, models.Mail, {
'address': address,
'subject': subject,
'body': body,
'tid': tid,
})
def db_create(session, desc):
appdata = load_appdata()
t = models.db_forge_obj(session, models.Tenant, desc)
# required to generate/retrive the id
session.flush()
db_update_defaults(session)
models.config.system_cfg_init(session, tid=t.id)
models.config.add_new_lang(session, t.id, u'en', appdata)
file_descs = [
(u'logo', 'data/logo.png'),
(u'favicon', 'data/favicon.ico')
]
for file_desc in file_descs:
with open(os.path.join(Settings.client_path, file_desc[1]), 'r') as f:
data = base64.b64encode(f.read())
file.db_add_file(session, t.id, file_desc[0], u'', data)
db_refresh_memory_variables(session, [t.id])
return t
def prologue(self):
appdata = load_appdata()
steps = appdata['default_questionnaire']['steps']
del appdata['default_questionnaire']['steps']
questionnaire = db_forge_obj(self.store_new, Questionnaire, appdata['default_questionnaire'])
for step in steps:
f_children = step['children']
del step['children']
s = db_forge_obj(self.store_new, Step, step)
db_import_fields(self.store_new, s, None, f_children)
s.questionnaire_id = questionnaire.id
self.store_new.commit()
def db_fix_fields_attrs(store):
# Load the field attributes descriptors
field_attrs = {}
with file(GLSettings.field_attrs_file, 'r') as f:
json_string = f.read()
field_attrs = json.loads(json_string)
# Get the list of keys
uptodate_keys = []
for k1 in field_attrs.keys():
for k2 in field_attrs[k1].keys():
field_attrs[k1][k2]['name'] = k2
uptodate_keys.append(k2)
# Remove keys that have been removed
store.find(models.FieldAttr,
Not(In(models.FieldAttr.name, uptodate_keys))).remove()
# Add keys that have been added
for field in store.find(models.Field):
attrs = field_attrs.get(field.type, {})
for attr_name in attrs.keys():
if not store.find(models.FieldAttr,
And(models.FieldAttr.field_id == field.id,
models.FieldAttr.name == attr_name)).one():
field.attrs.add(models.db_forge_obj(store, models.FieldAttr,
attrs[attr_name]))
def prologue(self):
default_questionnaire = read_json_file(os.path.join(GLSettings.questionnaires_path, 'default.json'))
steps = default_questionnaire.pop('steps')
questionnaire = db_forge_obj(self.store_new, self.model_to['Questionnaire'], default_questionnaire)
questionnaire.key = u'default'
for step in steps:
f_children = step.pop('children')
s = db_forge_obj(self.store_new, self.model_to['Step'], step)
for child in f_children:
child['step_id'] = s.id
db_create_field(self.store_new, child, None)
s.questionnaire_id = questionnaire.id
self.store_new.commit()
def db_schedule_email(session, tid, address, subject, body):
return models.db_forge_obj(session, models.Mail,
{
'address': address,
'subject': subject,
'body': body,
'tid': tid,
})
def prologue(self):
appdata = load_appdata()
steps = appdata['default_questionnaire']['steps']
del appdata['default_questionnaire']['steps']
questionnaire = db_forge_obj(self.store_new, Questionnaire,
appdata['default_questionnaire'])
for step in steps:
f_children = step['children']
del step['children']
s = db_forge_obj(self.store_new, Step, step)
db_import_fields(self.store_new, s, None, f_children)
s.questionnaire_id = questionnaire.id
self.store_new.commit()
def db_setup_default_questionnaire(store, context):
appdata = store.find(models.ApplicationData).one()
for step in copy.deepcopy(appdata.default_questionnaire):
f_children = copy.deepcopy(step['children'])
del step['children']
s = models.db_forge_obj(store, models.Step, step)
db_import_fields(store, s, None, f_children)
s.context_id = context.id
def load_default_questionnaires(store):
appdata = store.find(models.ApplicationData).one()
steps = appdata.default_questionnaire['steps']
del appdata.default_questionnaire['steps']
questionnaire = store.find(models.Questionnaire, models.Questionnaire.key == u'default').one()
if questionnaire is None:
questionnaire = models.db_forge_obj(store, models.Questionnaire, appdata.default_questionnaire)
else:
for step in questionnaire.steps:
store.remove(step)
for step in steps:
f_children = step['children']
del step['children']
s = models.db_forge_obj(store, models.Step, step)
db_import_fields(store, s, None, f_children)
s.questionnaire_id = questionnaire.id
def load_default_questionnaires(store):
appdata = store.find(models.ApplicationData).one()
steps = appdata.default_questionnaire['steps']
del appdata.default_questionnaire['steps']
questionnaire = store.find(models.Questionnaire, models.Questionnaire.key == u'default').one()
if questionnaire is None:
questionnaire = models.db_forge_obj(store, models.Questionnaire, appdata.default_questionnaire)
else:
for step in questionnaire.steps:
store.remove(step)
for step in steps:
f_children = step['children']
del step['children']
s = models.db_forge_obj(store, models.Step, step)
db_import_fields(store, s, None, f_children)
s.questionnaire_id = questionnaire.id
def db_create_context(session, state, tid, request, language):
request = fill_context_request(tid, request, language)
check_context_questionnaire_association(session, tid, request['questionnaire_id'])
context = models.db_forge_obj(session, models.Context, request)
db_associate_context_receivers(session, tid, context, request['receivers'])
return context
def db_create_questionnaire(store, questionnaire_dict, language):
fill_localized_keys(questionnaire_dict,
models.Questionnaire.localized_keys, language)
q = models.db_forge_obj(store, models.Questionnaire, questionnaire_dict)
for step in questionnaire_dict.get('steps', []):
db_create_step(store, step, language)
return q
def db_create_context(session, tid, request, language):
request = fill_context_request(tid, request, language)
check_context_questionnaire_association(session, tid, request['questionnaire_id'])
context = models.db_forge_obj(session, models.Context, request)
db_associate_context_receivers(session, tid, context, request['receivers'])
return context
def db_create_field(session, tid, field_dict, language):
"""
Create and add a new field to the session, then return the new serialized object.
:param session: the session on which perform queries.
:param field_dict: the field definition dict
:param language: the language of the field definition dict
:return: a serialization of the object
"""
field_dict['tid'] = tid
fill_localized_keys(field_dict, models.Field.localized_keys, language)
check_field_association(session, tid, field_dict)
field = models.db_forge_obj(session, models.Field, field_dict)
if field.template_id is not None:
# special handling of the whistleblower_identity field
if field.template_id == 'whistleblower_identity':
field_attrs = read_json_file(Settings.field_attrs_file)
attrs = field_attrs.get(field.template_id, {})
db_add_field_attrs(session, field.id, attrs)
if field.step_id is not None:
questionnaire = session.query(models.Questionnaire) \
.filter(models.Field.id == field.id,
models.Field.step_id == models.Step.id,
models.Step.questionnaire_id == models.Questionnaire.id,
models.Questionnaire.tid == tid).one()
if questionnaire.enable_whistleblower_identity is False:
questionnaire.enable_whistleblower_identity = True
else:
raise errors.InputValidationError(
"Whistleblower identity field already present")
else:
raise errors.InputValidationError(
"Cannot associate whistleblower identity field to a fieldgroup"
)
else:
attrs = field_dict.get('attrs', [])
options = field_dict.get('options', [])
db_update_fieldattrs(session, tid, field.id, attrs, language)
db_update_fieldoptions(session, tid, field.id, options, language)
if field.instance != 'reference':
for c in field_dict.get('children', []):
c['tid'] = field.tid
c['fieldgroup_id'] = field.id
db_create_field(session, tid, c, language)
return field
def db_fix_fields_attrs(store):
'''
Ensures that the current store and the field_attrs.json file correspond.
The content of the field_attrs dict is used to add and remove all of the
excepted forms of field_attrs for FieldAttrs in the db.
'''
# Load the field attributes descriptors
field_attrs = {}
with file(GLSettings.field_attrs_file, 'r') as f:
json_string = f.read()
field_attrs = json.loads(json_string)
std_lst = [
'inputbox', 'textarea', 'multichoice', 'checkbox', 'tos', 'date'
]
for field_type, attrs_dict in field_attrs.iteritems():
attrs_to_keep_for_type = attrs_dict.keys()
if field_type in std_lst:
# Ensure that the standard field attrs do not have extra attr rows
res = store.find(
models.FieldAttr,
Not(In(models.FieldAttr.name, attrs_to_keep_for_type)),
models.FieldAttr.field_id == models.Field.id,
models.Field.type == field_type,
models.Field.key == unicode(''))
else:
# Look for dropped attrs in non-standard field_groups like whistleblower_identity
res = store.find(
models.FieldAttr,
Not(In(models.FieldAttr.name, attrs_to_keep_for_type)),
models.FieldAttr.field_id == models.Field.id,
models.Field.key == field_type)
count = res.count()
if count:
log.debug("Removing %d attributes from fields of type %s" %
(count, field_type))
for r in res:
store.remove(r)
# Add keys to the db that have been added to field_attrs
for field in store.find(models.Field):
typ = field.type if field.key == '' else field.key
attrs = field_attrs.get(typ, {})
for attr_name, attr_dict in attrs.iteritems():
if not store.find(
models.FieldAttr,
And(models.FieldAttr.field_id == field.id,
models.FieldAttr.name == attr_name)).one():
log.debug("Adding new field attr %s.%s" % (typ, attr_name))
attr_dict['name'] = attr_name
field.attrs.add(
models.db_forge_obj(store, models.FieldAttr, attr_dict))
def db_import_steps(store, context_id, steps):
for step in steps:
for f_child in step['children']:
if not field_is_present(store, f_child):
f_child['is_template'] = False
for step in steps:
f_children = copy.deepcopy(step['children'])
del step['children']
s = models.db_forge_obj(store, models.Step, step)
db_import_fields(store, s, None, f_children)
s.context_id = context_id
def db_create_context(store, request, language):
request = fill_context_request(request, language)
if not request['allow_recipients_selection']:
request['select_all_receivers'] = True
context = models.db_forge_obj(store, models.Context, request)
db_associate_context_receivers(store, context, request['receivers'])
return context
def db_create_field(store, field_dict, language):
"""
Create and add a new field to the store, then return the new serialized object.
:param store: the store on which perform queries.
:param field_dict: the field definition dict
:param language: the language of the field definition dict
:return: a serialization of the object
"""
fill_localized_keys(field_dict, models.Field.localized_keys, language)
if field_dict.get('fieldgroup_id', ''):
ancestors = set(fieldtree_ancestors(store,
field_dict['fieldgroup_id']))
if field_dict['id'] == field_dict['fieldgroup_id'] or field_dict[
'id'] in ancestors:
raise errors.InvalidInputFormat(
"Provided field association would cause recursion loop")
field = models.db_forge_obj(store, models.Field, field_dict)
if field.template_id is not None:
# special handling of the whistleblower_identity field
if field.template_id == 'whistleblower_identity':
if field.step_id is not None:
questionnaire = store.find(
models.Questionnaire, models.Field.id == field.id,
models.Field.step_id == models.Step.id,
models.Step.questionnaire_id ==
models.Questionnaire.id).one()
if questionnaire.enable_whistleblower_identity is False:
questionnaire.enable_whistleblower_identity = True
else:
raise errors.InvalidInputFormat(
"Whistleblower identity field already present")
else:
raise errors.InvalidInputFormat(
"Cannot associate whistleblower identity field to a fieldgroup"
)
else:
db_update_fieldattrs(store, field, field_dict.get('attrs', []),
language)
db_update_fieldoptions(store, field, field_dict.get('options', []),
language)
if field.instance != 'reference':
for c in field_dict.get('children', []):
c['fieldgroup_id'] = field.id
db_create_field(store, c, language)
return field
def db_create_questionnaire(session, tid, questionnaire_dict, language):
fill_localized_keys(questionnaire_dict, models.Questionnaire.localized_keys, language)
questionnaire_dict['tid'] = tid
q = models.db_forge_obj(session, models.Questionnaire, questionnaire_dict)
for step in questionnaire_dict.get('steps', []):
step['questionnaire_id'] = q.id
db_create_step(session, tid, step, language)
return q
def db_create_questionnaire(session, state, tid, questionnaire_dict, language):
fill_localized_keys(questionnaire_dict, models.Questionnaire.localized_keys, language)
questionnaire_dict['tid'] = tid
q = models.db_forge_obj(session, models.Questionnaire, questionnaire_dict)
for step in questionnaire_dict.get('steps', []):
step['questionnaire_id'] = q.id
db_create_step(session, tid, step, language)
return q
def db_import_steps(store, context_id, steps):
for step in steps:
for f_child in step['children']:
if not field_is_present(store, f_child):
f_child['is_template'] = False
for step in steps:
f_children = copy.deepcopy(step['children'])
del step['children']
s = models.db_forge_obj(store, models.Step, step)
db_import_fields(store, s, None, f_children)
s.context_id = context_id
def create(session, tid, request):
"""
Transaction for registering the creation of a redirect for a tenant
:param session: An ORM session
:param tid: A tenant ID
:param request: The request data
:return: The descriptor of the registered redirect
"""
request['tid'] = tid
redirect = models.db_forge_obj(session, models.Redirect, request)
return serialize_redirect(redirect)
def import_fixture(store, fixture):
"""
Import a valid json object holding all informations, and stores them in the database.
:return: The traditional deferred used for transaction in GlobaLeaks.
"""
with open(os.path.join(FIXTURES_PATH, fixture)) as f:
data = json.loads(f.read())
for mock in data:
if mock['class'] == 'Field':
if mock['fields']['instance'] != 'reference':
del mock['fields']['template_id']
if mock['fields']['step_id'] == '':
del mock['fields']['step_id']
if mock['fields']['fieldgroup_id'] == '':
del mock['fields']['fieldgroup_id']
mock_class = getattr(models, mock['class'])
models.db_forge_obj(store, mock_class, mock['fields'])
store.commit()
def import_fixture(store, fixture):
"""
Import a valid json object holding all informations, and stores them in the database.
:return: The traditional deferred used for transaction in GlobaLeaks.
"""
with open(os.path.join(FIXTURES_PATH, fixture)) as f:
data = json.loads(f.read())
for mock in data:
if mock['class'] == 'Field':
if mock['fields']['instance'] != 'reference':
del mock['fields']['template_id']
if mock['fields']['step_id'] == '':
del mock['fields']['step_id']
if mock['fields']['fieldgroup_id'] == '':
del mock['fields']['fieldgroup_id']
mock_class = getattr(models, mock['class'])
models.db_forge_obj(store, mock_class, mock['fields'])
store.commit()
def db_update_fieldattr(session, field_id, attr_name, attr_dict, language):
attr_dict['name'] = attr_name
attr_dict['field_id'] = field_id
if attr_dict['type'] == u'localized' and language is not None:
fill_localized_keys(attr_dict, ['value'], language)
o = session.query(models.FieldAttr).filter(models.FieldAttr.field_id == field_id, models.FieldAttr.name == attr_name).one_or_none()
if o is None:
o = models.db_forge_obj(session, models.FieldAttr, attr_dict)
else:
o.update(attr_dict)
return o.id
def db_create_receiver_user(session, state, tid, request, language):
"""
Creates a new receiver
Returns:
(dict) the receiver descriptor
"""
fill_localized_keys(request, models.Receiver.localized_keys, language)
user = db_create_user(session, state, tid, request, language)
request['id'] = user.id
receiver = models.db_forge_obj(session, models.Receiver, request)
return receiver, user
def db_import_fields(store, step, fieldgroup, fields):
for field in fields:
f_attrs = copy.deepcopy(field['attrs'])
f_options = copy.deepcopy(field['options'])
f_children = copy.deepcopy(field['children'])
del field['attrs'], field['options'], field['children']
f = models.db_forge_obj(store, models.Field, field)
for attr in f_attrs:
f_attrs[attr]['name'] = attr
f.attrs.add(models.db_forge_obj(store, models.FieldAttr, f_attrs[attr]))
for option in f_options:
f.options.add(models.db_forge_obj(store, models.FieldOption, option))
if (step):
step.children.add(f)
else:
fieldgroup.children.add(f)
if f_children:
db_import_fields(store, None, f, f_children)
def db_update_fieldoption(store, field, fieldoption_id, option_dict, language):
option_dict['field_id'] = field.id
fill_localized_keys(option_dict, models.FieldOption.localized_keys,
language)
o = None
if fieldoption_id is not None:
o = store.find(models.FieldOption, id=fieldoption_id).one()
if o is None:
o = models.db_forge_obj(store, models.FieldOption, option_dict)
else:
o.update(option_dict)
return o.id
def db_create_step(store, step_dict, language):
"""
Create the specified step
:param store: the store on which perform queries.
:param language: the language of the specified steps.
"""
fill_localized_keys(step_dict, models.Step.localized_keys, language)
step = models.db_forge_obj(store, models.Step, step_dict)
for c in step_dict['children']:
c['step_id'] = step.id
db_create_field(store, c, language)
return step
def db_fix_fields_attrs(store):
'''
Ensures that the current store and the field_attrs.json file correspond.
The content of the field_attrs dict is used to add and remove all of the
excepted forms of field_attrs for FieldAttrs in the db.
'''
# Load the field attributes descriptors
field_attrs = {}
with file(GLSettings.field_attrs_file, 'r') as f:
json_string = f.read()
field_attrs = json.loads(json_string)
std_lst = ['inputbox', 'textarea', 'multichoice', 'checkbox', 'tos', 'date']
for field_type, attrs_dict in field_attrs.iteritems():
attrs_to_keep_for_type = attrs_dict.keys()
if field_type in std_lst:
# Ensure that the standard field attrs do not have extra attr rows
res = store.find(models.FieldAttr, Not(In(models.FieldAttr.name, attrs_to_keep_for_type)),
models.FieldAttr.field_id == models.Field.id,
models.Field.type == field_type,
models.Field.key == unicode(''))
else:
# Look for dropped attrs in non-standard field_groups like whistleblower_identity
res = store.find(models.FieldAttr, Not(In(models.FieldAttr.name, attrs_to_keep_for_type)),
models.FieldAttr.field_id == models.Field.id,
models.Field.key == field_type)
count = res.count()
if count:
log.debug("Removing %d attributes from fields of type %s" % (count, field_type))
for r in res:
store.remove(r)
# Add keys to the db that have been added to field_attrs
for field in store.find(models.Field):
typ = field.type if field.key == '' else field.key
attrs = field_attrs.get(typ, {})
for attr_name, attr_dict in attrs.iteritems():
if not store.find(models.FieldAttr,
And(models.FieldAttr.field_id == field.id,
models.FieldAttr.name == attr_name)).one():
log.debug("Adding new field attr %s.%s" % (typ, attr_name))
attr_dict['name'] = attr_name
field.attrs.add(models.db_forge_obj(store, models.FieldAttr, attr_dict))
def db_update_fieldoption(session, field_id, fieldoption_id, option_dict, language, idx):
option_dict['field_id'] = field_id
fill_localized_keys(option_dict, models.FieldOption.localized_keys, language)
o = None
if fieldoption_id is not None:
o = session.query(models.FieldOption).filter(models.FieldOption.id == fieldoption_id).one_or_none()
if o is None:
o = models.db_forge_obj(session, models.FieldOption, option_dict)
else:
o.update(option_dict)
o.presentation_order = idx
return o.id
def db_create_step(session, tid, step_dict, language):
"""
Create the specified step
:param session: the session on which perform queries.
:param language: the language of the specified steps.
"""
fill_localized_keys(step_dict, models.Step.localized_keys, language)
step = models.db_forge_obj(session, models.Step, step_dict)
for trigger in step_dict.get('triggered_by_options', []):
db_create_trigger(session, tid, trigger['option'], 'step', step.id, trigger.get('sufficient', True))
for c in step_dict['children']:
c['tid'] = tid
c['step_id'] = step.id
db_create_field(session, tid, c, language)
return step
def migrate_Context(self):
old_contexts = self.store_old.find(self.model_from['Context'])
new_step_model = self.model_to['Step']
new_field_model = self.model_to['Field']
new_fieldoption_model = self.model_to['FieldOption']
steps = self.appdata['default_questionnaire']
i = 1
for step in steps:
step['number'] = i
del step['children']
i += 1
for old_context in old_contexts:
new_context = self.model_to['Context']()
step1 = db_forge_obj(self.store_new, new_step_model, steps[0])
step1.context_id = new_context.id
step2 = db_forge_obj(self.store_new, new_step_model, steps[1])
step2.context_id = new_context.id
for _, v in new_context._storm_columns.iteritems():
if v.name == 'steps':
continue
setattr(new_context, v.name, getattr(old_context, v.name))
for f in old_context.unique_fields:
try:
field_dict = {}
field_dict['label'] = {}
field_dict['hint'] = {}
field_dict['description'] = {}
field_dict['multi_entry'] = False
field_dict['required'] = old_context.unique_fields[f]['required']
field_dict['preview'] = old_context.unique_fields[f]['preview']
field_dict['stats_enabled'] = False
field_dict['is_template'] = False
field_dict['x'] = 0
field_dict['y'] = old_context.unique_fields[f]['presentation_order']
if old_context.unique_fields[f]['type'] in ['email',
'phone',
'url',
'number',
'text']:
field_dict['type'] = 'inputbox'
elif old_context.unique_fields[f]['type'] in ['radio', 'select']:
field_dict['type'] = 'selectbox'
elif old_context.unique_fields[f]['type'] in ['multiple', 'checkboxes']:
field_dict['type'] = 'checkbox'
else:
field_dict['type'] = old_context.unique_fields[f]['type']
for l in old_context.localized_fields:
if f in old_context.localized_fields[l]:
field_dict['label'][l] = old_context.localized_fields[l][f]['name']
field_dict['hint'][l] = old_context.localized_fields[l][f]['hint']
field = db_forge_obj(self.store_new, new_field_model, field_dict)
if field_dict['type'] in ['selectbox', 'checkbox'] and 'options' in old_context.unique_fields[f]:
j = 1
for o in old_context.unique_fields[f]['options']:
opt_dict = {}
opt_dict['number'] = j
opt_dict['field_id'] = field.id
opt_dict['attrs'] = {}
opt_dict['attrs']['name'] = {}
for lang in LANGUAGES_SUPPORTED_CODES:
opt_dict['attrs']['name'][lang] = o['name']
option = db_forge_obj(self.store_new, new_fieldoption_model, opt_dict)
field.options.add(option)
j += 1
step1.children.add(field)
except Exception:
self.entries_count['Context'] -= 1
continue
self.store_new.add(new_context)
def create_shorturl(session, tid, request):
request['tid'] = tid
request['shorturl'] = '/u/' + request['shorturl']
shorturl = models.db_forge_obj(session, models.ShortURL, request)
return serialize_shorturl(shorturl)
def db_create_context(store, request, language=GLSetting.memory_copy.default_language):
"""
Creates a new context from the request of a client.
We associate to the context the list of receivers and if the receiver is
not valid we raise a ReceiverIdNotFound exception.
Args:
(dict) the request containing the keys to set on the model.
Returns:
(dict) representing the configured context
"""
receivers = request.get('receivers', [])
steps = request.get('steps', [])
fill_localized_keys(request, models.Context.localized_strings, language)
context = models.Context(request)
# Integrity checks related on name (need to exists, need to be unique)
# are performed only using the default language at the moment (XXX)
try:
context_name = request['name'][language]
except Exception as excep:
raise errors.InvalidInputFormat("language %s do not provide name: %s" %
(language, excep) )
if len(context_name) < 1:
log.err("Invalid request: name is an empty string")
raise errors.InvalidInputFormat("Context name is missing (1 char required)")
if request['select_all_receivers']:
if request['maximum_selectable_receivers']:
log.debug("Resetting maximum_selectable_receivers (%d) because 'select_all_receivers' is True" %
request['maximum_selectable_receivers'])
request['maximum_selectable_receivers'] = 0
# tip_timetolive and submission_timetolive need to be converted in seconds since hours and days
(context.submission_timetolive, context.tip_timetolive) = acquire_context_timetolive(request)
c = store.add(context)
for receiver_id in receivers:
receiver = models.Receiver.get(store, receiver_id)
if not receiver:
log.err("Creation error: unexistent context can't be associated")
raise errors.ReceiverIdNotFound
c.receivers.add(receiver)
if steps:
db_create_step(store, context.id, steps, language)
else:
appdata = store.find(models.ApplicationData).one()
steps = copy.deepcopy(appdata.fields)
n_s = 1
for step in steps:
for f_child in step['children']:
if not field_is_present(store, f_child):
f_child['is_template'] = False
for step in steps:
f_children = copy.deepcopy(step['children'])
del step['children']
s = models.db_forge_obj(store, models.Step, step)
for f_child in f_children:
o_children = copy.deepcopy(f_child['options'])
del f_child['options']
# FIXME currently default updata do not handle fieldgroups
# all this block must be redesigned in order to be called recursively
del f_child['children']
f = models.db_forge_obj(store, models.Field, f_child)
n_o = 1
for o_child in o_children:
o = models.db_forge_obj(store, models.FieldOption, o_child)
o.field_id = f.id
o.number = n_o
f.options.add(o)
n_o += 1
f.step_id = s.id
s.children.add(f)
s.context_id = context.id
s.number = n_s
context.steps.add(s)
n_s += 1
log.debug("Created context %s (using %s)" % (context_name, language) )
return admin_serialize_context(store, context, language)
def migrate_Context(self):
print "%s Context migration assistant" % self.std_fancy
old_contexts = self.store_old.find(self.get_right_model("Context", 14))
steps = load_appdata()['fields']
i = 1
for step in steps:
step['number'] = i
del step['children']
i += 1
for old_context in old_contexts:
new_context = self.get_right_model("Context", 15)()
step1 = db_forge_obj(self.store_new, self.get_right_model("Step", 15), steps[0])
step1.context_id = new_context.id
step2 = db_forge_obj(self.store_new, self.get_right_model("Step", 15), steps[1])
step2.context_id = new_context.id
for _, v in new_context._storm_columns.iteritems():
if v.name == 'steps':
continue
setattr(new_context, v.name, getattr(old_context, v.name))
for f in old_context.unique_fields:
try:
field_dict = {}
field_dict['label'] = {}
field_dict['hint'] = {}
field_dict['description'] = {}
field_dict['multi_entry'] = False
field_dict['required'] = old_context.unique_fields[f]['required']
field_dict['preview'] = old_context.unique_fields[f]['preview']
field_dict['stats_enabled'] = False
field_dict['is_template'] = False
field_dict['x'] = 0
field_dict['y'] = old_context.unique_fields[f]['presentation_order']
if old_context.unique_fields[f]['type'] in ['email',
'phone',
'url',
'number',
'text']:
field_dict['type'] = 'inputbox'
elif old_context.unique_fields[f]['type'] in ['radio', 'select']:
field_dict['type'] = 'selectbox'
elif old_context.unique_fields[f]['type'] in ['multiple', 'checkboxes']:
field_dict['type'] = 'checkbox'
else:
field_dict['type'] = old_context.unique_fields[f]['type']
for l in old_context.localized_fields:
if f in old_context.localized_fields[l]:
field_dict['label'][l] = old_context.localized_fields[l][f]['name']
field_dict['hint'][l] = old_context.localized_fields[l][f]['hint']
continue
field = db_forge_obj(self.store_new, self.get_right_model("Field", 15), field_dict)
if field_dict['type'] in ['selectbox', 'checkbox'] and 'options' in old_context.unique_fields[f]:
j = 1
for o in old_context.unique_fields[f]['options']:
opt_dict = {}
opt_dict['number'] = j
opt_dict['field_id'] = field.id
opt_dict['attrs'] = {}
opt_dict['attrs']['name'] = {}
for lang in LANGUAGES_SUPPORTED_CODES:
opt_dict['attrs']['name'][lang] = o['name']
option = db_forge_obj(self.store_new, self.get_right_model("FieldOption", 15), opt_dict)
field.options.add(option)
j += 1
step1.children.add(field)
except Exception:
continue
self.store_new.add(new_context)
self.store_new.commit()
