def hash_to_model(config, model_name, data_hash, instance):
"""
Takes a model specified in the config global and a
hash of json data and attempts to populate a django
model. Does not save.
"""
validate_model_dict(config, model_name, data_hash)
common_fields = config[model_name].get('common_fields', set())
renamed_fields = config[model_name].get('renamed_fields', {})
dependency_fields = config[model_name].get('dependencies', {})
model = config[model_name]['model_class']()
identity = (lambda x: x)
for field in (common_fields
.union(renamed_fields)
.union(dependency_fields.values())):
transform_fn = (config[model_name]
.get('value_transformers', {})
.get(field, identity))
transformed_value = transform_fn(data_hash['fields'][field])
field = renamed_fields.get(field, field)
if field in dependency_fields.values():
field += '_id'
setattr(model, field, transformed_value)
if model_hasattr(model, 'instance'):
model.instance = instance
return model
def hash_to_model(model_name, data_hash, instance, user):
"""
Takes a model specified in the MIGRATION_RULES global and a
hash of json data and attempts to populate a django
model. Does not save.
"""
validate_model(model_name, data_hash)
common_fields = MIGRATION_RULES[model_name].get('common_fields', set())
renamed_fields = MIGRATION_RULES[model_name].get('renamed_fields', {})
model = MIGRATION_RULES[model_name]['model_class']()
identity = (lambda x: x)
for field in common_fields.union(renamed_fields):
transformers = (MIGRATION_RULES[model_name]
.get('value_transformers', {}))
transform_value_fn = transformers.get(field, identity)
try:
transformed_value = transform_value_fn(data_hash['fields'][field])
field = renamed_fields.get(field, field)
setattr(model, field, transformed_value)
except ObjectDoesNotExist as d:
logger.warning("Warning: %s ... SKIPPING" % d)
if model_hasattr(model, 'instance'):
model.instance = instance
return model
def extra_select_and_values_for_model(instance,
job,
table,
model,
prefix=None):
if prefix:
prefix += '__'
else:
prefix = ''
perms = permissions(job.user, instance, model)
extra_select = {}
prefixed_names = []
dummy_instance = safe_get_model_class(model)()
for perm in (perm for perm in perms
if perm.permission_level >= FieldPermission.READ_ONLY):
field_name = perm.field_name
prefixed_name = prefix + field_name
if field_name.startswith('udf:'):
name = field_name[4:]
extra_select[prefixed_name] = "%s.udfs->'%s'" % (table, name)
else:
if not model_hasattr(dummy_instance, field_name):
# Exception will be raised downstream if you look for
# a field on a model that no longer exists but still
# has a stale permission record. Here we check for that
# case and don't include the field if it does not exist.
continue
prefixed_names.append(prefixed_name)
return (extra_select, prefixed_names)
def extra_select_and_values_for_model(
instance, job, table, model, prefix=None):
if prefix:
prefix += '__'
else:
prefix = ''
perms = permissions(job.user, instance, model)
extra_select = {}
prefixed_names = []
dummy_instance = safe_get_model_class(model)()
for perm in (perm for perm in perms
if perm.permission_level >= FieldPermission.READ_ONLY):
field_name = perm.field_name
prefixed_name = prefix + field_name
if field_name.startswith('udf:'):
name = field_name[4:]
extra_select[prefixed_name] = "%s.udfs->'%s'" % (table, name)
else:
if not model_hasattr(dummy_instance, field_name):
# Exception will be raised downstream if you look for
# a field on a model that no longer exists but still
# has a stale permission record. Here we check for that
# case and don't include the field if it does not exist.
continue
prefixed_names.append(prefixed_name)
return (extra_select, prefixed_names)
def _values_for_model(instance,
job,
table,
model,
select,
select_params,
prefix=None):
if prefix:
prefix += '__'
else:
prefix = ''
prefixed_names = []
model_class = safe_get_model_class(model)
dummy_instance = model_class()
if hasattr(model_class, 'instance'):
dummy_instance.instance = instance
for field_name in dummy_instance.visible_fields(job.user):
prefixed_name = prefix + field_name
if field_name.startswith('udf:'):
name = field_name[4:]
if name in model_class.collection_udf_settings.keys():
field_definition_id = None
for udfd in udf_defs(instance, model):
if udfd.iscollection and udfd.name == name:
field_definition_id = udfd.id
if field_definition_id is None:
continue
select[prefixed_name] = ("""
WITH formatted_data AS (
SELECT concat('(', data, ')') as fdata
FROM %s
WHERE field_definition_id = %s and model_id = %s.id
)
SELECT array_to_string(array_agg(fdata), ', ', '*')
FROM formatted_data
""" % (UserDefinedCollectionValue._meta.db_table,
field_definition_id, table))
else:
select[prefixed_name] = "{0}.udfs->%s".format(table)
select_params.append(name)
else:
if not model_hasattr(dummy_instance, field_name):
# Exception will be raised downstream if you look for
# a field on a model that no longer exists but still
# has a stale permission record. Here we check for that
# case and don't include the field if it does not exist.
continue
prefixed_names.append(prefixed_name)
return prefixed_names
def _values_for_model(instance, job, table, model, select, select_params, prefix=None):
if prefix:
prefix += "__"
else:
prefix = ""
prefixed_names = []
model_class = safe_get_model_class(model)
dummy_instance = model_class()
for field_name in (
perm.field_name
for perm in field_permissions(job.user, instance, model)
if perm.permission_level >= FieldPermission.READ_ONLY
):
prefixed_name = prefix + field_name
if field_name.startswith("udf:"):
name = field_name[4:]
if name in model_class.collection_udf_settings.keys():
field_definition_id = None
for udfd in udf_defs(instance, model):
if udfd.iscollection and udfd.name == name:
field_definition_id = udfd.id
if field_definition_id is None:
continue
select[
prefixed_name
] = """
WITH formatted_data AS (
SELECT concat('(', data, ')') as fdata
FROM %s
WHERE field_definition_id = %s and model_id = %s.id
)
SELECT array_to_string(array_agg(fdata), ', ', '*')
FROM formatted_data
""" % (
UserDefinedCollectionValue._meta.db_table,
field_definition_id,
table,
)
else:
select[prefixed_name] = "{0}.udfs->%s".format(table)
select_params.append(name)
else:
if not model_hasattr(dummy_instance, field_name):
# Exception will be raised downstream if you look for
# a field on a model that no longer exists but still
# has a stale permission record. Here we check for that
# case and don't include the field if it does not exist.
continue
prefixed_names.append(prefixed_name)
return prefixed_names
def dict_to_model(config, model_name, data_dict, instance):
"""
Takes a model specified in the config global and a
dict of json data and attempts to populate a django
model. Does not save.
"""
validate_model_dict(config, model_name, data_dict)
common_fields = config[model_name].get('common_fields', set())
renamed_fields = config[model_name].get('renamed_fields', {})
dependency_fields = config[model_name].get('dependencies', {})
ModelClass = config[model_name].get('model_class')
if ModelClass is None:
return PROCESS_WITHOUT_SAVE
else:
model = ModelClass()
# instance *must* be set before UDF assignment
if model_hasattr(model, 'instance'):
model.instance = instance
for field in (common_fields
.union(renamed_fields)
.union(dependency_fields.values())):
transform_fn = (config[model_name]
.get('value_transformers', {})
.get(field, None))
transformed_value = (transform_fn(data_dict['fields'][field])
if transform_fn else data_dict['fields'][field])
transformed_field = renamed_fields.get(field, field)
if transformed_field.startswith('udf:'):
if transformed_value is not None:
model.udfs[transformed_field[4:]] = transformed_value
else:
suffix = ('_id'
if transformed_field in dependency_fields.values()
else '')
setattr(model, transformed_field + suffix, transformed_value)
result = model
for mutator in config[model_name].get('presave_actions', []):
if result != DO_NOT_PROCESS and result is not None:
result = mutator(result, data_dict)
# result can be one of three things:
# * A valid model object. This will be saved and a relic created for it.
# * A pseudo-enum value, DO_NOT_PROCESS, indicating that this
# record should be wholly disregarded. This means not even
# creating a relic.
# * A psuedo-enum value, PROCESS_WITHOUT_SAVE, indicating that
# this record won't be saved, but a relic should be create for it.
return result
def extra_select_and_values_for_model(
instance, job, table, model, prefix=None):
if prefix:
prefix += '__'
else:
prefix = ''
perms = permissions(job.user, instance, model)
extra_select = {}
prefixed_names = []
dummy_instance = safe_get_model_class(model)()
for perm in (perm for perm in perms
if perm.permission_level >= FieldPermission.READ_ONLY):
field_name = perm.field_name
prefixed_name = prefix + field_name
if field_name in _UDFC_FIELDS:
field_definition_id = None
for udfd in udf_defs(instance, model):
if udfd.iscollection and udfd.name == field_name[4:]:
field_definition_id = udfd.id
if field_definition_id is None:
continue
extra_select[prefixed_name] = (
"""
WITH formatted_data AS (
SELECT concat('(', data, ')') as fdata
FROM %s
WHERE field_definition_id = %s and model_id = %s.id
)
SELECT array_to_string(array_agg(fdata), ', ', '*')
FROM formatted_data
"""
% (UserDefinedCollectionValue._meta.db_table,
field_definition_id, table))
elif field_name.startswith('udf:'):
name = field_name[4:]
extra_select[prefixed_name] = "%s.udfs->'%s'" % (table, name)
else:
if not model_hasattr(dummy_instance, field_name):
# Exception will be raised downstream if you look for
# a field on a model that no longer exists but still
# has a stale permission record. Here we check for that
# case and don't include the field if it does not exist.
continue
prefixed_names.append(prefixed_name)
return (extra_select, prefixed_names)
def dict_to_model(config, model_name, data_dict, instance):
"""
Takes a model specified in the config global and a
dict of json data and attempts to populate a django
model. Does not save.
"""
validate_model_dict(config, model_name, data_dict)
common_fields = config[model_name].get('common_fields', set())
renamed_fields = config[model_name].get('renamed_fields', {})
dependency_fields = config[model_name].get('dependencies', {})
ModelClass = config[model_name].get('model_class')
if ModelClass is None:
return PROCESS_WITHOUT_SAVE
else:
model = ModelClass()
# instance *must* be set before UDF assignment
if model_hasattr(model, 'instance'):
model.instance = instance
for field in (common_fields.union(renamed_fields).union(
dependency_fields.values())):
transform_fn = (config[model_name].get('value_transformers',
{}).get(field, None))
transformed_value = (transform_fn(data_dict['fields'][field])
if transform_fn else data_dict['fields'][field])
transformed_field = renamed_fields.get(field, field)
if transformed_field.startswith('udf:'):
if transformed_value is not None:
model.udfs[transformed_field[4:]] = transformed_value
else:
suffix = ('_id' if transformed_field in dependency_fields.values()
else '')
setattr(model, transformed_field + suffix, transformed_value)
result = model
for mutator in config[model_name].get('presave_actions', []):
if result != DO_NOT_PROCESS and result is not None:
result = mutator(result, data_dict)
# result can be one of three things:
# * A valid model object. This will be saved and a relic created for it.
# * A pseudo-enum value, DO_NOT_PROCESS, indicating that this
# record should be wholly disregarded. This means not even
# creating a relic.
# * A psuedo-enum value, PROCESS_WITHOUT_SAVE, indicating that
# this record won't be saved, but a relic should be create for it.
return result
