def hash_to_model(config, model_name, data_hash, instance): """ Takes a model specified in the config global and a hash of json data and attempts to populate a django model. Does not save. """ validate_model_dict(config, model_name, data_hash) common_fields = config[model_name].get('common_fields', set()) renamed_fields = config[model_name].get('renamed_fields', {}) dependency_fields = config[model_name].get('dependencies', {}) model = config[model_name]['model_class']() identity = (lambda x: x) for field in (common_fields .union(renamed_fields) .union(dependency_fields.values())): transform_fn = (config[model_name] .get('value_transformers', {}) .get(field, identity)) transformed_value = transform_fn(data_hash['fields'][field]) field = renamed_fields.get(field, field) if field in dependency_fields.values(): field += '_id' setattr(model, field, transformed_value) if model_hasattr(model, 'instance'): model.instance = instance return model
def hash_to_model(model_name, data_hash, instance, user): """ Takes a model specified in the MIGRATION_RULES global and a hash of json data and attempts to populate a django model. Does not save. """ validate_model(model_name, data_hash) common_fields = MIGRATION_RULES[model_name].get('common_fields', set()) renamed_fields = MIGRATION_RULES[model_name].get('renamed_fields', {}) model = MIGRATION_RULES[model_name]['model_class']() identity = (lambda x: x) for field in common_fields.union(renamed_fields): transformers = (MIGRATION_RULES[model_name] .get('value_transformers', {})) transform_value_fn = transformers.get(field, identity) try: transformed_value = transform_value_fn(data_hash['fields'][field]) field = renamed_fields.get(field, field) setattr(model, field, transformed_value) except ObjectDoesNotExist as d: logger.warning("Warning: %s ... SKIPPING" % d) if model_hasattr(model, 'instance'): model.instance = instance return model
def extra_select_and_values_for_model(instance, job, table, model, prefix=None): if prefix: prefix += '__' else: prefix = '' perms = permissions(job.user, instance, model) extra_select = {} prefixed_names = [] dummy_instance = safe_get_model_class(model)() for perm in (perm for perm in perms if perm.permission_level >= FieldPermission.READ_ONLY): field_name = perm.field_name prefixed_name = prefix + field_name if field_name.startswith('udf:'): name = field_name[4:] extra_select[prefixed_name] = "%s.udfs->'%s'" % (table, name) else: if not model_hasattr(dummy_instance, field_name): # Exception will be raised downstream if you look for # a field on a model that no longer exists but still # has a stale permission record. Here we check for that # case and don't include the field if it does not exist. continue prefixed_names.append(prefixed_name) return (extra_select, prefixed_names)
def extra_select_and_values_for_model( instance, job, table, model, prefix=None): if prefix: prefix += '__' else: prefix = '' perms = permissions(job.user, instance, model) extra_select = {} prefixed_names = [] dummy_instance = safe_get_model_class(model)() for perm in (perm for perm in perms if perm.permission_level >= FieldPermission.READ_ONLY): field_name = perm.field_name prefixed_name = prefix + field_name if field_name.startswith('udf:'): name = field_name[4:] extra_select[prefixed_name] = "%s.udfs->'%s'" % (table, name) else: if not model_hasattr(dummy_instance, field_name): # Exception will be raised downstream if you look for # a field on a model that no longer exists but still # has a stale permission record. Here we check for that # case and don't include the field if it does not exist. continue prefixed_names.append(prefixed_name) return (extra_select, prefixed_names)
def _values_for_model(instance, job, table, model, select, select_params, prefix=None): if prefix: prefix += '__' else: prefix = '' prefixed_names = [] model_class = safe_get_model_class(model) dummy_instance = model_class() if hasattr(model_class, 'instance'): dummy_instance.instance = instance for field_name in dummy_instance.visible_fields(job.user): prefixed_name = prefix + field_name if field_name.startswith('udf:'): name = field_name[4:] if name in model_class.collection_udf_settings.keys(): field_definition_id = None for udfd in udf_defs(instance, model): if udfd.iscollection and udfd.name == name: field_definition_id = udfd.id if field_definition_id is None: continue select[prefixed_name] = (""" WITH formatted_data AS ( SELECT concat('(', data, ')') as fdata FROM %s WHERE field_definition_id = %s and model_id = %s.id ) SELECT array_to_string(array_agg(fdata), ', ', '*') FROM formatted_data """ % (UserDefinedCollectionValue._meta.db_table, field_definition_id, table)) else: select[prefixed_name] = "{0}.udfs->%s".format(table) select_params.append(name) else: if not model_hasattr(dummy_instance, field_name): # Exception will be raised downstream if you look for # a field on a model that no longer exists but still # has a stale permission record. Here we check for that # case and don't include the field if it does not exist. continue prefixed_names.append(prefixed_name) return prefixed_names
def _values_for_model(instance, job, table, model, select, select_params, prefix=None): if prefix: prefix += "__" else: prefix = "" prefixed_names = [] model_class = safe_get_model_class(model) dummy_instance = model_class() for field_name in ( perm.field_name for perm in field_permissions(job.user, instance, model) if perm.permission_level >= FieldPermission.READ_ONLY ): prefixed_name = prefix + field_name if field_name.startswith("udf:"): name = field_name[4:] if name in model_class.collection_udf_settings.keys(): field_definition_id = None for udfd in udf_defs(instance, model): if udfd.iscollection and udfd.name == name: field_definition_id = udfd.id if field_definition_id is None: continue select[ prefixed_name ] = """ WITH formatted_data AS ( SELECT concat('(', data, ')') as fdata FROM %s WHERE field_definition_id = %s and model_id = %s.id ) SELECT array_to_string(array_agg(fdata), ', ', '*') FROM formatted_data """ % ( UserDefinedCollectionValue._meta.db_table, field_definition_id, table, ) else: select[prefixed_name] = "{0}.udfs->%s".format(table) select_params.append(name) else: if not model_hasattr(dummy_instance, field_name): # Exception will be raised downstream if you look for # a field on a model that no longer exists but still # has a stale permission record. Here we check for that # case and don't include the field if it does not exist. continue prefixed_names.append(prefixed_name) return prefixed_names
def dict_to_model(config, model_name, data_dict, instance): """ Takes a model specified in the config global and a dict of json data and attempts to populate a django model. Does not save. """ validate_model_dict(config, model_name, data_dict) common_fields = config[model_name].get('common_fields', set()) renamed_fields = config[model_name].get('renamed_fields', {}) dependency_fields = config[model_name].get('dependencies', {}) ModelClass = config[model_name].get('model_class') if ModelClass is None: return PROCESS_WITHOUT_SAVE else: model = ModelClass() # instance *must* be set before UDF assignment if model_hasattr(model, 'instance'): model.instance = instance for field in (common_fields .union(renamed_fields) .union(dependency_fields.values())): transform_fn = (config[model_name] .get('value_transformers', {}) .get(field, None)) transformed_value = (transform_fn(data_dict['fields'][field]) if transform_fn else data_dict['fields'][field]) transformed_field = renamed_fields.get(field, field) if transformed_field.startswith('udf:'): if transformed_value is not None: model.udfs[transformed_field[4:]] = transformed_value else: suffix = ('_id' if transformed_field in dependency_fields.values() else '') setattr(model, transformed_field + suffix, transformed_value) result = model for mutator in config[model_name].get('presave_actions', []): if result != DO_NOT_PROCESS and result is not None: result = mutator(result, data_dict) # result can be one of three things: # * A valid model object. This will be saved and a relic created for it. # * A pseudo-enum value, DO_NOT_PROCESS, indicating that this # record should be wholly disregarded. This means not even # creating a relic. # * A psuedo-enum value, PROCESS_WITHOUT_SAVE, indicating that # this record won't be saved, but a relic should be create for it. return result
def extra_select_and_values_for_model( instance, job, table, model, prefix=None): if prefix: prefix += '__' else: prefix = '' perms = permissions(job.user, instance, model) extra_select = {} prefixed_names = [] dummy_instance = safe_get_model_class(model)() for perm in (perm for perm in perms if perm.permission_level >= FieldPermission.READ_ONLY): field_name = perm.field_name prefixed_name = prefix + field_name if field_name in _UDFC_FIELDS: field_definition_id = None for udfd in udf_defs(instance, model): if udfd.iscollection and udfd.name == field_name[4:]: field_definition_id = udfd.id if field_definition_id is None: continue extra_select[prefixed_name] = ( """ WITH formatted_data AS ( SELECT concat('(', data, ')') as fdata FROM %s WHERE field_definition_id = %s and model_id = %s.id ) SELECT array_to_string(array_agg(fdata), ', ', '*') FROM formatted_data """ % (UserDefinedCollectionValue._meta.db_table, field_definition_id, table)) elif field_name.startswith('udf:'): name = field_name[4:] extra_select[prefixed_name] = "%s.udfs->'%s'" % (table, name) else: if not model_hasattr(dummy_instance, field_name): # Exception will be raised downstream if you look for # a field on a model that no longer exists but still # has a stale permission record. Here we check for that # case and don't include the field if it does not exist. continue prefixed_names.append(prefixed_name) return (extra_select, prefixed_names)
def dict_to_model(config, model_name, data_dict, instance): """ Takes a model specified in the config global and a dict of json data and attempts to populate a django model. Does not save. """ validate_model_dict(config, model_name, data_dict) common_fields = config[model_name].get('common_fields', set()) renamed_fields = config[model_name].get('renamed_fields', {}) dependency_fields = config[model_name].get('dependencies', {}) ModelClass = config[model_name].get('model_class') if ModelClass is None: return PROCESS_WITHOUT_SAVE else: model = ModelClass() # instance *must* be set before UDF assignment if model_hasattr(model, 'instance'): model.instance = instance for field in (common_fields.union(renamed_fields).union( dependency_fields.values())): transform_fn = (config[model_name].get('value_transformers', {}).get(field, None)) transformed_value = (transform_fn(data_dict['fields'][field]) if transform_fn else data_dict['fields'][field]) transformed_field = renamed_fields.get(field, field) if transformed_field.startswith('udf:'): if transformed_value is not None: model.udfs[transformed_field[4:]] = transformed_value else: suffix = ('_id' if transformed_field in dependency_fields.values() else '') setattr(model, transformed_field + suffix, transformed_value) result = model for mutator in config[model_name].get('presave_actions', []): if result != DO_NOT_PROCESS and result is not None: result = mutator(result, data_dict) # result can be one of three things: # * A valid model object. This will be saved and a relic created for it. # * A pseudo-enum value, DO_NOT_PROCESS, indicating that this # record should be wholly disregarded. This means not even # creating a relic. # * A psuedo-enum value, PROCESS_WITHOUT_SAVE, indicating that # this record won't be saved, but a relic should be create for it. return result