def count_queries_for_model(self, app, model): try: model_instance = new(model, print_errors=False) except Exception as e: self.report.add_error('- Could not prepare %s: %s' % (get_unique_model_name(model), str(e))) return db.reset_queries() try: model_instance.save() except Exception as e: self.report.add_error('- Could not insert %s: %s' % (get_unique_model_name(model), str(e))) return queries_insert = len(connection.queries) db.reset_queries() try: model_instance.save() except Exception as e: self.report.add_error('- Could not update %s: %s' % (get_unique_model_name(model), str(e))) return queries_update = len(connection.queries) self.report.add_record(app, model, queries_insert, queries_update)
def new(self, model_class, shelve=False, named_shelve=None, persist_dependencies=True, **kwargs): """ Create an instance filled with data without persist it. 1) validate all kwargs match Model.fields. 2) validate model is a model.Model class. 3) Iterate model fields: for each field, fill it with data. :shelve: the current configuration will be stored in the DDF library. It can be True or a string (named shelve). :named_shelve: restore configuration saved in DDF library with a name. :persist_dependencies: tell if internal dependencies will be saved in the database or not. """ if self.debug_mode: LOGGER.debug('>>> [%s] Generating instance.' % get_unique_model_name(model_class)) configuration = self._configure_params(model_class, shelve, named_shelve, **kwargs) instance = model_class() if not is_model_class(instance): raise InvalidModelError(get_unique_model_name(model_class)) for field in get_fields_from_model(model_class): if is_key_field(field) and 'id' not in configuration: continue if field.name not in self.kwargs and self._is_ignored_field(field.name): continue self.set_data_for_a_field(model_class, instance, field, persist_dependencies=persist_dependencies, **configuration) number_of_pending_fields = len(self.pending_fields) # For Copier fixtures: dealing with pending fields that need to receive values of another fields. i = 0 while self.pending_fields != []: field_name = self.pending_fields.pop(0) field = get_field_by_name_or_raise(model_class, field_name) self.set_data_for_a_field(model_class, instance, field, persist_dependencies=persist_dependencies, **configuration) i += 1 if i > 2 * number_of_pending_fields: # dealing with infinite loop too. raise InvalidConfigurationError(get_unique_field_name(field), 'Cyclic dependency of Copiers.') if self.debug_mode: LOGGER.debug('<<< [%s] Instance created.' % get_unique_model_name(model_class)) return instance
def new(self, model_class, shelve=False, named_shelve=None, persist_dependencies=True, **kwargs): """ Create an instance filled with data without persist it. 1) validate all kwargs match Model.fields. 2) validate model is a model.Model class. 3) Iterate model fields: for each field, fill it with data. @shelve: the current configuration will be stored in the DDF library. It can be True or a string (named shelve). @named_shelve: restore configuration saved in DDF library with a name. @persist_dependencies: tell if internal dependencies will be saved in the database or not. """ if self.debug_mode: LOGGER.debug('>>> [%s] Generating instance.' % get_unique_model_name(model_class)) configuration = self._configure_params(model_class, shelve, named_shelve, **kwargs) instance = model_class() if not is_model_class(instance): raise InvalidModelError(get_unique_model_name(model_class)), None, sys.exc_info()[2] for field in get_fields_from_model(model_class): if is_key_field(field) and 'id' not in configuration: continue if field.name in self.ignore_fields: continue self.set_data_for_a_field(model_class, instance, field, persist_dependencies=persist_dependencies, **configuration) number_of_pending_fields = len(self.pending_fields) # For Copier fixtures: dealing with pending fields that need to receive values of another fields. i = 0 while self.pending_fields != []: field_name = self.pending_fields.pop(0) field = get_field_by_name_or_raise(model_class, field_name) self.set_data_for_a_field(model_class, instance, field, persist_dependencies=persist_dependencies, **configuration) i += 1 if i > 2 * number_of_pending_fields: # dealing with infinite loop too. raise InvalidConfigurationError(get_unique_field_name(field), u'Cyclic dependency of Copiers.'), None, sys.exc_info()[2] if self.debug_mode: LOGGER.debug('<<< [%s] Instance created.' % get_unique_model_name(model_class)) return instance
def get(self, model_class, shelve=False, named_shelve=None, **kwargs): """ Create an instance with data and persist it. :shelve: the current configuration will be stored in the DDF library. :named_shelve: restore configuration saved in DDF library with a name. """ if 'locators' in kwargs: locators = kwargs.pop('locators') else: locators = [] if len(kwargs) > 0 and locators == [] and _get_instance_by_args(model_class, **kwargs) != None: ret = _get_instance_by_args(model_class, **kwargs) return ret elif locators: # we need to be sure that we havent build something meeting the locators yet. ret = self._check_against_db(model_class, locators, **kwargs) if ret: return ret if(model_class in get_created_classes() and model_class not in ALLOW_MULTIPLE): return get_instance_by_class(model_class) instance = self.new(model_class, locators, shelve=shelve, named_shelve=named_shelve, **kwargs) if is_model_abstract(model_class): raise InvalidModelError(get_unique_model_name(model_class)) try: if self.validate_models: instance.full_clean() if model_class in _PRE_SAVE: try: _PRE_SAVE[model_class](instance) except Exception as e: six.reraise(InvalidReceiverError, InvalidReceiverError(e), sys.exc_info()[2]) instance = self._save_the_instance(instance) created_instances.add(instance) if model_class in _POST_SAVE: try: _POST_SAVE[model_class](instance) except Exception as e: six.reraise(InvalidReceiverError, InvalidReceiverError(e), sys.exc_info()[2]) except Exception as e: if self.print_errors: print_field_values(instance) six.reraise(BadDataError, BadDataError(get_unique_model_name(model_class), e), sys.exc_info()[2]) self.fields_processed = [] # TODO: need more tests for M2M and Copier self.pending_fields = [] for field in get_many_to_many_fields_from_model(model_class): if field.name in kwargs.keys(): # TODO: library manytomany_field = getattr(instance, field.name) fixture = kwargs[field.name] try: self._process_many_to_many_field(field, manytomany_field, fixture, instance) except InvalidManyToManyConfigurationError as e: six.reraise(InvalidManyToManyConfigurationError, e, sys.exc_info()[2]) except Exception as e: six.reraise(InvalidManyToManyConfigurationError, InvalidManyToManyConfigurationError(get_unique_field_name(field), e), sys.exc_info()[2]) return instance
def get(self, model_class, shelve=False, named_shelve=None, **kwargs): """ Create an instance with data and persist it. :shelve: the current configuration will be stored in the DDF library. :named_shelve: restore configuration saved in DDF library with a name. """ instance = self.new(model_class, shelve=shelve, named_shelve=named_shelve, **kwargs) if is_model_abstract(model_class): raise InvalidModelError(get_unique_model_name(model_class)) try: if self.validate_models: instance.full_clean() if model_class in _PRE_SAVE: try: _PRE_SAVE[model_class](instance) except Exception as e: six.reraise(InvalidReceiverError, InvalidReceiverError(e), sys.exc_info()[2]) self._save_the_instance(instance) if model_class in _POST_SAVE: try: _POST_SAVE[model_class](instance) except Exception as e: six.reraise(InvalidReceiverError, InvalidReceiverError(e), sys.exc_info()[2]) except Exception as e: if self.print_errors: print_field_values(instance) six.reraise(BadDataError, BadDataError(get_unique_model_name(model_class), e), sys.exc_info()[2]) self.fields_processed = [] # TODO: need more tests for M2M and Copier self.pending_fields = [] for field in get_many_to_many_fields_from_model(model_class): if field.name in kwargs.keys(): # TODO: library manytomany_field = getattr(instance, field.name) fixture = kwargs[field.name] try: self._process_many_to_many_field(field, manytomany_field, fixture, instance) except InvalidManyToManyConfigurationError as e: six.reraise(InvalidManyToManyConfigurationError, e, sys.exc_info()[2]) except Exception as e: six.reraise( InvalidManyToManyConfigurationError, InvalidManyToManyConfigurationError( get_unique_field_name(field), e), sys.exc_info()[2]) return instance
def get(self, model_class, ddf_lesson=None, **kwargs): ''' Create an instance with data and persist it. :ddf_lesson: a custom lesson that will be used to create the model object. ''' instance = self.new(model_class, ddf_lesson=ddf_lesson, **kwargs) if is_model_abstract(model_class): raise InvalidModelError(get_unique_model_name(model_class)) try: if self.validate_models: instance.full_clean() if model_class in _PRE_SAVE: try: _PRE_SAVE[model_class](instance) except Exception as e: six.reraise(InvalidReceiverError, InvalidReceiverError(e), sys.exc_info()[2]) self._save_the_instance(instance) if model_class in _POST_SAVE: try: _POST_SAVE[model_class](instance) except Exception as e: six.reraise(InvalidReceiverError, InvalidReceiverError(e), sys.exc_info()[2]) except Exception as e: if self.print_errors: print_field_values(instance) six.reraise(BadDataError, BadDataError(get_unique_model_name(model_class), e), sys.exc_info()[2]) self.fields_processed = [] # TODO: need more tests for M2M and Copier self.pending_fields = [] for field in get_many_to_many_fields_from_model(model_class): if field.name in kwargs.keys(): # TODO: library manytomany_field = getattr(instance, field.name) fixture = kwargs[field.name] try: self._process_many_to_many_field(field, manytomany_field, fixture, instance) except InvalidManyToManyConfigurationError as e: six.reraise(InvalidManyToManyConfigurationError, e, sys.exc_info()[2]) except Exception as e: six.reraise( InvalidManyToManyConfigurationError, InvalidManyToManyConfigurationError( get_unique_field_name(field), e), sys.exc_info()[2]) return instance
def set_data_for_a_field(self, model_class, instance, field, persist_dependencies=True, **kwargs): if field.name in kwargs: config = kwargs[field.name] try: data = self._process_field_with_customized_fixture(instance, field, config, persist_dependencies) except PendingField: return # ignore this field for a while. except Exception as e: raise InvalidConfigurationError(get_unique_field_name(field), e), None, sys.exc_info()[2] else: data = self._process_field_with_default_fixture(field, model_class, persist_dependencies) if is_file_field(field) and data: django_file = data if isinstance(django_file, File): setattr(instance, field.name, data.name) # set the attribute if django_file.file.mode != 'rb': django_file.file.close() # this file may be open in another mode, for example, in a+b opened_file = open(django_file.file.name, 'rb') # to save the file it must be open in rb mode django_file.file = opened_file # we update the reference to the rb mode opened file getattr(instance, field.name).save(django_file.name, django_file) # save the file into the file storage system django_file.close() else: # string (saving just a name in the file, without saving the file to the storage file system setattr(instance, field.name, data) # Model.field = data else: if self.debug_mode: LOGGER.debug('%s.%s = %s' % (get_unique_model_name(model_class), field.name, data)) setattr(instance, field.name, data) # Model.field = data self.fields_processed.append(field.name)
def set_data_for_a_field(self, model_class, __instance, __field, persist_dependencies=True, **kwargs): if __field.name in kwargs: config = kwargs[__field.name] try: data = self._process_field_with_customized_fixture( __instance, __field, config, persist_dependencies) except PendingField: return # ignore this field for a while. except Exception as e: six.reraise( InvalidConfigurationError, InvalidConfigurationError(get_unique_field_name(__field), e), sys.exc_info()[2]) else: data = self._process_field_with_default_fixture( __field, model_class, persist_dependencies) if is_file_field(__field) and data: django_file = data if isinstance(django_file, File): setattr(__instance, __field.name, data.name) # set the attribute if hasattr(django_file.file, 'mode') and django_file.file.mode != 'rb': django_file.file.close( ) # this file may be open in another mode, for example, in a+b opened_file = open( django_file.file.name, 'rb') # to save the file it must be open in rb mode django_file.file = opened_file # we update the reference to the rb mode opened file # https://github.com/paulocheque/django-dynamic-fixture/issues/10 # getattr(__instance, __field.name).save(django_file.name, django_file) # save the file into the file storage system # django_file.close() getattr(__instance, __field.name).save(django_file.name, django_file, save=False) else: # string (saving just a name in the file, without saving the file to the storage file system setattr(__instance, __field.name, data) # Model.field = data else: if self.debug_mode: LOGGER.debug( '%s.%s = %s' % (get_unique_model_name(model_class), __field.name, data)) try: setattr(__instance, __field.name, data) # Model.field = data except ValueError as e: if is_relationship_field(__field): setattr(__instance, "%s_id" % __field.name, data) # Model.field = data else: six.reraise(*sys.exc_info()) self.fields_processed.append(__field.name)
def export_csv(self, order_by_quantity_queries=False): if order_by_quantity_queries: self.data.sort(key=lambda t: t[2], reverse=True) print 'APP.MODEL;QUERIES ON INSERT;QUERIES ON UPDATE' for app, model, queries_insert, queries_update in self.data: print '%s;%s;%s' % (get_unique_model_name(model), queries_insert, queries_update) for err in self.errors: print err
def export_csv(self, order_by_quantity_queries=False): if order_by_quantity_queries: self.data.sort(key=lambda t: t[2], reverse=True) print('APP.MODEL;QUERIES ON INSERT;QUERIES ON UPDATE') for app, model, queries_insert, queries_update in self.data: print('%s;%s;%s' % (get_unique_model_name(model), queries_insert, queries_update)) for err in self.errors: print(err)
def get_configuration(self, model_class, name=None): if name is None: name = self.DEFAULT_KEY # copy is important because this dict will be updated every time in the algorithm. config = self.configs.get(model_class, {}) if name != self.DEFAULT_KEY and name not in config.keys(): raise InvalidConfigurationError( 'There is no shelved configuration for model %s with the name "%s"' % (get_unique_model_name(model_class), name)) return config.get( name, {}).copy() # default configuration never raises an error
def get(self, model_class, shelve=False, named_shelve=None, **kwargs): """ Create an instance with data and persist it. :shelve: the current configuration will be stored in the DDF library. :named_shelve: restore configuration saved in DDF library with a name. """ instance = self.new(model_class, shelve=shelve, named_shelve=named_shelve, **kwargs) if is_model_abstract(model_class): raise InvalidModelError(get_unique_model_name(model_class)) try: if self.validate_models: instance.full_clean() if model_class in _PRE_SAVE: try: _PRE_SAVE[model_class](instance) except Exception as e: six.reraise(InvalidReceiverError, InvalidReceiverError(e), sys.exc_info()[2]) self._save_the_instance(instance) if model_class in _POST_SAVE: try: _POST_SAVE[model_class](instance) except Exception as e: six.reraise(InvalidReceiverError, InvalidReceiverError(e), sys.exc_info()[2]) except Exception as e: if self.print_errors: print_field_values(instance) six.reraise(BadDataError, BadDataError(get_unique_model_name(model_class), e), sys.exc_info()[2]) self.fields_processed = [] # TODO: need more tests for M2M and Copier self.pending_fields = [] for field in get_many_to_many_fields_from_model(model_class): if field.name in kwargs.keys(): # TODO: library manytomany_field = getattr(instance, field.name) fixture = kwargs[field.name] try: self._process_many_to_many_field(field, manytomany_field, fixture, instance) except InvalidManyToManyConfigurationError as e: six.reraise(InvalidManyToManyConfigurationError, e, sys.exc_info()[2]) except Exception as e: six.reraise(InvalidManyToManyConfigurationError, InvalidManyToManyConfigurationError(get_unique_field_name(field), e), sys.exc_info()[2]) return instance
def set_data_for_a_field(self, model_class, instance, field, persist_dependencies=True, **kwargs): if field.name in kwargs: config = kwargs[field.name] try: data = self._process_field_with_customized_fixture( instance, field, config, persist_dependencies) except PendingField: return # ignore this field for a while. except Exception as e: six.reraise( InvalidConfigurationError, InvalidConfigurationError(get_unique_field_name(field), e), sys.exc_info()[2]) else: data = self._process_field_with_default_fixture( field, model_class, persist_dependencies) if is_file_field(field) and data: django_file = data if isinstance(django_file, File): setattr(instance, field.name, data.name) # set the attribute if django_file.file.mode != 'rb': django_file.file.close( ) # this file may be open in another mode, for example, in a+b opened_file = open( django_file.file.name, 'rb') # to save the file it must be open in rb mode django_file.file = opened_file # we update the reference to the rb mode opened file getattr(instance, field.name).save( django_file.name, django_file) # save the file into the file storage system django_file.close() else: # string (saving just a name in the file, without saving the file to the storage file system setattr(instance, field.name, data) # Model.field = data else: if self.debug_mode: LOGGER.debug( '%s.%s = %s' % (get_unique_model_name(model_class), field.name, data)) setattr(instance, field.name, data) # Model.field = data self.fields_processed.append(field.name)
def set_data_for_a_field(self, model_class, __instance, __field, persist_dependencies=True, **kwargs): if __field.name in kwargs: config = kwargs[__field.name] try: data = self._process_field_with_customized_fixture(__instance, __field, config, persist_dependencies) except PendingField: return # ignore this field for a while. except Exception as e: six.reraise(InvalidConfigurationError, InvalidConfigurationError(get_unique_field_name(__field), e), sys.exc_info()[2]) else: data = self._process_field_with_default_fixture(__field, model_class, persist_dependencies) if is_file_field(__field) and data: django_file = data if isinstance(django_file, File): setattr(__instance, __field.name, data.name) # set the attribute if hasattr(django_file.file, 'mode') and django_file.file.mode != 'rb': django_file.file.close() # this file may be open in another mode, for example, in a+b opened_file = open(django_file.file.name, 'rb') # to save the file it must be open in rb mode django_file.file = opened_file # we update the reference to the rb mode opened file # https://github.com/paulocheque/django-dynamic-fixture/issues/10 # getattr(__instance, __field.name).save(django_file.name, django_file) # save the file into the file storage system # django_file.close() getattr(__instance, __field.name).save(django_file.name, django_file, save=False) else: # string (saving just a name in the file, without saving the file to the storage file system setattr(__instance, __field.name, data) # Model.field = data else: if self.debug_mode: LOGGER.debug('%s.%s = %s' % (get_unique_model_name(model_class), __field.name, data)) try: setattr(__instance, __field.name, data) # Model.field = data except ValueError as e: if is_relationship_field(__field): setattr(__instance, "%s_id" % __field.name, data) # Model.field = data else: six.reraise(*sys.exc_info()) self.fields_processed.append(__field.name)
def get_configuration(self, model_class, name=None): if name is None: name = self.DEFAULT_KEY # copy is important because this dict will be updated every time in the algorithm. config = self.configs.get(model_class, {}) if name != self.DEFAULT_KEY and name not in config.keys(): raise InvalidConfigurationError('There is no shelved configuration for model %s with the name "%s"' % (get_unique_model_name(model_class), name)) return config.get(name, {}).copy() # default configuration never raises an error
def new(self, model_class, ddf_lesson=None, persist_dependencies=True, **kwargs): ''' Create an instance filled with data without persist it. 1) validate all kwargs match Model.fields. 2) validate model is a model.Model class. 3) Iterate model fields: for each field, fill it with data. :ddf_lesson: the lesson that will be used to create the model instance, if exists. :persist_dependencies: tell if internal dependencies will be saved in the database or not. ''' if self.debug_mode: LOGGER.debug('>>> [%s] Generating instance.' % get_unique_model_name(model_class)) configuration = self._configure_params(model_class, ddf_lesson, **kwargs) instance = model_class() if not is_model_class(instance): raise InvalidModelError(get_unique_model_name(model_class)) try: # https://github.com/paulocheque/django-dynamic-fixture/pull/112 from polymorphic import PolymorphicModel is_polymorphic = isinstance(instance, PolymorphicModel) except ImportError: # Django-polymorphic is not installed so the model can't be polymorphic. is_polymorphic = False for field in get_fields_from_model(model_class): if is_key_field(field) and field.name not in configuration: continue if field.name not in self.kwargs and self._is_ignored_field( field.name): continue if is_polymorphic and (field.name == 'polymorphic_ctype' or field.primary_key): continue self.set_data_for_a_field( model_class, instance, field, persist_dependencies=persist_dependencies, **configuration) number_of_pending_fields = len(self.pending_fields) # For Copier fixtures: dealing with pending fields that need to receive values of another fields. i = 0 while self.pending_fields != []: field_name = self.pending_fields.pop(0) field = get_field_by_name_or_raise(model_class, field_name) self.set_data_for_a_field( model_class, instance, field, persist_dependencies=persist_dependencies, **configuration) i += 1 if i > 2 * number_of_pending_fields: # dealing with infinite loop too. raise InvalidConfigurationError( get_unique_field_name(field), 'Cyclic dependency of Copiers.') if self.debug_mode: LOGGER.debug('<<< [%s] Instance created.' % get_unique_model_name(model_class)) return instance