def test_signature_save(self): """Testing Version.signature field serializes JSON-encoded v2 signatures """ project_sig = ProjectSignature() project_sig.add_app_sig(AppSignature('app1')) project_sig.add_app_sig(AppSignature('app2')) version = Version.objects.create(signature=project_sig) raw_signature = (Version.objects.filter( pk=version.pk).values_list('signature'))[0][0] self.assertTrue(raw_signature.startswith('json!')) sig_data = json.loads(raw_signature[len('json!'):]) self.assertEqual( sig_data, { '__version__': 2, 'apps': { 'app1': { 'legacy_app_label': 'app1', 'models': {}, }, 'app2': { 'legacy_app_label': 'app2', 'models': {}, }, }, })
def test_with_bad_model(self): """Testing ChangeField with model not in signature""" mutation = ChangeField('TestModel', 'char_field1') project_sig = ProjectSignature() project_sig.add_app_sig(AppSignature(app_id='tests')) message = ('Cannot change the field "char_field1" on model ' '"tests.TestModel". The model could not be found in the ' 'signature.') with self.assertRaisesMessage(SimulationFailure, message): mutation.run_simulation(app_label='tests', project_sig=project_sig, database_state=None)
def test_with_bad_model(self): """Testing DeleteModel with model not in signature""" mutation = DeleteModel('TestModel') project_sig = ProjectSignature() project_sig.add_app_sig(AppSignature(app_id='tests')) message = ( 'Cannot delete the model "tests.TestModel". The model could ' 'not be found in the signature.') with self.assertRaisesMessage(SimulationFailure, message): mutation.run_simulation(app_label='tests', project_sig=project_sig, database_state=None)
def test_with_bad_model(self): """Testing DeleteModel with model not in signature""" mutation = DeleteModel('TestModel') project_sig = ProjectSignature() project_sig.add_app_sig(AppSignature(app_id='tests')) message = ( 'Cannot delete the model "tests.TestModel". The model could ' 'not be found in the signature.' ) with self.assertRaisesMessage(SimulationFailure, message): mutation.run_simulation(app_label='tests', project_sig=project_sig, database_state=None)
def to_python(self, value): """Return a ProjectSignature value from the field contents. Args: value (object): The current value assigned to the field. This might be serialized string content or a :py:class:`~django_evolution.signatures.ProjectSignature` instance. Returns: django_evolution.signatures.ProjectSignature: The project signature stored in the field. Raises: django.core.exceptions.ValidationError: The field contents are of an unexpected type. """ if isinstance(value, six.string_types): return ProjectSignature.deserialize(pickle_loads(value)) elif isinstance(value, ProjectSignature): return value else: raise ValidationError('Unsupported serialized signature type %s' % type(value), code='invalid', params={ 'value': value, })
def test_with_bad_model(self): """Testing AddField with model not in signature""" mutation = AddField('TestModel', 'char_field1', models.CharField) project_sig = ProjectSignature() project_sig.add_app_sig(AppSignature(app_id='tests')) message = ( 'Cannot add the field "char_field1" to model "tests.TestModel". ' 'The model could not be found in the signature.' ) with self.assertRaisesMessage(SimulationFailure, message): mutation.run_simulation(app_label='tests', project_sig=project_sig, database_state=None)
def to_python(self, value): """Return a ProjectSignature value from the field contents. Args: value (object): The current value assigned to the field. This might be serialized string content or a :py:class:`~django_evolution.signatures.ProjectSignature` instance. Returns: django_evolution.signatures.ProjectSignature: The project signature stored in the field. Raises: django.core.exceptions.ValidationError: The field contents are of an unexpected type. """ if isinstance(value, six.string_types): if value.startswith('json!'): loaded_value = json.loads(value[len('json!'):], object_pairs_hook=OrderedDict) else: loaded_value = pickle_loads(value) return ProjectSignature.deserialize(loaded_value) elif isinstance(value, ProjectSignature): return value else: raise ValidationError( 'Unsupported serialized signature type %s' % type(value), code='invalid', params={ 'value': value, })
def simulate(self, simulation): """Simulate a mutation for an application. This will run the :py:attr:`update_func` provided when instantiating the mutation, passing it ``app_label`` and ``project_sig``. It should then modify the signature to match what the SQL statement would do. Args: simulation (Simulation): The state for the simulation. Raises: django_evolution.errors.CannotSimulate: :py:attr:`update_func` was not provided or was not a function. django_evolution.errors.SimulationFailure: The simulation failed. The reason is in the exception's message. This would be run by :py:attr:`update_func`. """ if callable(self.update_func): if hasattr(inspect, 'getfullargspec'): # Python 3 argspec = inspect.getfullargspec(self.update_func) else: # Python 2 argspec = inspect.getargspec(self.update_func) if len(argspec.args) == 1 and argspec.args[0] == 'simulation': # New-style simulation function. self.update_func(simulation) return elif len(argspec.args) == 2: # Legacy simulation function. project_sig = simulation.project_sig serialized_sig = project_sig.serialize(sig_version=1) self.update_func(simulation.app_label, serialized_sig) new_project_sig = ProjectSignature.deserialize(serialized_sig) # We have to reconstruct the existing project signature's state # based on this. app_sig_ids = [ app_sig.app_id for app_sig in new_project_sig.app_sigs ] for app_sig_id in app_sig_ids: project_sig.remove_app_sig(app_sig_id) for app_sig in new_project_sig.app_sigs: project_sig.add_app_sig(app_sig) return raise CannotSimulate( 'SQLMutations must provide an update_func(simulation) or ' 'legacy update_func(app_label, project_sig) parameter ' 'in order to be simulated.')
def test_current_version_with_dup_timestamps(self): """Testing Version.current_version() with two entries with same timestamps""" # Remove anything that may already exist. Version.objects.all().delete() timestamp = datetime(year=2015, month=12, day=10, hour=12, minute=13, second=14) Version.objects.create(signature=ProjectSignature(), when=timestamp) version = Version.objects.create(signature=ProjectSignature(), when=timestamp) latest_version = Version.objects.current_version() self.assertEqual(latest_version, version)
def test_add_fields_bad_update_func_signature(self): """Testing SQLMutation and bad update_func signature""" mutation = SQLMutation('test', '', update_func=lambda a, b, c: None) message = ('SQLMutations must provide an update_func(simulation) or ' 'legacy update_func(app_label, project_sig) parameter in ' 'order to be simulated.') with self.assertRaisesMessage(CannotSimulate, message): mutation.run_simulation(app_label='tests', project_sig=ProjectSignature(), database_state=None)
def test_with_bad_app(self): """Testing DeleteApplication with application not in signature""" mutation = DeleteApplication() message = ( 'Cannot delete the application "badapp". The application could ' 'not be found in the signature.') with self.assertRaisesMessage(SimulationFailure, message): mutation.run_simulation(app_label='badapp', project_sig=ProjectSignature(), database_state=None)
def test_with_bad_field(self): """Testing RenameField with field not in signature""" mutation = RenameField('TestModel', 'char_field1', 'char_field2') model_sig = ModelSignature(model_name='TestModel', table_name='tests_testmodel') app_sig = AppSignature(app_id='tests') app_sig.add_model_sig(model_sig) project_sig = ProjectSignature() project_sig.add_app_sig(app_sig) message = ('Cannot rename the field "char_field1" on model ' '"tests.TestModel". The field could not be found in the ' 'signature.') with self.assertRaisesMessage(SimulationFailure, message): mutation.run_simulation(app_label='tests', project_sig=project_sig, database_state=None)
def test_with_bad_app(self): """Testing AddField with application not in signature""" mutation = AddField('TestModel', 'char_field1', models.CharField) message = ( 'Cannot add the field "char_field1" to model "badapp.TestModel". ' 'The application could not be found in the signature.') with self.assertRaisesMessage(SimulationFailure, message): mutation.run_simulation(app_label='badapp', project_sig=ProjectSignature(), database_state=None)
def test_with_bad_app(self): """Testing RenameField with application not in signature""" mutation = RenameField('TestModel', 'char_field1', 'char_field2') message = ( 'Cannot rename the field "char_field1" on model ' '"badapp.TestModel". The application could not be found in the ' 'signature.') with self.assertRaisesMessage(SimulationFailure, message): mutation.run_simulation(app_label='badapp', project_sig=ProjectSignature(), database_state=None)
def test_with_bad_field(self): """Testing RenameField with field not in signature""" mutation = RenameField('TestModel', 'char_field1', 'char_field2') model_sig = ModelSignature(model_name='TestModel', table_name='tests_testmodel') app_sig = AppSignature(app_id='tests') app_sig.add_model_sig(model_sig) project_sig = ProjectSignature() project_sig.add_app_sig(app_sig) message = ( 'Cannot rename the field "char_field1" on model ' '"tests.TestModel". The field could not be found in the ' 'signature.' ) with self.assertRaisesMessage(SimulationFailure, message): mutation.run_simulation(app_label='tests', project_sig=project_sig, database_state=None)
def test_with_bad_app(self): """Testing RenameModel with application not in signature""" mutation = RenameModel('TestModel', 'DestModel', db_table='tests_destmodel') message = ( 'Cannot rename the model "badapp.TestModel". The application ' 'could not be found in the signature.' ) with self.assertRaisesMessage(SimulationFailure, message): mutation.run_simulation(app_label='badapp', project_sig=ProjectSignature(), database_state=None)
def test_with_bad_field(self): """Testing AddField with field already in signature""" mutation = AddField('TestModel', 'char_field1', models.CharField) model_sig = ModelSignature(model_name='TestModel', table_name='tests_testmodel') model_sig.add_field_sig(FieldSignature(field_name='char_field1', field_type=models.CharField)) app_sig = AppSignature(app_id='tests') app_sig.add_model_sig(model_sig) project_sig = ProjectSignature() project_sig.add_app_sig(app_sig) message = ( 'Cannot add the field "char_field1" to model "tests.TestModel". ' 'A field with this name already exists.' ) with self.assertRaisesMessage(SimulationFailure, message): mutation.run_simulation(app_label='tests', project_sig=project_sig, database_state=None)
def test_with_bad_field(self): """Testing AddField with field already in signature""" mutation = AddField('TestModel', 'char_field1', models.CharField) model_sig = ModelSignature(model_name='TestModel', table_name='tests_testmodel') model_sig.add_field_sig( FieldSignature(field_name='char_field1', field_type=models.CharField)) app_sig = AppSignature(app_id='tests') app_sig.add_model_sig(model_sig) project_sig = ProjectSignature() project_sig.add_app_sig(app_sig) message = ( 'Cannot add the field "char_field1" to model "tests.TestModel". ' 'A field with this name already exists.') with self.assertRaisesMessage(SimulationFailure, message): mutation.run_simulation(app_label='tests', project_sig=project_sig, database_state=None)
def create_test_project_sig(models, app_label='tests', version=1): """Return a dummy project signature for the given models. Args: models (list of django.db.models.Model): The list of models for the project signature. app_label (unicode, optional): The application label that will contain the models. version (int, optional): The signature version to use for the project signature. Returns: dict: The new project signature. """ app_sig = AppSignature(app_id=app_label) project_sig = ProjectSignature() project_sig.add_app_sig(app_sig) for full_name, model in models: parts = full_name.split('.') if len(parts) == 1: app_sig.add_model_sig(ModelSignature.from_model(model)) else: model_app_label, model_name = parts model_app_sig = project_sig.get_app_sig(model_app_label) if model_app_sig is None: model_app_sig = AppSignature(app_id=model_app_label) project_sig.add_app_sig(model_app_sig) model_app_sig.add_model_sig(ModelSignature.from_model(model)) return project_sig
def create_test_project_sig(models, app_label='tests', version=1): """Return a dummy project signature for the given models. Args: models (list of django.db.models.Model): The list of models for the project signature. app_label (unicode, optional): The application label that will contain the models. version (int, optional): The signature version to use for the project signature. Returns: dict: The new project signature. """ app_sig = AppSignature(app_id=app_label) project_sig = ProjectSignature() project_sig.add_app_sig(app_sig) for full_name, model in models: parts = full_name.split('.') if len(parts) == 1: app_sig.add_model_sig(ModelSignature.from_model(model)) else: model_app_label, model_name = parts model_app_sig = project_sig.get_app_sig(model_app_label) if model_app_sig is None: model_app_sig = AppSignature(app_id=model_app_label) project_sig.add_app_sig(model_app_sig) model_app_sig.add_model_sig(ModelSignature.from_model(model)) return project_sig
def get_mutations(app, evolution_labels, database=DEFAULT_DB_ALIAS): """Return the mutations provided by the given evolution names. Args: app (module): The app the evolutions belong to. evolution_labels (unicode): The labels of the evolutions to return mutations for. database (unicode, optional): The name of the database the evolutions cover. Returns: list of django_evolution.mutations.BaseMutation: The list of mutations provided by the evolutions. Raises: django_evolution.errors.EvolutionException: One or more evolutions are missing. """ # For each item in the evolution sequence. Check each item to see if it is # a python file or an sql file. try: app_name = get_app_name(app) if app_name in BUILTIN_SEQUENCES: module_name = 'django_evolution.builtin_evolutions' else: module_name = '%s.evolutions' % app_name evolution_module = import_module(module_name) except ImportError: return [] mutations = [] for label in evolution_labels: directory_name = os.path.dirname(evolution_module.__file__) # The first element is used for compatibility purposes. filenames = [ os.path.join(directory_name, label + '.sql'), os.path.join(directory_name, "%s_%s.sql" % (database, label)), ] found = False for filename in filenames: if os.path.exists(filename): sql_file = open(filename, 'r') sql = sql_file.readlines() sql_file.close() mutations.append(SQLMutation(label, sql)) found = True break if not found: try: module_name = [evolution_module.__name__, label] module = __import__('.'.join(module_name), {}, {}, [module_name]) mutations.extend(module.MUTATIONS) except ImportError: raise EvolutionException( 'Error: Failed to find an SQL or Python evolution named %s' % label) latest_version = Version.objects.current_version(using=database) app_id = get_app_label(app) old_project_sig = latest_version.signature project_sig = ProjectSignature.from_database(database) old_app_sig = old_project_sig.get_app_sig(app_id) app_sig = project_sig.get_app_sig(app_id) if old_app_sig is not None and app_sig is not None: # We want to go through now and make sure we're only applying # evolutions for models where the signature is different between # what's stored and what's current. # # The reason for this is that we may have just installed a baseline, # which would have the up-to-date signature, and we might be trying # to apply evolutions on top of that (which would already be applied). # These would generate errors. So, try hard to prevent that. # # First, Find the list of models in the latest signature of this app # that aren't in the old signature. changed_models = set( model_sig.model_name for model_sig in app_sig.model_sigs if old_app_sig.get_model_sig(model_sig.model_name) != model_sig ) # Now do the same for models in the old signature, in case the # model has been deleted. changed_models.update( old_model_sig.model_name for old_model_sig in old_app_sig.model_sigs if app_sig.get_model_sig(old_model_sig.model_name) is None ) # We should now have a full list of which models changed. Filter # the list of mutations appropriately. # # Changes affecting a model that was newly-introduced are removed, # unless the mutation is a RenameModel, in which case we'll need it # during the optimization step (and will remove it if necessary then). mutations = [ mutation for mutation in mutations if (not hasattr(mutation, 'model_name') or mutation.model_name in changed_models or isinstance(mutation, RenameModel)) ] return mutations
def __init__(self, hinted=False, verbosity=0, interactive=False, database_name=DEFAULT_DB_ALIAS): """Initialize the evolver. Args: hinted (bool, optional): Whether to operate against hinted evolutions. This may result in changes to the database without there being any accompanying evolution files backing those changes. verbosity (int, optional): The verbosity level for any output. This is passed along to signal emissions. interactive (bool, optional): Whether the evolution operations are being performed in a way that allows interactivity on the command line. This is passed along to signal emissions. database_name (unicode, optional): The name of the database to evolve. Raises: django_evolution.errors.EvolutionBaselineMissingError: An initial baseline for the project was not yet installed. This is due to ``syncdb``/``migrate`` not having been run. """ self.database_name = database_name self.hinted = hinted self.verbosity = verbosity self.interactive = interactive self.evolved = False self.initial_diff = None self.project_sig = None self.version = None self.installed_new_database = False self.connection = connections[database_name] if hasattr(self.connection, 'prepare_database'): # Django >= 1.8 self.connection.prepare_database() self.database_state = DatabaseState(self.database_name) self.target_project_sig = \ ProjectSignature.from_database(database_name) self._tasks_by_class = OrderedDict() self._tasks_by_id = OrderedDict() self._tasks_prepared = False latest_version = None if self.database_state.has_model(Version): try: latest_version = \ Version.objects.current_version(using=database_name) except Version.DoesNotExist: # We'll populate this next. pass if latest_version is None: # Either the models aren't yet synced to the database, or we # don't have a saved project signature, so let's set these up. self.installed_new_database = True self.project_sig = ProjectSignature() app = get_app('django_evolution') task = EvolveAppTask(evolver=self, app=app) task.prepare(hinted=False) with self.sql_executor() as sql_executor: task.execute(sql_executor=sql_executor, create_models_now=True) self.database_state.rescan_tables() app_sig = AppSignature.from_app(app=app, database=database_name) self.project_sig.add_app_sig(app_sig) # Let's make completely sure that we've only found the models # we expect. This is mostly for the benefit of unit tests. model_names = set(model_sig.model_name for model_sig in app_sig.model_sigs) expected_model_names = set(['Evolution', 'Version']) assert model_names == expected_model_names, ( 'Unexpected models found for django_evolution app: %s' % ', '.join(model_names - expected_model_names)) self._save_project_sig(new_evolutions=task.new_evolutions) latest_version = self.version self.project_sig = latest_version.signature self.initial_diff = Diff(self.project_sig, self.target_project_sig)
class Evolver(object): """The main class for managing database evolutions. The evolver is used to queue up tasks that modify the database. These allow for evolving database models and purging applications across an entire Django project or only for specific applications. Custom tasks can even be written by an application if very specific database operations need to be made outside of what's available in an evolution. Tasks are executed in order, but batched by the task type. That is, if two instances of ``TaskType1`` are queued, followed by an instance of ``TaskType2``, and another of ``TaskType1``, all 3 tasks of ``TaskType1`` will be executed at once, with the ``TaskType2`` task following. Callers are expected to create an instance and queue up one or more tasks. Once all tasks are queued, the changes can be made using :py:meth:`evolve`. Alternatively, evolution hints can be generated using :py:meth:`generate_hints`. Projects will generally utilize this through the existing ``evolve`` Django management command. Attributes: connection (django.db.backends.base.base.BaseDatabaseWrapper): The database connection object being used for the evolver. database_name (unicode): The name of the database being evolved. database_state (django_evolution.db.state.DatabaseState): The state of the database, for evolution purposes. evolved (bool): Whether the evolver has already performed its evolutions. These can only be done once per evolver. hinted (bool): Whether the evolver is operating against hinted evolutions. This may result in changes to the database without there being any accompanying evolution files backing those changes. interactive (bool): Whether the evolution operations are being performed in a way that allows interactivity on the command line. This is passed along to signal emissions. initial_diff (django_evolution.diff.Diff): The initial diff between the stored project signature and the current project signature. project_sig (django_evolution.signature.ProjectSignature): The project signature. This will start off as the previous signature stored in the database, but will be modified when mutations are simulated. verbosity (int): The verbosity level for any output. This is passed along to signal emissions. version (django_evolution.models.Version): The project version entry saved as the result of any evolution operations. This contains the current version of the project signature. It may be ``None`` until :py:meth:`evolve` is called. """ def __init__(self, hinted=False, verbosity=0, interactive=False, database_name=DEFAULT_DB_ALIAS): """Initialize the evolver. Args: hinted (bool, optional): Whether to operate against hinted evolutions. This may result in changes to the database without there being any accompanying evolution files backing those changes. verbosity (int, optional): The verbosity level for any output. This is passed along to signal emissions. interactive (bool, optional): Whether the evolution operations are being performed in a way that allows interactivity on the command line. This is passed along to signal emissions. database_name (unicode, optional): The name of the database to evolve. Raises: django_evolution.errors.EvolutionBaselineMissingError: An initial baseline for the project was not yet installed. This is due to ``syncdb``/``migrate`` not having been run. """ self.database_name = database_name self.hinted = hinted self.verbosity = verbosity self.interactive = interactive self.evolved = False self.initial_diff = None self.project_sig = None self.version = None self.installed_new_database = False self.connection = connections[database_name] if hasattr(self.connection, 'prepare_database'): # Django >= 1.8 self.connection.prepare_database() self.database_state = DatabaseState(self.database_name) self.target_project_sig = \ ProjectSignature.from_database(database_name) self._tasks_by_class = OrderedDict() self._tasks_by_id = OrderedDict() self._tasks_prepared = False latest_version = None if self.database_state.has_model(Version): try: latest_version = \ Version.objects.current_version(using=database_name) except Version.DoesNotExist: # We'll populate this next. pass if latest_version is None: # Either the models aren't yet synced to the database, or we # don't have a saved project signature, so let's set these up. self.installed_new_database = True self.project_sig = ProjectSignature() app = get_app('django_evolution') task = EvolveAppTask(evolver=self, app=app) task.prepare(hinted=False) with self.sql_executor() as sql_executor: task.execute(sql_executor=sql_executor, create_models_now=True) self.database_state.rescan_tables() app_sig = AppSignature.from_app(app=app, database=database_name) self.project_sig.add_app_sig(app_sig) # Let's make completely sure that we've only found the models # we expect. This is mostly for the benefit of unit tests. model_names = set(model_sig.model_name for model_sig in app_sig.model_sigs) expected_model_names = set(['Evolution', 'Version']) assert model_names == expected_model_names, ( 'Unexpected models found for django_evolution app: %s' % ', '.join(model_names - expected_model_names)) self._save_project_sig(new_evolutions=task.new_evolutions) latest_version = self.version self.project_sig = latest_version.signature self.initial_diff = Diff(self.project_sig, self.target_project_sig) @property def tasks(self): """A list of all tasks that will be performed. This can only be accessed after all necessary tasks have been queued. """ # If a caller is interested in the list of tasks, then it's likely # interested in state on those tasks. That means we'll need to prepare # all the tasks before we can return any of them. self._prepare_tasks() return six.itervalues(self._tasks_by_id) def can_simulate(self): """Return whether all queued tasks can be simulated. If any tasks cannot be simulated (for instance, a hinted evolution requiring manually-entered values), then this will return ``False``. This can only be called after all tasks have been queued. Returns: bool: ``True`` if all queued tasks can be simulated. ``False`` if any cannot. """ return all(task.can_simulate or not task.evolution_required for task in self.tasks) def get_evolution_required(self): """Return whether there are any evolutions required. This can only be called after all tasks have been queued. Returns: bool: ``True`` if any tasks require evolution. ``False`` if none do. """ return any(task.evolution_required for task in self.tasks) def diff_evolutions(self): """Return a diff between stored and post-evolution project signatures. This will run through all queued tasks, preparing them and simulating their changes. The returned diff will represent the changes made in those tasks. This can only be called after all tasks have been queued. Returns: django_evolution.diff.Diff: The diff between the stored signature and the queued changes. """ self._prepare_tasks() return Diff(self.project_sig, self.target_project_sig) def iter_evolution_content(self): """Generate the evolution content for all queued tasks. This will loop through each tasks and yield any evolution content provided. This can only be called after all tasks have been queued. Yields: tuple: A tuple of ``(task, evolution_content)``. """ for task in self.tasks: content = task.get_evolution_content() if content: yield task, content def queue_evolve_all_apps(self): """Queue an evolution of all registered Django apps. This cannot be used if :py:meth:`queue_evolve_app` is also being used. Raises: django_evolution.errors.EvolutionTaskAlreadyQueuedError: An evolution for an app was already queued. django_evolution.errors.QueueEvolverTaskError: Error queueing a non-duplicate task. Tasks may have already been prepared and finalized. """ for app in get_apps(): self.queue_evolve_app(app) def queue_evolve_app(self, app): """Queue an evolution of a registered Django app. Args: app (module): The Django app to queue an evolution for. Raises: django_evolution.errors.EvolutionTaskAlreadyQueuedError: An evolution for this app was already queued. django_evolution.errors.QueueEvolverTaskError: Error queueing a non-duplicate task. Tasks may have already been prepared and finalized. """ try: self.queue_task(EvolveAppTask(self, app)) except EvolutionTaskAlreadyQueuedError: raise EvolutionTaskAlreadyQueuedError( _('"%s" is already being tracked for evolution') % get_app_label(app)) def queue_purge_old_apps(self): """Queue the purging of all old, stale Django apps. This will purge any apps that exist in the stored project signature but that are no longer registered in Django. This generally should not be used if :py:meth:`queue_purge_app` is also being used. Raises: django_evolution.errors.EvolutionTaskAlreadyQueuedError: A purge of an app was already queued. django_evolution.errors.QueueEvolverTaskError: Error queueing a non-duplicate task. Tasks may have already been prepared and finalized. """ for app_label in self.initial_diff.deleted: self.queue_purge_app(app_label) def queue_purge_app(self, app_label): """Queue the purging of a Django app. Args: app_label (unicode): The label of the app to purge. Raises: django_evolution.errors.EvolutionTaskAlreadyQueuedError: A purge of this app was already queued. django_evolution.errors.QueueEvolverTaskError: Error queueing a non-duplicate task. Tasks may have already been prepared and finalized. """ try: self.queue_task(PurgeAppTask(evolver=self, app_label=app_label)) except EvolutionTaskAlreadyQueuedError: raise EvolutionTaskAlreadyQueuedError( _('"%s" is already being tracked for purging') % app_label) def queue_task(self, task): """Queue a task to run during evolution. This should only be directly called if working with custom tasks. Otherwise, use a more specific queue method. Args: task (BaseEvolutionTask): The task to queue. Raises: django_evolution.errors.EvolutionTaskAlreadyQueuedError: A purge of this app was already queued. django_evolution.errors.QueueEvolverTaskError: Error queueing a non-duplicate task. Tasks may have already been prepared and finalized. """ assert task.id if self._tasks_prepared: raise QueueEvolverTaskError( _('Evolution tasks have already been prepared. New tasks ' 'cannot be added.')) if task.id in self._tasks_by_id: raise EvolutionTaskAlreadyQueuedError( _('A task with ID "%s" is already queued.') % task.id) self._tasks_by_id[task.id] = task self._tasks_by_class.setdefault(type(task), []).append(task) def evolve(self): """Perform the evolution. This will run through all queued tasks and attempt to apply them in a database transaction, tracking each new batch of evolutions as the tasks finish. This can only be called once per evolver instance. Raises: django_evolution.errors.EvolutionException: Something went wrong during the evolution process. Details are in the error message. Note that a more specific exception may be raised. django_evolution.errors.EvolutionExecutionError: A specific evolution task failed. Details are in the error. """ if self.evolved: raise EvolutionException( _('Evolver.evolve() has already been run once. It cannot be ' 'run again.')) self._prepare_tasks() evolving.send(sender=self) try: new_evolutions = [] for task_cls, tasks in six.iteritems(self._tasks_by_class): # Perform the evolution for the app. This is responsible # for raising any exceptions. task_cls.execute_tasks(evolver=self, tasks=tasks) for task in tasks: new_evolutions += task.new_evolutions # Things may have changed, so rescan the database. self.database_state.rescan_tables() self._save_project_sig(new_evolutions=new_evolutions) self.evolved = True except Exception as e: evolving_failed.send(sender=self, exception=e) raise evolved.send(sender=self) def _prepare_tasks(self): """Prepare all queued tasks for further operations. Once prepared, no new tasks can be added. This will be done before performing any operations requiring state from queued tasks. """ if not self._tasks_prepared: self._tasks_prepared = True for task_cls, tasks in six.iteritems(self._tasks_by_class): task_cls.prepare_tasks(evolver=self, tasks=tasks, hinted=self.hinted) def sql_executor(self, **kwargs): """Return an SQLExecutor for executing SQL. This is a convenience method for creating an :py:class:`~django_evolution.utils.sql.SQLExecutor` to operate using the evolver's current database. Version Added: 2.1 Args: **kwargs (dict): Additional keyword arguments used to construct the executor. Returns: django_evolution.utils.sql.SQLExecutor: The new SQLExecutor. """ return SQLExecutor(database=self.database_name, **kwargs) @contextmanager def transaction(self): """Execute database operations in a transaction. This is a convenience method for executing in a transaction using the evolver's current database. Deprecated: 2.1: This has been replaced with manual calls to :py:class:`~django_evolution.utils.sql.SQLExecutor`. Context: django.db.backends.util.CursorWrapper: The cursor used to execute statements. """ with atomic(using=self.database_name): cursor = self.connection.cursor() try: yield cursor finally: cursor.close() def _save_project_sig(self, new_evolutions): """Save the project signature and any new evolutions. This will serialize the current modified project signature to the database and write any new evolutions, attaching them to the current project version. This can be called many times for one evolver instance. After the first time, the version already saved will simply be updated. Args: new_evolutions (list of django_evolution.models.Evolution): The list of new evolutions to save to the database. Raises: django_evolution.errors.EvolutionExecutionError: There was an error saving to the database. """ version = self.version if version is None: version = Version(signature=self.project_sig) self.version = version try: version.save(using=self.database_name) if new_evolutions: for evolution in new_evolutions: evolution.version = version Evolution.objects.using( self.database_name).bulk_create(new_evolutions) except Exception as e: raise EvolutionExecutionError( _('Error saving new evolution version information: %s') % e, detailed_error=six.text_type(e))
def _on_app_models_updated(app, verbosity=1, using=DEFAULT_DB_ALIAS, **kwargs): """Handler for when an app's models were updated. This is called in response to a syncdb or migrate operation for an app. It will install baselines for any new models, record the changes in the evolution history, and notify the user if any of the changes require an evolution. Args: app (module): The app models module that was updated. verbosity (int, optional): The verbosity used to control output. This will have been provided by the syncdb or migrate command. using (str, optional): The database being updated. **kwargs (dict): Additional keyword arguments provided by the signal handler for the syncdb or migrate operation. """ project_sig = ProjectSignature.from_database(using) try: latest_version = Version.objects.current_version(using=using) except Version.DoesNotExist: # We need to create a baseline version. if verbosity > 0: print("Installing baseline version") latest_version = Version(signature=project_sig) latest_version.save(using=using) for a in get_apps(): _install_baseline(app=a, latest_version=latest_version, using=using, verbosity=verbosity) unapplied = get_unapplied_evolutions(app, using) if unapplied: print(style.NOTICE('There are unapplied evolutions for %s.' % get_app_label(app))) # Evolutions are checked over the entire project, so we only need to check # once. We do this check when Django Evolutions itself is synchronized. if app is get_app('django_evolution'): old_project_sig = latest_version.signature # If any models or apps have been added, a baseline must be set # for those new models changed = False new_apps = [] for new_app_sig in project_sig.app_sigs: app_id = new_app_sig.app_id old_app_sig = old_project_sig.get_app_sig(app_id) if old_app_sig is None: # App has been added old_project_sig.add_app_sig(new_app_sig.clone()) new_apps.append(app_id) changed = True else: for new_model_sig in new_app_sig.model_sigs: model_name = new_model_sig.model_name old_model_sig = old_app_sig.get_model_sig(model_name) if old_model_sig is None: # Model has been added old_app_sig.add_model_sig( project_sig .get_app_sig(app_id) .get_model_sig(model_name) .clone()) changed = True if changed: if verbosity > 0: print("Adding baseline version for new models") latest_version = Version(signature=old_project_sig) latest_version.save(using=using) for app_name in new_apps: app = get_app(app_name, True) if app: _install_baseline(app=app, latest_version=latest_version, using=using, verbosity=verbosity) # TODO: Model introspection step goes here. # # If the current database state doesn't match the last # # saved signature (as reported by latest_version), # # then we need to update the Evolution table. # actual_sig = introspect_project_sig() # acutal = pickle.dumps(actual_sig) # if actual != latest_version.signature: # nudge = Version(signature=actual) # nudge.save() # latest_version = nudge diff = Diff(old_project_sig, project_sig) if not diff.is_empty(): print(style.NOTICE( 'Project signature has changed - an evolution is required')) if verbosity > 1: print(diff)
def _on_app_models_updated(app, verbosity=1, using=DEFAULT_DB_ALIAS, **kwargs): """Handler for when an app's models were updated. This is called in response to a syncdb or migrate operation for an app. It will install baselines for any new models, record the changes in the evolution history, and notify the user if any of the changes require an evolution. Args: app (module): The app models module that was updated. verbosity (int, optional): The verbosity used to control output. This will have been provided by the syncdb or migrate command. using (str, optional): The database being updated. **kwargs (dict): Additional keyword arguments provided by the signal handler for the syncdb or migrate operation. """ project_sig = ProjectSignature.from_database(using) try: latest_version = Version.objects.current_version(using=using) except Version.DoesNotExist: # We need to create a baseline version. if verbosity > 0: print("Installing baseline version") latest_version = Version(signature=project_sig) latest_version.save(using=using) for a in get_apps(): _install_baseline(app=a, latest_version=latest_version, using=using, verbosity=verbosity) unapplied = get_unapplied_evolutions(app, using) if unapplied: print( style.NOTICE('There are unapplied evolutions for %s.' % get_app_label(app))) # Evolutions are checked over the entire project, so we only need to check # once. We do this check when Django Evolutions itself is synchronized. if app is get_app('django_evolution'): old_project_sig = latest_version.signature # If any models or apps have been added, a baseline must be set # for those new models changed = False new_apps = [] for new_app_sig in project_sig.app_sigs: app_id = new_app_sig.app_id old_app_sig = old_project_sig.get_app_sig(app_id) if old_app_sig is None: # App has been added old_project_sig.add_app_sig(new_app_sig.clone()) new_apps.append(app_id) changed = True else: for new_model_sig in new_app_sig.model_sigs: model_name = new_model_sig.model_name old_model_sig = old_app_sig.get_model_sig(model_name) if old_model_sig is None: # Model has been added old_app_sig.add_model_sig( project_sig.get_app_sig(app_id).get_model_sig( model_name).clone()) changed = True if changed: if verbosity > 0: print("Adding baseline version for new models") latest_version = Version(signature=old_project_sig) latest_version.save(using=using) for app_name in new_apps: app = get_app(app_name, True) if app: _install_baseline(app=app, latest_version=latest_version, using=using, verbosity=verbosity) # TODO: Model introspection step goes here. # # If the current database state doesn't match the last # # saved signature (as reported by latest_version), # # then we need to update the Evolution table. # actual_sig = introspect_project_sig() # acutal = pickle.dumps(actual_sig) # if actual != latest_version.signature: # nudge = Version(signature=actual) # nudge.save() # latest_version = nudge diff = Diff(old_project_sig, project_sig) if not diff.is_empty(): print( style.NOTICE( 'Project signature has changed - an evolution is required') ) if verbosity > 1: print(diff)
def get_app_pending_mutations(app, evolution_labels=[], mutations=None, old_project_sig=None, project_sig=None, database=DEFAULT_DB_ALIAS): """Return an app's pending mutations provided by the given evolution names. This is similar to :py:meth:`get_app_mutations`, but filters the list of mutations down to remove any that are unnecessary (ones that do not operate on changed parts of the project signature). Args: app (module): The app the evolutions belong to. evolution_labels (list of unicode, optional): The labels of the evolutions to return mutations for. If ``None``, this will factor in all evolution labels for the app. mutations (list of django_evolution.mutations.BaseMutation, optional): An explicit list of mutations to use. If provided, ``evolution_labels`` will be ignored. old_project_sig (django_evolution.signature.ProjectSignature, optional): A pre-fetched old project signature. If provided, this will be used instead of the latest one in the database. project_sig (django_evolution.signature.ProjectSignature, optional): A project signature representing the current state of the database. If provided, this will be used instead of generating a new one from the current database state. database (unicode, optional): The name of the database the evolutions cover. Returns: list of django_evolution.mutations.BaseMutation: The list of mutations provided by the evolutions. Raises: django_evolution.errors.EvolutionException: One or more evolutions are missing. """ # Avoids a nasty circular import. Util modules should always be # importable, so we compensate here. from django_evolution.models import Version from django_evolution.mutations import RenameModel from django_evolution.signature import ProjectSignature if mutations is None: mutations = get_app_mutations(app=app, evolution_labels=evolution_labels, database=database) if old_project_sig is None: latest_version = Version.objects.current_version(using=database) old_project_sig = latest_version.signature if project_sig is None: project_sig = ProjectSignature.from_database(database) app_id = get_app_label(app) old_app_sig = old_project_sig.get_app_sig(app_id) app_sig = project_sig.get_app_sig(app_id) if old_app_sig is not None and app_sig is not None: # We want to go through now and make sure we're only applying # evolutions for models where the signature is different between # what's stored and what's current. # # The reason for this is that we may have just installed a baseline, # which would have the up-to-date signature, and we might be trying # to apply evolutions on top of that (which would already be applied). # Or we may be working with models that weren't present in the old # signature and will soon be added. In either case, these would # generate errors. So, try hard to prevent that. # # First, Find the list of models in the latest signature of this app # that aren't in the old signature. If a model signature isn't found # in the old app signature, it's a new model, and we don't want to # try to apply evolutions to it. changed_models = set( model_sig.model_name for model_sig in app_sig.model_sigs if old_app_sig.get_model_sig(model_sig.model_name) not in ( None, model_sig)) # Now do the same for models in the old signature, in case the # model has been deleted. changed_models.update( old_model_sig.model_name for old_model_sig in old_app_sig.model_sigs if app_sig.get_model_sig(old_model_sig.model_name) is None) # We should now have a full list of which models changed. Filter # the list of mutations appropriately. # # Changes affecting a model that was newly-introduced are removed, # unless the mutation is a RenameModel, in which case we'll need it # during the optimization step (and will remove it if necessary then). mutations = [ mutation for mutation in mutations if (not hasattr(mutation, 'model_name') or mutation.model_name in changed_models or isinstance(mutation, RenameModel)) ] return mutations