def save_schema_dump( server, database_schema_name, perform_checks=False, check_user=None): """Creates database schema (if needed) and schema version.""" if perform_checks: privileges_logic.UserPrivileges(check_user).check_save_schema_dump() conn_opts = {} conn_opts['host'] = server.hostname if server.port: conn_opts['port'] = server.port if settings.MYSQL_USER: conn_opts['user'] = settings.MYSQL_USER if settings.MYSQL_PASSWORD: conn_opts['passwd'] = settings.MYSQL_PASSWORD structure = mysql_functions.dump_schema(database_schema_name, **conn_opts) checksum = mysql_functions.generate_schema_hash(structure) database_schema, __ = ( models.DatabaseSchema.objects.get_or_create( name=database_schema_name)) schema_version, __ = ( models.SchemaVersion.objects.get_or_create( database_schema=database_schema, checksum=checksum)) schema_version.ddl=structure schema_version.checksum=checksum schema_version.pulled_from = server schema_version.pull_datetime = timezone.now() schema_version.save() return schema_version
def dump_schema(self, schema_name, connection_options=None): if connection_options is None: connection_options = { 'user': settings.MYSQL_USER, 'passwd': settings.MYSQL_PASSWORD } connection_options = connection_options.copy() connection_options.update({'host': self.hostname}) if self.port: connection_options['port'] = self.port return mysql_functions.dump_schema(schema_name, **connection_options)
def run(self): try: if models.ChangesetApply.objects.filter( changeset=self.changeset, server=self.server, success=True).exists(): raise exceptions.Error( "Changeset has been successfully applied already at " "server '%s'." % ( self.server.name,)) host_before_ddl = mysql_functions.dump_schema( **self.connection_options) host_before_checksum = mysql_functions.generate_schema_hash( host_before_ddl) schema_version_before_apply = None # # If changeset does not have a before_version yet, # then it is the first time the changeset is being applied, # ensure that db on host has a known schema version. # if not self.changeset.before_version: schema_version_qs = schemaversions_models.SchemaVersion.objects.filter( database_schema=self.changeset.database_schema, checksum=host_before_checksum) if not schema_version_qs.exists(): raise exceptions.Error('Schema version on host is unknown.') else: # Remember this version schema_version_before_apply = schema_version_qs[0] # # If not first time to apply, check that the host schema version # is the same as what the changeset expects. # elif self.changeset.before_version.checksum != host_before_checksum: before_version_checksum = self.changeset.before_version.checksum before_version_ddl = self.changeset.before_version.ddl before_version_ddl_lines = before_version_ddl.splitlines(True) current_ddl_lines = host_before_ddl.splitlines(True) delta = [ line for line in difflib.context_diff( before_version_ddl_lines, current_ddl_lines, fromfile='expected', tofile='actual')] msg = ( u"Cannot apply changeset, existing schema on host " u"does not match the expected schema.") raise exceptions.SchemaDoesNotMatchError( msg, before_version_ddl, host_before_ddl, ''.join(delta)) self.apply_changeset_details() host_after_ddl = mysql_functions.dump_schema( **self.connection_options) host_after_checksum = mysql_functions.generate_schema_hash( host_after_ddl) schema_version_after_apply = None # # If changeset is being applied for the first time, # record the schema versions -- before and after it is applied. # if not self.changeset.before_version: schema_version_after_apply, created = ( schemaversions_models.SchemaVersion.objects.get_or_create( database_schema=self.changeset.database_schema, checksum=host_after_checksum )) if created: schema_version_after_apply.ddl = host_after_ddl schema_version_after_apply.pulled_from = self.server schema_version_after_apply.pull_datetime = timezone.now() schema_version_after_apply.save() self.changeset.before_version = schema_version_before_apply self.changeset.after_version = schema_version_after_apply self.changeset.save() # # If not first time to apply, ensure that final schema version on # host is what the changeset expects. # elif self.changeset.after_version.checksum != host_after_checksum: after_version_checksum = self.changeset.after_version.checksum after_version_ddl = self.changeset.after_version.ddl after_version_ddl_lines = after_version_ddl.splitlines(True) current_ddl_lines = host_after_ddl.splitlines(True) delta = [ line for line in difflib.context_diff( after_version_ddl_lines, current_ddl_lines, fromfile='expected', tofile='actual')] msg = ( u"Final schema on host does not match the expected " u"schema." ) raise exceptions.SchemaDoesNotMatchError( msg, after_version_ddl, host_after_ddl, ''.join(delta)) changeset_action = changesets_models.ChangesetAction.objects.create( changeset=self.changeset, type=changesets_models.ChangesetAction.TYPE_APPLIED, timestamp=timezone.now()) changesets_models.ChangesetActionServerMap.objects.create( changeset_action=changeset_action, server=self.server) self.changeset_apply = models.ChangesetApply.objects.create( changeset=self.changeset, server=self.server, applied_at=timezone.now(), applied_by=self.applied_by, success=True, changeset_action=changeset_action, task_id=self.task_id) if not self.unit_testing: event_handlers.on_changeset_applied( self.changeset_apply, request=self.request) except exceptions.SchemaDoesNotMatchError, e: msg = 'ERROR %s: %s' % (type(e), e.message) log.exception(msg) extra = dict(delta=e.delta) self.store_message(msg, 'error', extra) self.has_errors = True try: changeset_action = changesets_models.ChangesetAction.objects.create( changeset=self.changeset, type=changesets_models.ChangesetAction.TYPE_APPLIED_FAILED, timestamp=timezone.now()) changesets_models.ChangesetActionServerMap.objects.create( changeset_action=changeset_action, server=self.server) self.changeset_apply = models.ChangesetApply.objects.create( changeset=self.changeset, server=self.server, applied_at=timezone.now(), applied_by=self.applied_by, results_log=u'%s\nSchema delta:\n%s' % (msg, e.delta), success=False, changeset_action=changeset_action, task_id=self.task_id) if not self.unit_testing: event_handlers.on_changeset_apply_failed( self.changeset_apply, request=self.request) except: log.exception('EXCEPTION') pass
def run_test(self): """Main logic for testing changeset.""" msg = ( 'Changeset syntax test started.\n' 'Testing against SchemaVersion: id=%s' % self.schema_version.pk) log.info(msg) self.store_message(msg) log.debug('Changeset: id=%s', self.changeset.pk) schema_name = self.changeset.database_schema.name conn = MySQLdb.connect(**self.connection_options) cursor = None try: cursor = conn.cursor() # # Create schema. # try: sql = 'DROP SCHEMA IF EXISTS %s' % schema_name log.debug('Executing: %s', sql) cursor.execute(sql) except Warning, e: log.warn('EXCEPTION %s: %s' % (type(e), e)) # ignore warnings pass while cursor.nextset() is not None: pass sql = 'CREATE SCHEMA %s' % schema_name log.debug('Executing: %s', sql) cursor.execute(sql) while cursor.nextset() is not None: pass msg = "Database schema '%s' was created." % (schema_name,) log.info(msg) self.store_message(msg) # # Connect to newly created schema. # sql = 'USE %s' % schema_name log.debug('Executing: %s', sql) cursor.execute(sql) dump_connection_options = self.connection_options.copy() dump_connection_options['db'] = schema_name # # Load initial schema. # log.debug( 'Loading initial schema:\n%s', self.schema_version.ddl) ddls = sqlparse.split(self.schema_version.ddl) for ddl in ddls: try: ddl = ddl.rstrip(unicode(string.whitespace + ';')) if ddl: cursor.execute(ddl) finally: while cursor.nextset() is not None: pass # # Delete existing results. # changesettests_models.ChangesetTest.objects.filter( changeset_detail__changeset=self.changeset).delete() log.debug('Existing test results deleted.') # # Apply all changeset details. # structure_after = '' hash_after = '' first_run = True for changeset_detail in ( self.changeset.changesetdetail_set.select_related() .order_by('id')): msg = u'Testing ChangesetDetail: id=%s' % changeset_detail.pk log.info(msg) self.store_message(msg) if first_run: log.debug('first_run=%s', first_run) # # Initial before structure and checksum should be the # same with schema version. # structure_before = self.schema_version.ddl hash_before = self.schema_version.checksum structure_after = structure_before hash_after = hash_before first_run = False else: # # For succeeding runs, before structure and checksum # is equal to after structure and checksum of the # preceeding operation. # structure_before = structure_after hash_before = hash_after structure_after = structure_before hash_after = hash_before # # Track final structure after applying changes. # self.structure_after = structure_after self.hash_after = hash_after started_at = timezone.now() results_log_items = [] try: # # Execute apply_sql # msg = u'Executing apply_sql:\n%s' % changeset_detail.apply_sql log.info(msg) self.store_message(msg) self.execute_query(cursor, changeset_detail.apply_sql) cursor.execute('FLUSH TABLES') # # Track resulting schema after executing apply_sql. # structure_after = mysql_functions.dump_schema( **dump_connection_options) hash_after = mysql_functions.generate_schema_hash( structure_after) log.debug( 'Resulting schema:\nStructure=\n%s\nChecksum=%s', structure_after, hash_after) # # Execute apply_verification_sql # try: if changeset_detail.apply_verification_sql: log.debug( 'Executing apply_verification_sql:\n%s', changeset_detail.apply_verification_sql) self.execute_query( cursor, changeset_detail.apply_verification_sql) except Exception, e: msg = ( u'Apply verification failed (Error %s: %s).' % ( type(e), e)) raise exceptions.Error(msg) # # Execute revert_sql # log.debug( u'Executing revert_sql:\n%s', changeset_detail.revert_sql) self.execute_query(cursor, changeset_detail.revert_sql) cursor.execute('FLUSH TABLES') # # Check if revert_sql has resulted to the original # schema before apply_sql was applied. # structure_after_revert = mysql_functions.dump_schema( **dump_connection_options) hash_after_revert = mysql_functions.generate_schema_hash( structure_after_revert) if hash_after_revert != hash_before: raise exceptions.Error( 'Checksum after revert_sql was applied was ' 'not the same as before apply_sql was ' 'applied.') # # Execute revert_verification_sql # try: if changeset_detail.revert_verification_sql: log.debug( 'Executing revert_verification_sql: \n%s', changeset_detail.revert_verification_sql) self.execute_query( cursor, changeset_detail.revert_verification_sql) except Exception, e: msg = ( u'Revert verification failed (Error %s: %s).' % ( type(e), e)) raise exceptions.Error(msg) # # revert_sql worked, finalize change by reapplying # apply_sql # log.debug('Reapplying apply_sql.') self.execute_query(cursor, changeset_detail.apply_sql) # # Update changeset_detail with info on schema versions. # changeset_detail.before_checksum = hash_before changeset_detail.after_checksum = hash_after changeset_detail.save() # # Remember the last schema changes # self.structure_after = structure_after self.hash_after = hash_after