def generate_schema_version(server, schema_name, connection_options=None):
if connection_options is None:
connection_options = {}
schema_dump = server.dump_schema(schema_name, connection_options)
#
# Save dump as latest version for the schema
#
database_schema, __ = models.DatabaseSchema.objects.get_or_create(
name=schema_name)
checksum = mysql_functions.generate_schema_hash(schema_dump)
schema_version_created = False
try:
schema_version = models.SchemaVersion.objects.get(
database_schema=database_schema,
checksum=checksum)
schema_version.ddl = schema_dump
schema_version.pulled_from = server
schema_version.pull_datetime = timezone.now()
schema_version.save()
except ObjectDoesNotExist:
schema_version = models.SchemaVersion.objects.create(
database_schema=database_schema,
ddl=schema_dump,
checksum=mysql_functions.generate_schema_hash(schema_dump),
pulled_from=server,
pull_datetime=timezone.now())
schema_version_created = True
return (schema_version, schema_version_created)
def save_schema_dump(
server, database_schema_name, perform_checks=False, check_user=None):
"""Creates database schema (if needed) and schema version."""
if perform_checks:
privileges_logic.UserPrivileges(check_user).check_save_schema_dump()
conn_opts = {}
conn_opts['host'] = server.hostname
if server.port:
conn_opts['port'] = server.port
if settings.MYSQL_USER:
conn_opts['user'] = settings.MYSQL_USER
if settings.MYSQL_PASSWORD:
conn_opts['passwd'] = settings.MYSQL_PASSWORD
structure = mysql_functions.dump_schema(database_schema_name, **conn_opts)
checksum = mysql_functions.generate_schema_hash(structure)
database_schema, __ = (
models.DatabaseSchema.objects.get_or_create(
name=database_schema_name))
schema_version, __ = (
models.SchemaVersion.objects.get_or_create(
database_schema=database_schema,
checksum=checksum))
schema_version.ddl=structure
schema_version.checksum=checksum
schema_version.pulled_from = server
schema_version.pull_datetime = timezone.now()
schema_version.save()
return schema_version
def generate_server_data(self, servers, connection_options=None):
if connection_options is None:
connection_options = {
'user': settings.MYSQL_USER,
'passwd': settings.MYSQL_PASSWORD
}
for server in servers:
schema_exists = server.schema_exists(
self.name, connection_options)
if schema_exists:
schema_dump = server.dump_schema(self.name, connection_options)
else:
schema_dump = ''
schema_hash = mysql_functions.generate_schema_hash(schema_dump)
schema_version = None
try:
if schema_exists:
schema_version = SchemaVersion.objects.get(
database_schema=self, checksum=schema_hash)
except ObjectDoesNotExist:
pass
schema_version_diff = ''
if schema_version is None:
# get different of host schema from latest schema version
latest_schema_version = self.get_latest_schema_version()
latest_schema_version_ddl = ''
if latest_schema_version:
latest_schema_version_ddl = latest_schema_version.ddl
schema_version_diff = helpers.generate_delta(
latest_schema_version_ddl,
schema_dump,
fromfile='saved version', tofile='host version')
server_data, created = servers_models.ServerData.objects.get_or_create(
server=server, database_schema=self,
defaults={
'schema_exists': schema_exists,
'schema_version': schema_version,
'schema_version_diff': schema_version_diff
})
if not created:
server_data.schema_exists = schema_exists
server_data.schema_version = schema_version
server_data.schema_version_diff = schema_version_diff
server_data.save()
def run(self):
try:
if models.ChangesetApply.objects.filter(
changeset=self.changeset, server=self.server,
success=True).exists():
raise exceptions.Error(
"Changeset has been successfully applied already at "
"server '%s'." % (
self.server.name,))
host_before_ddl = mysql_functions.dump_schema(
**self.connection_options)
host_before_checksum = mysql_functions.generate_schema_hash(
host_before_ddl)
schema_version_before_apply = None
#
# If changeset does not have a before_version yet,
# then it is the first time the changeset is being applied,
# ensure that db on host has a known schema version.
#
if not self.changeset.before_version:
schema_version_qs = schemaversions_models.SchemaVersion.objects.filter(
database_schema=self.changeset.database_schema,
checksum=host_before_checksum)
if not schema_version_qs.exists():
raise exceptions.Error('Schema version on host is unknown.')
else:
# Remember this version
schema_version_before_apply = schema_version_qs[0]
#
# If not first time to apply, check that the host schema version
# is the same as what the changeset expects.
#
elif self.changeset.before_version.checksum != host_before_checksum:
before_version_checksum = self.changeset.before_version.checksum
before_version_ddl = self.changeset.before_version.ddl
before_version_ddl_lines = before_version_ddl.splitlines(True)
current_ddl_lines = host_before_ddl.splitlines(True)
delta = [
line for line in difflib.context_diff(
before_version_ddl_lines, current_ddl_lines,
fromfile='expected', tofile='actual')]
msg = (
u"Cannot apply changeset, existing schema on host "
u"does not match the expected schema.")
raise exceptions.SchemaDoesNotMatchError(
msg, before_version_ddl, host_before_ddl, ''.join(delta))
self.apply_changeset_details()
host_after_ddl = mysql_functions.dump_schema(
**self.connection_options)
host_after_checksum = mysql_functions.generate_schema_hash(
host_after_ddl)
schema_version_after_apply = None
#
# If changeset is being applied for the first time,
# record the schema versions -- before and after it is applied.
#
if not self.changeset.before_version:
schema_version_after_apply, created = (
schemaversions_models.SchemaVersion.objects.get_or_create(
database_schema=self.changeset.database_schema,
checksum=host_after_checksum
))
if created:
schema_version_after_apply.ddl = host_after_ddl
schema_version_after_apply.pulled_from = self.server
schema_version_after_apply.pull_datetime = timezone.now()
schema_version_after_apply.save()
self.changeset.before_version = schema_version_before_apply
self.changeset.after_version = schema_version_after_apply
self.changeset.save()
#
# If not first time to apply, ensure that final schema version on
# host is what the changeset expects.
#
elif self.changeset.after_version.checksum != host_after_checksum:
after_version_checksum = self.changeset.after_version.checksum
after_version_ddl = self.changeset.after_version.ddl
after_version_ddl_lines = after_version_ddl.splitlines(True)
current_ddl_lines = host_after_ddl.splitlines(True)
delta = [
line for line in difflib.context_diff(
after_version_ddl_lines, current_ddl_lines,
fromfile='expected', tofile='actual')]
msg = (
u"Final schema on host does not match the expected "
u"schema."
)
raise exceptions.SchemaDoesNotMatchError(
msg, after_version_ddl, host_after_ddl, ''.join(delta))
changeset_action = changesets_models.ChangesetAction.objects.create(
changeset=self.changeset,
type=changesets_models.ChangesetAction.TYPE_APPLIED,
timestamp=timezone.now())
changesets_models.ChangesetActionServerMap.objects.create(
changeset_action=changeset_action, server=self.server)
self.changeset_apply = models.ChangesetApply.objects.create(
changeset=self.changeset, server=self.server,
applied_at=timezone.now(), applied_by=self.applied_by,
success=True,
changeset_action=changeset_action,
task_id=self.task_id)
if not self.unit_testing:
event_handlers.on_changeset_applied(
self.changeset_apply, request=self.request)
except exceptions.SchemaDoesNotMatchError, e:
msg = 'ERROR %s: %s' % (type(e), e.message)
log.exception(msg)
extra = dict(delta=e.delta)
self.store_message(msg, 'error', extra)
self.has_errors = True
try:
changeset_action = changesets_models.ChangesetAction.objects.create(
changeset=self.changeset,
type=changesets_models.ChangesetAction.TYPE_APPLIED_FAILED,
timestamp=timezone.now())
changesets_models.ChangesetActionServerMap.objects.create(
changeset_action=changeset_action, server=self.server)
self.changeset_apply = models.ChangesetApply.objects.create(
changeset=self.changeset, server=self.server,
applied_at=timezone.now(), applied_by=self.applied_by,
results_log=u'%s\nSchema delta:\n%s' % (msg, e.delta),
success=False,
changeset_action=changeset_action,
task_id=self.task_id)
if not self.unit_testing:
event_handlers.on_changeset_apply_failed(
self.changeset_apply, request=self.request)
except:
log.exception('EXCEPTION')
pass
def run_test(self):
"""Main logic for testing changeset."""
msg = (
'Changeset syntax test started.\n'
'Testing against SchemaVersion: id=%s' % self.schema_version.pk)
log.info(msg)
self.store_message(msg)
log.debug('Changeset: id=%s', self.changeset.pk)
schema_name = self.changeset.database_schema.name
conn = MySQLdb.connect(**self.connection_options)
cursor = None
try:
cursor = conn.cursor()
#
# Create schema.
#
try:
sql = 'DROP SCHEMA IF EXISTS %s' % schema_name
log.debug('Executing: %s', sql)
cursor.execute(sql)
except Warning, e:
log.warn('EXCEPTION %s: %s' % (type(e), e))
# ignore warnings
pass
while cursor.nextset() is not None:
pass
sql = 'CREATE SCHEMA %s' % schema_name
log.debug('Executing: %s', sql)
cursor.execute(sql)
while cursor.nextset() is not None:
pass
msg = "Database schema '%s' was created." % (schema_name,)
log.info(msg)
self.store_message(msg)
#
# Connect to newly created schema.
#
sql = 'USE %s' % schema_name
log.debug('Executing: %s', sql)
cursor.execute(sql)
dump_connection_options = self.connection_options.copy()
dump_connection_options['db'] = schema_name
#
# Load initial schema.
#
log.debug(
'Loading initial schema:\n%s',
self.schema_version.ddl)
ddls = sqlparse.split(self.schema_version.ddl)
for ddl in ddls:
try:
ddl = ddl.rstrip(unicode(string.whitespace + ';'))
if ddl:
cursor.execute(ddl)
finally:
while cursor.nextset() is not None:
pass
#
# Delete existing results.
#
changesettests_models.ChangesetTest.objects.filter(
changeset_detail__changeset=self.changeset).delete()
log.debug('Existing test results deleted.')
#
# Apply all changeset details.
#
structure_after = ''
hash_after = ''
first_run = True
for changeset_detail in (
self.changeset.changesetdetail_set.select_related()
.order_by('id')):
msg = u'Testing ChangesetDetail: id=%s' % changeset_detail.pk
log.info(msg)
self.store_message(msg)
if first_run:
log.debug('first_run=%s', first_run)
#
# Initial before structure and checksum should be the
# same with schema version.
#
structure_before = self.schema_version.ddl
hash_before = self.schema_version.checksum
structure_after = structure_before
hash_after = hash_before
first_run = False
else:
#
# For succeeding runs, before structure and checksum
# is equal to after structure and checksum of the
# preceeding operation.
#
structure_before = structure_after
hash_before = hash_after
structure_after = structure_before
hash_after = hash_before
#
# Track final structure after applying changes.
#
self.structure_after = structure_after
self.hash_after = hash_after
started_at = timezone.now()
results_log_items = []
try:
#
# Execute apply_sql
#
msg = u'Executing apply_sql:\n%s' % changeset_detail.apply_sql
log.info(msg)
self.store_message(msg)
self.execute_query(cursor, changeset_detail.apply_sql)
cursor.execute('FLUSH TABLES')
#
# Track resulting schema after executing apply_sql.
#
structure_after = mysql_functions.dump_schema(
**dump_connection_options)
hash_after = mysql_functions.generate_schema_hash(
structure_after)
log.debug(
'Resulting schema:\nStructure=\n%s\nChecksum=%s',
structure_after, hash_after)
#
# Execute apply_verification_sql
#
try:
if changeset_detail.apply_verification_sql:
log.debug(
'Executing apply_verification_sql:\n%s',
changeset_detail.apply_verification_sql)
self.execute_query(
cursor,
changeset_detail.apply_verification_sql)
except Exception, e:
msg = (
u'Apply verification failed (Error %s: %s).' % (
type(e), e))
raise exceptions.Error(msg)
#
# Execute revert_sql
#
log.debug(
u'Executing revert_sql:\n%s',
changeset_detail.revert_sql)
self.execute_query(cursor, changeset_detail.revert_sql)
cursor.execute('FLUSH TABLES')
#
# Check if revert_sql has resulted to the original
# schema before apply_sql was applied.
#
structure_after_revert = mysql_functions.dump_schema(
**dump_connection_options)
hash_after_revert = mysql_functions.generate_schema_hash(
structure_after_revert)
if hash_after_revert != hash_before:
raise exceptions.Error(
'Checksum after revert_sql was applied was '
'not the same as before apply_sql was '
'applied.')
#
# Execute revert_verification_sql
#
try:
if changeset_detail.revert_verification_sql:
log.debug(
'Executing revert_verification_sql: \n%s',
changeset_detail.revert_verification_sql)
self.execute_query(
cursor,
changeset_detail.revert_verification_sql)
except Exception, e:
msg = (
u'Revert verification failed (Error %s: %s).' % (
type(e), e))
raise exceptions.Error(msg)
#
# revert_sql worked, finalize change by reapplying
# apply_sql
#
log.debug('Reapplying apply_sql.')
self.execute_query(cursor, changeset_detail.apply_sql)
#
# Update changeset_detail with info on schema versions.
#
changeset_detail.before_checksum = hash_before
changeset_detail.after_checksum = hash_after
changeset_detail.save()
#
# Remember the last schema changes
#
self.structure_after = structure_after
self.hash_after = hash_after
