def create_base_schema():
log.info('Creating base schema')
create_log.info("SCHEMA")
# Functions
functions = environ.find_resource('sql', 'functions.sql')
if db_settings.execute_sql(functions) != 0:
error(u'Failed to create functions')
# A Base schema shared between all RDBMS implementations
schema = _get_latest_schema()
if db_settings.execute_sql(schema) != 0:
error(u'Failed to create base schema')
try:
schema = environ.find_resource('sql',
'%s-schema.sql' % db_settings.rdbms)
if db_settings.execute_sql(schema) != 0:
error(u'Failed to create %s specific schema' %
(db_settings.rdbms, ))
except EnvironmentError:
pass
migration = StoqlibSchemaMigration()
migration.apply_all_patches()
def restore_database(user_hash, time=None):
assert user_hash
# If the database doesn't exist, get_default_store will fail
try:
default_store = get_default_store()
except Exception:
default_store = None
if default_store is not None and db_settings.has_database():
try:
default_store.lock_database()
except DatabaseError:
raise TaskException(
"Could not lock database. This means that there are other "
"clients connected. Make sure to close every Stoq client "
"before updating the database")
except Exception:
raise TaskException(
"Database is empty or in a corrupted state. Fix or drop it "
"before trying to proceed with the restore")
else:
default_store.unlock_database()
# FIXME: Windows will not liberate resource for other process to
# write to the file. We should write our own TemporaryFile on Stoq
# that handles all those cases for us and use here
with tempfile.NamedTemporaryFile(delete=False) as f:
pass
try:
if not db_settings.dump_database(f.name):
raise TaskException("Failed to dump the database")
backup_name = db_settings.restore_database(f.name)
logger.info("Created a backup of the current database state on %s",
backup_name)
finally:
os.unlink(f.name)
tmp_path = tempfile.mkdtemp()
try:
restore_path = os.path.join(tmp_path, 'stoq')
backup.restore(restore_path, user_hash, time=time)
# None will make the default store be closed, which we need
# to sucessfully restore the database
set_default_store(None)
db_settings.clean_database(db_settings.dbname, force=True)
db_settings.execute_sql(os.path.join(restore_path, 'stoq.dump'),
lock_database=True)
logger.info("Backup restore finished sucessfully")
finally:
# get_default_store will recreate it (since we closed it above)
get_default_store()
shutil.rmtree(tmp_path, ignore_errors=True)
def restore_database(user_hash, time=None):
assert user_hash
# If the database doesn't exist, get_default_store will fail
try:
default_store = get_default_store()
except Exception:
default_store = None
if default_store is not None and db_settings.has_database():
try:
default_store.lock_database()
except DatabaseError:
raise TaskException(
"Could not lock database. This means that there are other "
"clients connected. Make sure to close every Stoq client "
"before updating the database")
except Exception:
raise TaskException(
"Database is empty or in a corrupted state. Fix or drop it "
"before trying to proceed with the restore")
else:
default_store.unlock_database()
# FIXME: Windows will not liberate resource for other process to
# write to the file. We should write our own TemporaryFile on Stoq
# that handles all those cases for us and use here
with tempfile.NamedTemporaryFile(delete=False) as f:
pass
try:
if not db_settings.dump_database(f.name):
raise TaskException("Failed to dump the database")
backup_name = db_settings.restore_database(f.name)
logger.info("Created a backup of the current database state on %s",
backup_name)
finally:
os.unlink(f.name)
tmp_path = tempfile.mkdtemp()
try:
restore_path = os.path.join(tmp_path, 'stoq')
logger.info("restoring database to %s", restore_path)
backup.restore(restore_path, user_hash, time=time)
# None will make the default store be closed, which we need
# to sucessfully restore the database
set_default_store(None)
db_settings.clean_database(db_settings.dbname, force=True)
db_settings.execute_sql(os.path.join(restore_path, 'stoq.dump'),
lock_database=True)
logger.info("Backup restore finished sucessfully")
finally:
# get_default_store will recreate it (since we closed it above)
get_default_store()
def restore_database(user_hash, time=None):
assert user_hash
# If the database doesn't exist, get_default_store will fail
try:
default_store = get_default_store()
except Exception:
default_store = None
if default_store is not None and db_settings.has_database():
try:
default_store.lock_database()
except DatabaseError:
raise TaskException(
"Could not lock database. This means that there are other "
"clients connected. Make sure to close every Stoq client "
"before updating the database")
except Exception:
raise TaskException(
"Database is empty or in a corrupted state. Fix or drop it "
"before trying to proceed with the restore")
else:
default_store.unlock_database()
with tempfile.NamedTemporaryFile() as f:
if not db_settings.dump_database(f.name):
raise TaskException("Failed to dump the database")
backup_name = db_settings.restore_database(f.name)
logger.info("Created a backup of the current database state on %s",
backup_name)
tmp_path = tempfile.mkdtemp()
try:
# None will make the default store be closed, which we need
# to sucessfully restore the database
set_default_store(None)
restore_path = os.path.join(tmp_path, 'stoq')
backup.restore(restore_path, user_hash, time=time)
db_settings.clean_database(db_settings.dbname, force=True)
db_settings.execute_sql(os.path.join(restore_path, 'stoq.dump'),
lock_database=True)
logger.info("Backup restore finished sucessfully")
finally:
# get_default_store will recreate it (since we closed it above)
get_default_store()
shutil.rmtree(tmp_path, ignore_errors=True)
def create_database_functions():
"""Create some functions we define on the database
This will simply read data/sql/functions.sql and execute it
"""
with tempfile.NamedTemporaryFile(suffix='stoqfunctions-') as tmp_f:
functions = environ.get_resource_string('stoq', 'sql', 'functions.sql')
tmp_f.write(render_template_string(functions))
tmp_f.flush()
if db_settings.execute_sql(tmp_f.name) != 0:
error(u'Failed to create functions')
def create_base_schema():
log.info('Creating base schema')
create_log.info("SCHEMA")
create_database_functions()
# A Base schema shared between all RDBMS implementations
schema = _get_latest_schema()
if db_settings.execute_sql(schema) != 0:
error(u'Failed to create base schema')
try:
schema = environ.find_resource('sql', '%s-schema.sql' % db_settings.rdbms)
if db_settings.execute_sql(schema) != 0:
error(u'Failed to create %s specific schema' % (db_settings.rdbms, ))
except EnvironmentError:
pass
migration = StoqlibSchemaMigration()
migration.apply_all_patches()
def create_database_functions():
"""Create some functions we define on the database
This will simply read data/sql/functions.sql and execute it
"""
with tempfile.NamedTemporaryFile(suffix='stoqfunctions-') as tmp_f:
with open(environ.find_resource('sql', 'functions.sql')) as f:
tmp_f.write(render_template_string(f.read()))
tmp_f.flush()
if db_settings.execute_sql(tmp_f.name) != 0:
error(u'Failed to create functions')
def create_database_functions():
"""Create some functions we define on the database
This will simply read data/sql/functions.sql and execute it
"""
# We cant remove the file, otherwise it will fail on windows.
with tempfile.NamedTemporaryFile(prefix='stoqfunctions-', delete=False) as tmp_f:
functions = pkg_resources.resource_string('stoq', 'sql/functions.sql')
tmp_f.write(render_template_string(functions))
tmp_f.flush()
if db_settings.execute_sql(tmp_f.name) != 0:
error(u'Failed to create functions')
def create_database_functions():
"""Create some functions we define on the database
This will simply read data/sql/functions.sql and execute it
"""
# We cant remove the file, otherwise it will fail on windows.
with tempfile.NamedTemporaryFile(prefix='stoqfunctions-', delete=False) as tmp_f:
functions = environ.get_resource_string('stoq', 'sql', 'functions.sql')
tmp_f.write(render_template_string(functions))
tmp_f.flush()
if db_settings.execute_sql(tmp_f.name) != 0:
error(u'Failed to create functions')
def populate_initial_data(store):
from stoqlib.domain.system import SystemTable
generation = store.find(SystemTable).max(SystemTable.generation)
if generation < 4:
# FIXME: Initial data can (and needs to) only be sourced on schemas
# greater or equal than 4. Remove this in the future.
return
log.info('Populating initial data')
initial_data = pkg_resources.resource_filename('stoq', 'sql/initial.sql')
if db_settings.execute_sql(initial_data) != 0:
error(u'Failed to populate initial data')
def populate_initial_data(store):
from stoqlib.domain.system import SystemTable
generation = store.find(SystemTable).max(SystemTable.generation)
if generation < 4:
# FIXME: Initial data can (and needs to) only be sourced on schemas
# greater or equal than 4. Remove this in the future.
return
log.info('Populating initial data')
initial_data = environ.find_resource('sql', 'initial.sql')
if db_settings.execute_sql(initial_data) != 0:
error(u'Failed to populate initial data')
def apply(self, store):
"""Apply the patch
:param store: a store
"""
# Dont lock the database here, since StoqlibSchemaMigration.update has
# already did that before starting to apply the patches
# SQL statement to update the system_table
sql = self._migration.generate_sql_for_patch(self)
if self.filename.endswith('.sql'):
# Create a temporary file used for writing SQL statements
temporary = tempfile.mktemp(prefix="patch-%d-%d-" %
self.get_version())
# Overwrite the temporary file with the sql patch we want to apply
shutil.copy(self.filename, temporary)
# After successfully executing the SQL statements, we need to
# make sure that the system_table is updated with the correct
# schema generation and patchlevel
open(temporary, 'a').write(sql)
retcode = db_settings.execute_sql(temporary)
if retcode != 0:
error('Failed to apply %s, psql returned error code: %d' %
(os.path.basename(self.filename), retcode))
os.unlink(temporary)
elif self.filename.endswith('.py'):
# Execute the patch, we cannot use __import__() since there are
# hyphens in the filename and data/sql lacks an __init__.py
ns = {}
exec(compile(open(self.filename).read(), self.filename, 'exec'),
ns, ns)
function = ns['apply_patch']
# Create a new store that will be used to apply the patch and
# to update the system tables after the patch has been successfully
# applied
patch_store = new_store()
# Apply the patch itself
function(patch_store)
# After applying the patch, update the system_table within the same
# transaction
patch_store.execute(sql)
patch_store.commit(close=True)
else:
raise AssertionError("Unknown filename: %s" % (self.filename, ))
def create_base_schema():
log.info('Creating base schema')
create_log.info("SCHEMA")
create_database_functions()
# A Base schema shared between all RDBMS implementations
schema = _get_latest_schema()
if db_settings.execute_sql(schema) != 0:
error(u'Failed to create base schema')
migration = StoqlibSchemaMigration()
migration.apply_all_patches()
def create_base_schema():
log.info('Creating base schema')
create_log.info("SCHEMA")
create_database_functions()
# A Base schema shared between all RDBMS implementations
schema = _get_latest_schema()
if db_settings.execute_sql(schema) != 0:
error(u'Failed to create base schema')
migration = StoqlibSchemaMigration()
migration.apply_all_patches()
def apply(self, store):
"""Apply the patch
:param store: a store
"""
# Dont lock the database here, since StoqlibSchemaMigration.update has
# already did that before starting to apply the patches
# SQL statement to update the system_table
sql = self._migration.generate_sql_for_patch(self)
if self.filename.endswith('.sql'):
# Create a temporary file used for writing SQL statements
temporary = tempfile.mktemp(prefix="patch-%d-%d-" % self.get_version())
# Overwrite the temporary file with the sql patch we want to apply
shutil.copy(self.filename, temporary)
# After successfully executing the SQL statements, we need to
# make sure that the system_table is updated with the correct
# schema generation and patchlevel
open(temporary, 'a').write(sql)
retcode = db_settings.execute_sql(temporary)
if retcode != 0:
error('Failed to apply %s, psql returned error code: %d' % (
os.path.basename(self.filename), retcode))
os.unlink(temporary)
elif self.filename.endswith('.py'):
# Execute the patch, we cannot use __import__() since there are
# hyphens in the filename and data/sql lacks an __init__.py
ns = {}
execfile(self.filename, ns, ns)
function = ns['apply_patch']
# Create a new store that will be used to apply the patch and
# to update the system tables after the patch has been successfully
# applied
patch_store = new_store()
# Apply the patch itself
function(patch_store)
# After applying the patch, update the system_table within the same
# transaction
patch_store.execute(sql)
patch_store.commit(close=True)
else:
raise AssertionError("Unknown filename: %s" % (self.filename, ))
def _update_schema(self):
"""Check the current version of database and update the schema if
it's needed
"""
log.info("Updating schema")
if self.check_uptodate():
log.info("Schema is already up to date")
return
patches = self._get_patches()
latest_available = patches[-1].get_version()
current_version = self.get_current_version()
last_level = None
if current_version != latest_available:
patches_to_apply = []
for patch in patches:
if patch.get_version() <= current_version:
continue
patches_to_apply.append(patch)
functions = environ.get_resource_filename('stoq', 'sql',
'functions.sql')
if db_settings.execute_sql(functions) != 0:
error('Failed to create functions')
log.info("Applying %d patches" % (len(patches_to_apply), ))
create_log.info("PATCHES:%d" % (len(patches_to_apply), ))
for patch in patches_to_apply:
create_log.info("PATCH:%d.%d" %
(patch.generation, patch.level))
patch.apply(self.default_store)
assert patches_to_apply
log.info("All patches (%s) applied." %
(', '.join(str(p.level) for p in patches_to_apply)))
last_level = patches_to_apply[-1].get_version()
self.after_update()
return current_version, last_level
def _update_schema(self):
"""Check the current version of database and update the schema if
it's needed
"""
log.info("Updating schema")
if self.check_uptodate():
log.info("Schema is already up to date")
return
patches = self._get_patches()
latest_available = patches[-1].get_version()
current_version = self.get_current_version()
last_level = None
if current_version != latest_available:
patches_to_apply = []
for patch in patches:
if patch.get_version() <= current_version:
continue
patches_to_apply.append(patch)
functions = environ.get_resource_filename('stoq', 'sql', 'functions.sql')
if db_settings.execute_sql(functions) != 0:
error('Failed to create functions')
log.info("Applying %d patches" % (len(patches_to_apply), ))
create_log.info("PATCHES:%d" % (len(patches_to_apply), ))
for patch in patches_to_apply:
create_log.info("PATCH:%d.%d" % (patch.generation,
patch.level))
patch.apply(self.default_store)
assert patches_to_apply
log.info("All patches (%s) applied." % (
', '.join(str(p.level) for p in patches_to_apply)))
last_level = patches_to_apply[-1].get_version()
self.after_update()
return current_version, last_level
