class Config(object):
def __repr__(self):
return str(self.__config)
def __init__(self, config_file="simple-db-migrate.conf"):
self.__cli = CLI()
self.__config = {}
# read configurations
try:
f = codecs.open(config_file, "r", "utf-8")
exec(f.read())
except IOError:
self.__cli.error_and_exit("%s: file not found" % config_file)
else:
f.close()
try:
self.put("db_host", HOST)
self.put("db_user", USERNAME)
self.put("db_password", PASSWORD)
self.put("db_name", DATABASE)
self.put("db_version_table", "__db_version__")
migrations_dir = self.__get_migrations_absolute_dir(config_file, MIGRATIONS_DIR)
self.put("migrations_dir", migrations_dir)
except NameError, e:
self.__cli.error_and_exit("config file error: " + str(e))
class MSSQL(object):
def __init__(self, config=None, mssql_driver=_mssql):
self.__cli = CLI()
self.__mssql_driver = mssql_driver
self.__mssql_host = config.get("db_host")
self.__mssql_user = config.get("db_user")
self.__mssql_passwd = config.get("db_password")
self.__mssql_db = config.get("db_name")
self.__version_table = config.get("db_version_table")
if config.get("drop_db_first"):
self._drop_database()
self._create_database_if_not_exists()
self._create_version_table_if_not_exists()
def __mssql_connect(self, connect_using_db_name=True):
try:
conn = self.__mssql_driver.connect(server=self.__mssql_host, user=self.__mssql_user, password=self.__mssql_passwd)
if connect_using_db_name:
conn.select_db(self.__mssql_db)
return conn
except Exception, e:
self.__cli.error_and_exit("could not connect to database (%s)" % e)
class MySQL(object):
def __init__(self, config=None, mysql_driver=MySQLdb):
self.__cli = CLI()
self.__mysql_driver = mysql_driver
self.__mysql_host = config.get("db_host")
self.__mysql_user = config.get("db_user")
self.__mysql_passwd = config.get("db_password")
self.__mysql_db = config.get("db_name")
self.__version_table = config.get("db_version_table")
if config.get("drop_db_first"):
self._drop_database()
self._create_database_if_not_exists()
self._create_version_table_if_not_exists()
def __mysql_connect(self, connect_using_db_name=True):
try:
conn = self.__mysql_driver.connect(host=self.__mysql_host, user=self.__mysql_user, passwd=self.__mysql_passwd)
# this should be configured in the config file, not hardcoded
conn.set_character_set('utf8')
if connect_using_db_name:
conn.select_db(self.__mysql_db)
return conn
except Exception, e:
self.__cli.error_and_exit("could not connect to database (%s)" % e)
def _migrate(self):
""" Execute migrations based on git tags """
source = 'git'
current_ontology = None
current_version, origen = self.virtuoso.get_current_version()
# Making the first migration to the database
if current_version is None:
if self.config.get("file_migration", None) is not None:
self._execution_log(("- Current version is: %s" %
current_version),
"GREEN",
log_level_limit=1)
self._execution_log(("- Destination version is: %s" %
self.config.get("file_migration")),
"GREEN",
log_level_limit=1)
CLI.error_and_exit("Can't execute migration FROM None TO File "
"(TIP: version it using git --tag and then "
"use -m)")
else:
if origen == "file":
if self.config.get("file_migration", None) is not None:
self._execution_log(("- Current version is: %s" %
current_version),
"GREEN",
log_level_limit=1)
self._execution_log(("- Destination version is: %s" %
self.config.get("file_migration")),
"GREEN",
log_level_limit=1)
CLI.error_and_exit("Can't execute migration FROM File TO "
"File (TIP: version it using git --tag "
"and then use -m)")
current_ontology = self.virtuoso.get_ontology_by_version(
current_version)
if self.config.get("file_migration", None) is not None:
source = 'file'
destination_version = self.config.get("file_migration")
destination_ontology = self.virtuoso.get_ontology_from_file(
destination_version)
else:
destination_version = self._get_destination_version()
destination_ontology = self.virtuoso.get_ontology_by_version(
destination_version)
sparql_up, sparql_down = self.virtuoso.get_sparql(current_ontology,
destination_ontology,
current_version,
destination_version,
source)
self._execute_migrations(sparql_up,
sparql_down,
current_version,
destination_version)
def run():
cli = CLI()
try:
(options, args) = cli.parse()
if options.simple_db_migrate_version:
msg = 'simple-db-migrate v%s' % SIMPLE_DB_MIGRATE_VERSION
cli.info_and_exit(msg)
if options.show_colors:
CLI.show_colors()
# Create config
config = FileConfig(options.config_file, options.environment)
config.put('schema_version', options.schema_version)
config.put('show_sql', options.show_sql)
config.put('show_sql_only', options.show_sql_only)
config.put('new_migration', options.new_migration)
config.put('drop_db_first', options.drop_db_first)
config.put('paused_mode', options.paused_mode)
config.put('log_dir', options.log_dir)
config.put('label_version', options.label_version)
config.put('force_use_files_on_down', options.force_use_files_on_down)
config.put('force_execute_old_migrations_versions',
options.force_execute_old_migrations_versions)
# paused mode forces log_level to 2
log_level = int(options.log_level)
if options.paused_mode:
log_level = 2
config.put('log_level', log_level)
# Ask the password for user if configured
if config.get('db_password') == '<<ask_me>>':
if options.password:
passwd = options.password
else:
cli.msg(
'\nPlease inform password to connect to database "%s@%s:%s"'
% (config.get('db_user'), config.get('db_host'),
config.get('db_name')))
passwd = getpass()
config.remove('db_password')
config.put('db_password', passwd)
# If CLI was correctly parsed, execute db-migrate.
Main(config).execute()
except KeyboardInterrupt:
cli.info_and_exit("\nExecution interrupted by user...")
except Exception, e:
cli.error_and_exit(str(e))
def run():
cli = CLI()
try:
(options, args) = cli.parse()
if options.simple_db_migrate_version:
msg = 'simple-db-migrate v%s' % SIMPLE_DB_MIGRATE_VERSION
cli.info_and_exit(msg)
if options.show_colors:
CLI.show_colors()
# Create config
config = FileConfig(options.config_file, options.environment)
config.put('schema_version', options.schema_version)
config.put('show_sql', options.show_sql)
config.put('show_sql_only', options.show_sql_only)
config.put('new_migration', options.new_migration)
config.put('drop_db_first', options.drop_db_first)
config.put('paused_mode', options.paused_mode)
config.put('log_dir', options.log_dir)
config.put('label_version', options.label_version)
config.put('force_use_files_on_down', options.force_use_files_on_down)
config.put('force_execute_old_migrations_versions', options.force_execute_old_migrations_versions)
# paused mode forces log_level to 2
log_level = int(options.log_level)
if options.paused_mode:
log_level = 2
config.put('log_level', log_level)
# Ask the password for user if configured
if config.get('db_password') == '<<ask_me>>':
if options.password:
passwd = options.password
else:
cli.msg('\nPlease inform password to connect to database "%s@%s:%s"' % (config.get('db_user'), config.get('db_host'), config.get('db_name')))
passwd = getpass()
config.remove('db_password')
config.put('db_password', passwd)
# If CLI was correctly parsed, execute db-migrate.
Main(config).execute()
except KeyboardInterrupt:
cli.info_and_exit("\nExecution interrupted by user...")
except Exception, e:
cli.error_and_exit(str(e))
def run():
cli = CLI()
try:
(options, args) = cli.parse()
if options.torneira_version:
msg = "torneira v%s" % torneira.__version__
cli.info_and_exit(msg)
if options.show_colors:
CLI.show_colors()
Main().excecute()
except KeyboardInterrupt:
cli.info_and_exit("\nExecution interrupted by user...")
except Exception, e:
cli.error_and_exit(str(e))
def run(options):
try:
if options.get('simple_db_migrate_version'):
msg = 'simple-db-migrate v%s' % SIMPLE_DB_MIGRATE_VERSION
CLI.info_and_exit(msg)
if options.get('show_colors'):
CLI.show_colors()
# Create config
if options.get('config_file'):
config = FileConfig(options.get('config_file'), options.get('environment'))
else:
config = Config()
config.update('schema_version', options.get('schema_version'))
config.update('show_sql', options.get('show_sql'))
config.update('show_sql_only', options.get('show_sql_only'))
config.update('new_migration', options.get('new_migration'))
config.update('drop_db_first', options.get('drop_db_first'))
config.update('paused_mode', options.get('paused_mode'))
config.update('log_dir', options.get('log_dir'))
config.update('label_version', options.get('label_version'))
config.update('force_use_files_on_down', options.get('force_use_files_on_down'))
config.update('force_execute_old_migrations_versions', options.get('force_execute_old_migrations_versions'))
config.update('utc_timestamp', options.get('utc_timestamp'))
config.update('database_user', options.get('database_user'))
config.update('database_password', options.get('database_password'))
config.update('database_host', options.get('database_host'))
config.update('database_name', options.get('database_name'))
if options.get('database_migrations_dir'):
config.update("database_migrations_dir", Config._parse_migrations_dir(options.get('database_migrations_dir')))
config.update('database_engine', options.get('database_engine'))
if not config.get('database_engine', None):
config.update('database_engine', "mysql")
config.update('database_version_table', options.get('database_version_table'))
if not config.get('database_version_table', None):
config.update('database_version_table', "__db_version__")
# paused mode forces log_level to 2
log_level = int(options.get('log_level'))
if options.get('paused_mode'):
log_level = 2
config.update('log_level', log_level)
# Ask the password for user if configured
if config.get('database_password') == '<<ask_me>>':
if options.get('password'):
passwd = options.get('password')
else:
CLI.msg('\nPlease inform password to connect to database "%s@%s:%s"' % (config.get('database_user'), config.get('database_host'), config.get('database_name')))
passwd = getpass()
config.update('database_password', passwd)
# If CLI was correctly parsed, execute db-migrate.
Main(config).execute()
except KeyboardInterrupt:
CLI.info_and_exit("\nExecution interrupted by user...")
except Exception, e:
CLI.error_and_exit(str(e))
class Main(object):
def __init__(self, config=None, mysql=None, db_migrate=None):
self.__cli = CLI()
self.__config = config
self.__mysql = mysql
if self.__mysql is None and not self.__config.get("new_migration"):
self.__mysql = MySQL(config)
self.__db_migrate = db_migrate
if self.__db_migrate is None:
self.__db_migrate = Migrations(config)
def execute(self):
self.__cli.msg("\nStarting DB migration...", "PINK")
if self.__config.get("new_migration"):
self._create_migration()
else:
self._migrate()
self.__cli.msg("\nDone.\n", "PINK")
def _create_migration(self):
new_file = self.__db_migrate.create_migration(self.__config.get("new_migration"))
self.__cli.msg("- Created file '%s'" % (new_file))
def _migrate(self):
destination_version = self._get_destination_version()
current_version = self.__mysql.get_current_schema_version()
self.__cli.msg("- Current version is: %s" % current_version, "GREEN")
self.__cli.msg("- Destination version is: %s" % destination_version, "GREEN")
# if current and destination versions are the same,
# will consider a migration up to execute remaining files
is_migration_up = True
if int(current_version) > int(destination_version):
is_migration_up = False
# do it!
self._execute_migrations(current_version, destination_version, is_migration_up)
def _get_destination_version(self):
destination_version = self.__config.get("schema_version")
if destination_version is None:
destination_version = self.__db_migrate.latest_schema_version_available()
if not self.__db_migrate.check_if_version_exists(destination_version):
self.__cli.error_and_exit("version not found (%s)" % destination_version)
return destination_version
def _get_migration_files_to_be_executed(self, current_version, destination_version):
mysql_versions = self.__mysql.get_all_schema_versions()
migration_versions = self.__db_migrate.get_all_migration_versions()
# migration up: the easy part
if current_version <= destination_version:
remaining_versions_to_execute = Lists.subtract(migration_versions, mysql_versions)
remaining_versions_to_execute = [
version for version in remaining_versions_to_execute if version <= destination_version
]
return remaining_versions_to_execute
# migration down...
down_versions = [
version for version in mysql_versions if version <= current_version and version > destination_version
]
for version in down_versions:
if version not in migration_versions:
self.__cli.error_and_exit(
"impossible to migrate down: one of the versions was not found (%s)" % version
)
down_versions.reverse()
return down_versions
def _execute_migrations(self, current_version, destination_version, is_migration_up):
# getting only the migration sql files to be executed
versions_to_be_executed = self._get_migration_files_to_be_executed(current_version, destination_version)
if versions_to_be_executed is None or len(versions_to_be_executed) == 0:
self.__cli.msg("\nNothing to do.\n", "PINK")
sys.exit(0)
up_down_label = "up" if is_migration_up else "down"
if self.__config.get("show_sql_only"):
self.__cli.msg("WARNING: database migrations are not being executed ('--showsqlonly' activated)", "YELLOW")
else:
self.__cli.msg("\nStarting migration %s!" % up_down_label)
self.__cli.msg("*** versions: %s\n" % versions_to_be_executed, "GRAY")
sql_statements_executed = []
for migration_version in versions_to_be_executed:
sql_file = self.__db_migrate.get_migration_file_name_from_version_number(migration_version)
sql = self.__db_migrate.get_sql_command(sql_file, is_migration_up)
if not self.__config.get("show_sql_only"):
self.__cli.msg("===== executing %s (%s) =====" % (sql_file, up_down_label))
self.__mysql.change(sql, migration_version, is_migration_up)
# recording the last statement executed
sql_statements_executed.append(sql)
if self.__config.get("show_sql") or self.__config.get("show_sql_only"):
self.__cli.msg("__________ SQL statements executed __________", "YELLOW")
for sql in sql_statements_executed:
self.__cli.msg(sql, "YELLOW")
self.__cli.msg("_____________________________________________", "YELLOW")
def run(options):
""" Initial Module. Treat Parameters and call Main Module for execution """
try:
if options.get('simple_virtuoso_migrate_version'):
msg = ('simple-virtuoso-migrate v%s' %
SIMPLE_VIRTUOSO_MIGRATE_VERSION)
CLI.info_and_exit(msg)
if options.get('show_colors'):
CLI.show_colors()
# Create config
if options.get('config_file'):
config = FileConfig(options.get('config_file'),
options.get('environment'))
else:
config = Config()
config.update('schema_version', options.get('schema_version'))
config.update('show_sparql', options.get('show_sparql'))
config.update('show_sparql_only', options.get('show_sparql_only'))
config.update('file_migration', options.get('file_migration'))
#config.update('add_ttl', options.get('add_ttl'))
config.update('load_ttl', options.get('load_ttl'))
config.update('log_dir', options.get('log_dir'))
config.update('database_user', options.get('database_user'))
config.update('database_password', options.get('database_password'))
config.update('host_user', options.get('host_user'))
config.update('host_password', options.get('host_password'))
config.update('virtuoso_dirs_allowed',
options.get('virtuoso_dirs_allowed'))
config.update('database_host', options.get('database_host'))
config.update('database_port', options.get('database_port'))
config.update('database_endpoint', options.get('database_endpoint'))
config.update('database_graph', options.get('database_graph'))
config.update('database_ontology', options.get('database_ontology'))
if options.get('database_migrations_dir'):
config.update("database_migrations_dir",
Config._parse_migrations_dir(
options.get(
'database_migrations_dir')))
config.update("database_migrations_dir",
config.get("database_migrations_dir")[0])
config.update('log_level', int(options.get('log_level')))
# Ask the password for user if configured
if config.get('database_password') == '<<ask_me>>':
CLI.msg('\nPlease inform password to connect to virtuoso (DATABASE) "%s@%s"' % (config.get('database_user'),
config.get('database_host')))
passwd = getpass()
config.update('database_password', passwd)
is_local = config.get('database_host', '').lower() in ["localhost",
"127.0.0.1"]
if config.get('load_ttl') and\
config.get('virtuoso_dirs_allowed') is None and\
not is_local:
if config.get('host_password') == '<<ask_me>>':
msg = '\nPlease inform password to connect to virtuoso (HOST) "%s@%s"'
CLI.msg(msg % (config.get('host_user'),
config.get('database_host')))
passwd = getpass()
config.update('host_password', passwd)
# If CLI was correctly parsed, execute db-virtuoso.
Main(config).execute()
except KeyboardInterrupt:
CLI.info_and_exit("\nExecution interrupted by user...")
except Exception, e:
CLI.error_and_exit(unicode(e))
def run(options):
try:
if options.get('simple_db_migrate_version'):
msg = 'simple-db-migrate v%s' % SIMPLE_DB_MIGRATE_VERSION
CLI.info_and_exit(msg)
if options.get('show_colors'):
CLI.show_colors()
# Create config
if options.get('config_file') or os.path.exists('simple-db-migrate.conf'):
config = FileConfig(options.get('config_file') or 'simple-db-migrate.conf', options.get('environment'))
else:
config = Config()
config.update('schema_version', options.get('schema_version'))
config.update('show_sql', options.get('show_sql'))
config.update('show_sql_only', options.get('show_sql_only'))
config.update('new_migration', options.get('new_migration'))
config.update('drop_db_first', options.get('drop_db_first'))
config.update('paused_mode', options.get('paused_mode'))
config.update('log_dir', options.get('log_dir'))
config.update('label_version', options.get('label_version'))
config.update('force_use_files_on_down', options.get('force_use_files_on_down'))
config.update('force_execute_old_migrations_versions', options.get('force_execute_old_migrations_versions'))
config.update('utc_timestamp', options.get('utc_timestamp'))
config.update('database_user', options.get('database_user'))
config.update('database_password', options.get('database_password'))
config.update('database_host', options.get('database_host'))
config.update('database_port', options.get('database_port'))
config.update('database_name', options.get('database_name'))
if config.get('database_port', None):
config.update('database_port', int(config.get('database_port')))
if options.get('database_migrations_dir'):
config.update("database_migrations_dir", Config._parse_migrations_dir(options.get('database_migrations_dir')))
config.update('database_engine', options.get('database_engine'))
if not config.get('database_engine', None):
config.update('database_engine', "mysql")
config.update('database_version_table', options.get('database_version_table'))
if not config.get('database_version_table', None):
config.update('database_version_table', "__db_version__")
# paused mode forces log_level to 2
log_level = int(options.get('log_level'))
if options.get('paused_mode'):
log_level = 2
config.update('log_level', log_level)
# Ask the password for user if configured
if config.get('database_password') == '<<ask_me>>':
if options.get('password'):
passwd = options.get('password')
else:
CLI.msg('\nPlease inform password to connect to database "%s@%s:%s"' % (config.get('database_user'), config.get('database_host'), config.get('database_name')))
passwd = getpass()
config.update('database_password', passwd)
if options.get('info_database'):
if options.get('info_database').lower() == 'last_label':
CLI.info_and_exit(Main(config).last_label() or "NONE")
elif options.get('info_database').lower() == 'labels':
labels = Main(config).labels()
CLI.info_and_exit(labels and "\n".join(labels) or "NONE")
else:
CLI.error_and_exit("The '%s' is a wrong parameter for info" % options.get('info_database').lower())
# If CLI was correctly parsed, execute db-migrate.
Main(config).execute()
except KeyboardInterrupt:
CLI.info_and_exit("\nExecution interrupted by user...")
except Exception, e:
CLI.error_and_exit(str(e))
class Migrations(object):
__migration_files_extension = ".migration"
def __init__(self, config=None):
self.__migrations_dir = config.get("migrations_dir")
self.__cli = CLI()
def get_all_migration_files(self):
path = os.path.abspath(self.__migrations_dir)
dir_list = None
try:
dir_list = os.listdir(path)
except OSError:
self.__cli.error_and_exit("directory not found ('%s')" % path)
files = []
for dir_file in dir_list:
if self.is_file_name_valid(dir_file):
files.append(dir_file)
if len(files) == 0:
self.__cli.error_and_exit("no migration files found")
files.sort()
return files
def get_sql_command(self, sql_file, migration_up=True):
try:
f = codecs.open(self.__migrations_dir + "/" + sql_file, "r", "utf-8")
exec(f.read())
except IOError:
self.__cli.error_and_exit("%s: file not found" % self.__migrations_dir + "/" + sql_file)
else:
f.close()
try:
(SQL_UP, SQL_DOWN)
except NameError:
self.__cli.error_and_exit("migration file is incorrect; it does not define 'SQL_UP' or 'SQL_DOWN' ('%s')" % sql_file)
sql = u""
sql = SQL_UP if migration_up else SQL_DOWN
if sql is None or sql == "":
self.__cli.error_and_exit("migration command is empty ('%s')" % sql_file)
return sql
def get_all_migration_versions(self):
versions = []
migration_files = self.get_all_migration_files()
for each_file in migration_files:
versions.append(self.get_migration_version(each_file))
return versions
def get_all_migration_versions_up_to(self, limit_version):
all_versions = self.get_all_migration_versions()
return [version for version in all_versions if version < limit_version]
def get_migration_version(self, sql_file):
return sql_file[0:sql_file.find("_")]
def check_if_version_exists(self, version):
files = self.get_all_migration_files()
for file_name in files:
if file_name[0:14] == version:
return True
return False
def latest_schema_version_available(self):
all_files = self.get_all_migration_files()
all_files.sort()
all_files.reverse()
return self.get_migration_version(all_files[0])
def is_file_name_valid(self, file_name):
mask = r"[0-9]{14}\w+%s$" % self.__migration_files_extension
match = re.match(mask, file_name, re.IGNORECASE)
return match != None
def create_migration(self, migration_name):
timestamp = strftime("%Y%m%d%H%M%S")
file_name = "%s_%s%s" % (timestamp, migration_name, self.__migration_files_extension)
if not self.is_file_name_valid(file_name):
self.__cli.error_and_exit("invalid migration name; it should contain only letters, numbers and/or underscores ('%s')" % migration_name)
new_file = "%s/%s" % (self.__migrations_dir, file_name)
try:
f = codecs.open(new_file, "w", "utf-8")
f.write(MigrationFile.template)
f.close()
except IOError:
self.__cli.error_and_exit("could not create file ('%s')" % new_file)
return file_name
def get_migration_file_name_from_version_number(self, version):
all_files = self.get_all_migration_files()
for f in all_files:
if f.startswith(version):
return f
return None
def run(options):
""" Initial Module. Treat Parameters and call Main Module for execution """
try:
if options.get("simple_virtuoso_migrate_version"):
msg = "simple-virtuoso-migrate v%s" % SIMPLE_VIRTUOSO_MIGRATE_VERSION
CLI.info_and_exit(msg)
if options.get("show_colors"):
CLI.show_colors()
# Create config
if options.get("config_file"):
config = FileConfig(options.get("config_file"), options.get("environment"))
else:
config = Config()
config.update("schema_version", options.get("schema_version"))
config.update("show_sparql", options.get("show_sparql"))
config.update("show_sparql_only", options.get("show_sparql_only"))
config.update("file_migration", options.get("file_migration"))
# config.update('add_ttl', options.get('add_ttl'))
config.update("load_ttl", options.get("load_ttl"))
config.update("log_dir", options.get("log_dir"))
config.update("database_user", options.get("database_user"))
config.update("database_password", options.get("database_password"))
config.update("host_user", options.get("host_user"))
config.update("host_password", options.get("host_password"))
config.update("virtuoso_dirs_allowed", options.get("virtuoso_dirs_allowed"))
config.update("database_host", options.get("database_host"))
config.update("database_port", options.get("database_port"))
config.update("database_endpoint", options.get("database_endpoint"))
config.update("database_graph", options.get("database_graph"))
config.update("database_ontology", options.get("database_ontology"))
if options.get("database_migrations_dir"):
config.update(
"database_migrations_dir", Config._parse_migrations_dir(options.get("database_migrations_dir"))
)
config.update("database_migrations_dir", config.get("database_migrations_dir")[0])
config.update("log_level", int(options.get("log_level")))
# Ask the password for user if configured
if config.get("database_password") == "<<ask_me>>":
CLI.msg(
'\nPlease inform password to connect to\
virtuoso (DATABASE) "%s@%s"'
% (config.get("database_user"), config.get("database_host"))
)
passwd = getpass()
config.update("database_password", passwd)
is_local = config.get("database_host", "").lower() in ["localhost", "127.0.0.1"]
# import pdb; pdb.set_trace()
if config.get("load_ttl") and config.get("virtuoso_dirs_allowed") is None and not is_local:
if config.get("host_password") == "<<ask_me>>":
CLI.msg(
'\nPlease inform password to connect to\
virtuoso (HOST) "%s@%s"'
% (config.get("host_user"), config.get("database_host"))
)
passwd = getpass()
config.update("host_password", passwd)
# If CLI was correctly parsed, execute db-virtuoso.
Main(config).execute()
except KeyboardInterrupt:
CLI.info_and_exit("\nExecution interrupted by user...")
except Exception, e:
CLI.error_and_exit(unicode(e))
