class Config(object): def __repr__(self): return str(self.__config) def __init__(self, config_file="simple-db-migrate.conf"): self.__cli = CLI() self.__config = {} # read configurations try: f = codecs.open(config_file, "r", "utf-8") exec(f.read()) except IOError: self.__cli.error_and_exit("%s: file not found" % config_file) else: f.close() try: self.put("db_host", HOST) self.put("db_user", USERNAME) self.put("db_password", PASSWORD) self.put("db_name", DATABASE) self.put("db_version_table", "__db_version__") migrations_dir = self.__get_migrations_absolute_dir(config_file, MIGRATIONS_DIR) self.put("migrations_dir", migrations_dir) except NameError, e: self.__cli.error_and_exit("config file error: " + str(e))
class MSSQL(object): def __init__(self, config=None, mssql_driver=_mssql): self.__cli = CLI() self.__mssql_driver = mssql_driver self.__mssql_host = config.get("db_host") self.__mssql_user = config.get("db_user") self.__mssql_passwd = config.get("db_password") self.__mssql_db = config.get("db_name") self.__version_table = config.get("db_version_table") if config.get("drop_db_first"): self._drop_database() self._create_database_if_not_exists() self._create_version_table_if_not_exists() def __mssql_connect(self, connect_using_db_name=True): try: conn = self.__mssql_driver.connect(server=self.__mssql_host, user=self.__mssql_user, password=self.__mssql_passwd) if connect_using_db_name: conn.select_db(self.__mssql_db) return conn except Exception, e: self.__cli.error_and_exit("could not connect to database (%s)" % e)
class MySQL(object): def __init__(self, config=None, mysql_driver=MySQLdb): self.__cli = CLI() self.__mysql_driver = mysql_driver self.__mysql_host = config.get("db_host") self.__mysql_user = config.get("db_user") self.__mysql_passwd = config.get("db_password") self.__mysql_db = config.get("db_name") self.__version_table = config.get("db_version_table") if config.get("drop_db_first"): self._drop_database() self._create_database_if_not_exists() self._create_version_table_if_not_exists() def __mysql_connect(self, connect_using_db_name=True): try: conn = self.__mysql_driver.connect(host=self.__mysql_host, user=self.__mysql_user, passwd=self.__mysql_passwd) # this should be configured in the config file, not hardcoded conn.set_character_set('utf8') if connect_using_db_name: conn.select_db(self.__mysql_db) return conn except Exception, e: self.__cli.error_and_exit("could not connect to database (%s)" % e)
def _migrate(self): """ Execute migrations based on git tags """ source = 'git' current_ontology = None current_version, origen = self.virtuoso.get_current_version() # Making the first migration to the database if current_version is None: if self.config.get("file_migration", None) is not None: self._execution_log(("- Current version is: %s" % current_version), "GREEN", log_level_limit=1) self._execution_log(("- Destination version is: %s" % self.config.get("file_migration")), "GREEN", log_level_limit=1) CLI.error_and_exit("Can't execute migration FROM None TO File " "(TIP: version it using git --tag and then " "use -m)") else: if origen == "file": if self.config.get("file_migration", None) is not None: self._execution_log(("- Current version is: %s" % current_version), "GREEN", log_level_limit=1) self._execution_log(("- Destination version is: %s" % self.config.get("file_migration")), "GREEN", log_level_limit=1) CLI.error_and_exit("Can't execute migration FROM File TO " "File (TIP: version it using git --tag " "and then use -m)") current_ontology = self.virtuoso.get_ontology_by_version( current_version) if self.config.get("file_migration", None) is not None: source = 'file' destination_version = self.config.get("file_migration") destination_ontology = self.virtuoso.get_ontology_from_file( destination_version) else: destination_version = self._get_destination_version() destination_ontology = self.virtuoso.get_ontology_by_version( destination_version) sparql_up, sparql_down = self.virtuoso.get_sparql(current_ontology, destination_ontology, current_version, destination_version, source) self._execute_migrations(sparql_up, sparql_down, current_version, destination_version)
def run(): cli = CLI() try: (options, args) = cli.parse() if options.simple_db_migrate_version: msg = 'simple-db-migrate v%s' % SIMPLE_DB_MIGRATE_VERSION cli.info_and_exit(msg) if options.show_colors: CLI.show_colors() # Create config config = FileConfig(options.config_file, options.environment) config.put('schema_version', options.schema_version) config.put('show_sql', options.show_sql) config.put('show_sql_only', options.show_sql_only) config.put('new_migration', options.new_migration) config.put('drop_db_first', options.drop_db_first) config.put('paused_mode', options.paused_mode) config.put('log_dir', options.log_dir) config.put('label_version', options.label_version) config.put('force_use_files_on_down', options.force_use_files_on_down) config.put('force_execute_old_migrations_versions', options.force_execute_old_migrations_versions) # paused mode forces log_level to 2 log_level = int(options.log_level) if options.paused_mode: log_level = 2 config.put('log_level', log_level) # Ask the password for user if configured if config.get('db_password') == '<<ask_me>>': if options.password: passwd = options.password else: cli.msg( '\nPlease inform password to connect to database "%s@%s:%s"' % (config.get('db_user'), config.get('db_host'), config.get('db_name'))) passwd = getpass() config.remove('db_password') config.put('db_password', passwd) # If CLI was correctly parsed, execute db-migrate. Main(config).execute() except KeyboardInterrupt: cli.info_and_exit("\nExecution interrupted by user...") except Exception, e: cli.error_and_exit(str(e))
def run(): cli = CLI() try: (options, args) = cli.parse() if options.simple_db_migrate_version: msg = 'simple-db-migrate v%s' % SIMPLE_DB_MIGRATE_VERSION cli.info_and_exit(msg) if options.show_colors: CLI.show_colors() # Create config config = FileConfig(options.config_file, options.environment) config.put('schema_version', options.schema_version) config.put('show_sql', options.show_sql) config.put('show_sql_only', options.show_sql_only) config.put('new_migration', options.new_migration) config.put('drop_db_first', options.drop_db_first) config.put('paused_mode', options.paused_mode) config.put('log_dir', options.log_dir) config.put('label_version', options.label_version) config.put('force_use_files_on_down', options.force_use_files_on_down) config.put('force_execute_old_migrations_versions', options.force_execute_old_migrations_versions) # paused mode forces log_level to 2 log_level = int(options.log_level) if options.paused_mode: log_level = 2 config.put('log_level', log_level) # Ask the password for user if configured if config.get('db_password') == '<<ask_me>>': if options.password: passwd = options.password else: cli.msg('\nPlease inform password to connect to database "%s@%s:%s"' % (config.get('db_user'), config.get('db_host'), config.get('db_name'))) passwd = getpass() config.remove('db_password') config.put('db_password', passwd) # If CLI was correctly parsed, execute db-migrate. Main(config).execute() except KeyboardInterrupt: cli.info_and_exit("\nExecution interrupted by user...") except Exception, e: cli.error_and_exit(str(e))
def run(): cli = CLI() try: (options, args) = cli.parse() if options.torneira_version: msg = "torneira v%s" % torneira.__version__ cli.info_and_exit(msg) if options.show_colors: CLI.show_colors() Main().excecute() except KeyboardInterrupt: cli.info_and_exit("\nExecution interrupted by user...") except Exception, e: cli.error_and_exit(str(e))
def run(options): try: if options.get('simple_db_migrate_version'): msg = 'simple-db-migrate v%s' % SIMPLE_DB_MIGRATE_VERSION CLI.info_and_exit(msg) if options.get('show_colors'): CLI.show_colors() # Create config if options.get('config_file'): config = FileConfig(options.get('config_file'), options.get('environment')) else: config = Config() config.update('schema_version', options.get('schema_version')) config.update('show_sql', options.get('show_sql')) config.update('show_sql_only', options.get('show_sql_only')) config.update('new_migration', options.get('new_migration')) config.update('drop_db_first', options.get('drop_db_first')) config.update('paused_mode', options.get('paused_mode')) config.update('log_dir', options.get('log_dir')) config.update('label_version', options.get('label_version')) config.update('force_use_files_on_down', options.get('force_use_files_on_down')) config.update('force_execute_old_migrations_versions', options.get('force_execute_old_migrations_versions')) config.update('utc_timestamp', options.get('utc_timestamp')) config.update('database_user', options.get('database_user')) config.update('database_password', options.get('database_password')) config.update('database_host', options.get('database_host')) config.update('database_name', options.get('database_name')) if options.get('database_migrations_dir'): config.update("database_migrations_dir", Config._parse_migrations_dir(options.get('database_migrations_dir'))) config.update('database_engine', options.get('database_engine')) if not config.get('database_engine', None): config.update('database_engine', "mysql") config.update('database_version_table', options.get('database_version_table')) if not config.get('database_version_table', None): config.update('database_version_table', "__db_version__") # paused mode forces log_level to 2 log_level = int(options.get('log_level')) if options.get('paused_mode'): log_level = 2 config.update('log_level', log_level) # Ask the password for user if configured if config.get('database_password') == '<<ask_me>>': if options.get('password'): passwd = options.get('password') else: CLI.msg('\nPlease inform password to connect to database "%s@%s:%s"' % (config.get('database_user'), config.get('database_host'), config.get('database_name'))) passwd = getpass() config.update('database_password', passwd) # If CLI was correctly parsed, execute db-migrate. Main(config).execute() except KeyboardInterrupt: CLI.info_and_exit("\nExecution interrupted by user...") except Exception, e: CLI.error_and_exit(str(e))
class Main(object): def __init__(self, config=None, mysql=None, db_migrate=None): self.__cli = CLI() self.__config = config self.__mysql = mysql if self.__mysql is None and not self.__config.get("new_migration"): self.__mysql = MySQL(config) self.__db_migrate = db_migrate if self.__db_migrate is None: self.__db_migrate = Migrations(config) def execute(self): self.__cli.msg("\nStarting DB migration...", "PINK") if self.__config.get("new_migration"): self._create_migration() else: self._migrate() self.__cli.msg("\nDone.\n", "PINK") def _create_migration(self): new_file = self.__db_migrate.create_migration(self.__config.get("new_migration")) self.__cli.msg("- Created file '%s'" % (new_file)) def _migrate(self): destination_version = self._get_destination_version() current_version = self.__mysql.get_current_schema_version() self.__cli.msg("- Current version is: %s" % current_version, "GREEN") self.__cli.msg("- Destination version is: %s" % destination_version, "GREEN") # if current and destination versions are the same, # will consider a migration up to execute remaining files is_migration_up = True if int(current_version) > int(destination_version): is_migration_up = False # do it! self._execute_migrations(current_version, destination_version, is_migration_up) def _get_destination_version(self): destination_version = self.__config.get("schema_version") if destination_version is None: destination_version = self.__db_migrate.latest_schema_version_available() if not self.__db_migrate.check_if_version_exists(destination_version): self.__cli.error_and_exit("version not found (%s)" % destination_version) return destination_version def _get_migration_files_to_be_executed(self, current_version, destination_version): mysql_versions = self.__mysql.get_all_schema_versions() migration_versions = self.__db_migrate.get_all_migration_versions() # migration up: the easy part if current_version <= destination_version: remaining_versions_to_execute = Lists.subtract(migration_versions, mysql_versions) remaining_versions_to_execute = [ version for version in remaining_versions_to_execute if version <= destination_version ] return remaining_versions_to_execute # migration down... down_versions = [ version for version in mysql_versions if version <= current_version and version > destination_version ] for version in down_versions: if version not in migration_versions: self.__cli.error_and_exit( "impossible to migrate down: one of the versions was not found (%s)" % version ) down_versions.reverse() return down_versions def _execute_migrations(self, current_version, destination_version, is_migration_up): # getting only the migration sql files to be executed versions_to_be_executed = self._get_migration_files_to_be_executed(current_version, destination_version) if versions_to_be_executed is None or len(versions_to_be_executed) == 0: self.__cli.msg("\nNothing to do.\n", "PINK") sys.exit(0) up_down_label = "up" if is_migration_up else "down" if self.__config.get("show_sql_only"): self.__cli.msg("WARNING: database migrations are not being executed ('--showsqlonly' activated)", "YELLOW") else: self.__cli.msg("\nStarting migration %s!" % up_down_label) self.__cli.msg("*** versions: %s\n" % versions_to_be_executed, "GRAY") sql_statements_executed = [] for migration_version in versions_to_be_executed: sql_file = self.__db_migrate.get_migration_file_name_from_version_number(migration_version) sql = self.__db_migrate.get_sql_command(sql_file, is_migration_up) if not self.__config.get("show_sql_only"): self.__cli.msg("===== executing %s (%s) =====" % (sql_file, up_down_label)) self.__mysql.change(sql, migration_version, is_migration_up) # recording the last statement executed sql_statements_executed.append(sql) if self.__config.get("show_sql") or self.__config.get("show_sql_only"): self.__cli.msg("__________ SQL statements executed __________", "YELLOW") for sql in sql_statements_executed: self.__cli.msg(sql, "YELLOW") self.__cli.msg("_____________________________________________", "YELLOW")
def run(options): """ Initial Module. Treat Parameters and call Main Module for execution """ try: if options.get('simple_virtuoso_migrate_version'): msg = ('simple-virtuoso-migrate v%s' % SIMPLE_VIRTUOSO_MIGRATE_VERSION) CLI.info_and_exit(msg) if options.get('show_colors'): CLI.show_colors() # Create config if options.get('config_file'): config = FileConfig(options.get('config_file'), options.get('environment')) else: config = Config() config.update('schema_version', options.get('schema_version')) config.update('show_sparql', options.get('show_sparql')) config.update('show_sparql_only', options.get('show_sparql_only')) config.update('file_migration', options.get('file_migration')) #config.update('add_ttl', options.get('add_ttl')) config.update('load_ttl', options.get('load_ttl')) config.update('log_dir', options.get('log_dir')) config.update('database_user', options.get('database_user')) config.update('database_password', options.get('database_password')) config.update('host_user', options.get('host_user')) config.update('host_password', options.get('host_password')) config.update('virtuoso_dirs_allowed', options.get('virtuoso_dirs_allowed')) config.update('database_host', options.get('database_host')) config.update('database_port', options.get('database_port')) config.update('database_endpoint', options.get('database_endpoint')) config.update('database_graph', options.get('database_graph')) config.update('database_ontology', options.get('database_ontology')) if options.get('database_migrations_dir'): config.update("database_migrations_dir", Config._parse_migrations_dir( options.get( 'database_migrations_dir'))) config.update("database_migrations_dir", config.get("database_migrations_dir")[0]) config.update('log_level', int(options.get('log_level'))) # Ask the password for user if configured if config.get('database_password') == '<<ask_me>>': CLI.msg('\nPlease inform password to connect to virtuoso (DATABASE) "%s@%s"' % (config.get('database_user'), config.get('database_host'))) passwd = getpass() config.update('database_password', passwd) is_local = config.get('database_host', '').lower() in ["localhost", "127.0.0.1"] if config.get('load_ttl') and\ config.get('virtuoso_dirs_allowed') is None and\ not is_local: if config.get('host_password') == '<<ask_me>>': msg = '\nPlease inform password to connect to virtuoso (HOST) "%s@%s"' CLI.msg(msg % (config.get('host_user'), config.get('database_host'))) passwd = getpass() config.update('host_password', passwd) # If CLI was correctly parsed, execute db-virtuoso. Main(config).execute() except KeyboardInterrupt: CLI.info_and_exit("\nExecution interrupted by user...") except Exception, e: CLI.error_and_exit(unicode(e))
def run(options): try: if options.get('simple_db_migrate_version'): msg = 'simple-db-migrate v%s' % SIMPLE_DB_MIGRATE_VERSION CLI.info_and_exit(msg) if options.get('show_colors'): CLI.show_colors() # Create config if options.get('config_file') or os.path.exists('simple-db-migrate.conf'): config = FileConfig(options.get('config_file') or 'simple-db-migrate.conf', options.get('environment')) else: config = Config() config.update('schema_version', options.get('schema_version')) config.update('show_sql', options.get('show_sql')) config.update('show_sql_only', options.get('show_sql_only')) config.update('new_migration', options.get('new_migration')) config.update('drop_db_first', options.get('drop_db_first')) config.update('paused_mode', options.get('paused_mode')) config.update('log_dir', options.get('log_dir')) config.update('label_version', options.get('label_version')) config.update('force_use_files_on_down', options.get('force_use_files_on_down')) config.update('force_execute_old_migrations_versions', options.get('force_execute_old_migrations_versions')) config.update('utc_timestamp', options.get('utc_timestamp')) config.update('database_user', options.get('database_user')) config.update('database_password', options.get('database_password')) config.update('database_host', options.get('database_host')) config.update('database_port', options.get('database_port')) config.update('database_name', options.get('database_name')) if config.get('database_port', None): config.update('database_port', int(config.get('database_port'))) if options.get('database_migrations_dir'): config.update("database_migrations_dir", Config._parse_migrations_dir(options.get('database_migrations_dir'))) config.update('database_engine', options.get('database_engine')) if not config.get('database_engine', None): config.update('database_engine', "mysql") config.update('database_version_table', options.get('database_version_table')) if not config.get('database_version_table', None): config.update('database_version_table', "__db_version__") # paused mode forces log_level to 2 log_level = int(options.get('log_level')) if options.get('paused_mode'): log_level = 2 config.update('log_level', log_level) # Ask the password for user if configured if config.get('database_password') == '<<ask_me>>': if options.get('password'): passwd = options.get('password') else: CLI.msg('\nPlease inform password to connect to database "%s@%s:%s"' % (config.get('database_user'), config.get('database_host'), config.get('database_name'))) passwd = getpass() config.update('database_password', passwd) if options.get('info_database'): if options.get('info_database').lower() == 'last_label': CLI.info_and_exit(Main(config).last_label() or "NONE") elif options.get('info_database').lower() == 'labels': labels = Main(config).labels() CLI.info_and_exit(labels and "\n".join(labels) or "NONE") else: CLI.error_and_exit("The '%s' is a wrong parameter for info" % options.get('info_database').lower()) # If CLI was correctly parsed, execute db-migrate. Main(config).execute() except KeyboardInterrupt: CLI.info_and_exit("\nExecution interrupted by user...") except Exception, e: CLI.error_and_exit(str(e))
class Migrations(object): __migration_files_extension = ".migration" def __init__(self, config=None): self.__migrations_dir = config.get("migrations_dir") self.__cli = CLI() def get_all_migration_files(self): path = os.path.abspath(self.__migrations_dir) dir_list = None try: dir_list = os.listdir(path) except OSError: self.__cli.error_and_exit("directory not found ('%s')" % path) files = [] for dir_file in dir_list: if self.is_file_name_valid(dir_file): files.append(dir_file) if len(files) == 0: self.__cli.error_and_exit("no migration files found") files.sort() return files def get_sql_command(self, sql_file, migration_up=True): try: f = codecs.open(self.__migrations_dir + "/" + sql_file, "r", "utf-8") exec(f.read()) except IOError: self.__cli.error_and_exit("%s: file not found" % self.__migrations_dir + "/" + sql_file) else: f.close() try: (SQL_UP, SQL_DOWN) except NameError: self.__cli.error_and_exit("migration file is incorrect; it does not define 'SQL_UP' or 'SQL_DOWN' ('%s')" % sql_file) sql = u"" sql = SQL_UP if migration_up else SQL_DOWN if sql is None or sql == "": self.__cli.error_and_exit("migration command is empty ('%s')" % sql_file) return sql def get_all_migration_versions(self): versions = [] migration_files = self.get_all_migration_files() for each_file in migration_files: versions.append(self.get_migration_version(each_file)) return versions def get_all_migration_versions_up_to(self, limit_version): all_versions = self.get_all_migration_versions() return [version for version in all_versions if version < limit_version] def get_migration_version(self, sql_file): return sql_file[0:sql_file.find("_")] def check_if_version_exists(self, version): files = self.get_all_migration_files() for file_name in files: if file_name[0:14] == version: return True return False def latest_schema_version_available(self): all_files = self.get_all_migration_files() all_files.sort() all_files.reverse() return self.get_migration_version(all_files[0]) def is_file_name_valid(self, file_name): mask = r"[0-9]{14}\w+%s$" % self.__migration_files_extension match = re.match(mask, file_name, re.IGNORECASE) return match != None def create_migration(self, migration_name): timestamp = strftime("%Y%m%d%H%M%S") file_name = "%s_%s%s" % (timestamp, migration_name, self.__migration_files_extension) if not self.is_file_name_valid(file_name): self.__cli.error_and_exit("invalid migration name; it should contain only letters, numbers and/or underscores ('%s')" % migration_name) new_file = "%s/%s" % (self.__migrations_dir, file_name) try: f = codecs.open(new_file, "w", "utf-8") f.write(MigrationFile.template) f.close() except IOError: self.__cli.error_and_exit("could not create file ('%s')" % new_file) return file_name def get_migration_file_name_from_version_number(self, version): all_files = self.get_all_migration_files() for f in all_files: if f.startswith(version): return f return None
def run(options): """ Initial Module. Treat Parameters and call Main Module for execution """ try: if options.get("simple_virtuoso_migrate_version"): msg = "simple-virtuoso-migrate v%s" % SIMPLE_VIRTUOSO_MIGRATE_VERSION CLI.info_and_exit(msg) if options.get("show_colors"): CLI.show_colors() # Create config if options.get("config_file"): config = FileConfig(options.get("config_file"), options.get("environment")) else: config = Config() config.update("schema_version", options.get("schema_version")) config.update("show_sparql", options.get("show_sparql")) config.update("show_sparql_only", options.get("show_sparql_only")) config.update("file_migration", options.get("file_migration")) # config.update('add_ttl', options.get('add_ttl')) config.update("load_ttl", options.get("load_ttl")) config.update("log_dir", options.get("log_dir")) config.update("database_user", options.get("database_user")) config.update("database_password", options.get("database_password")) config.update("host_user", options.get("host_user")) config.update("host_password", options.get("host_password")) config.update("virtuoso_dirs_allowed", options.get("virtuoso_dirs_allowed")) config.update("database_host", options.get("database_host")) config.update("database_port", options.get("database_port")) config.update("database_endpoint", options.get("database_endpoint")) config.update("database_graph", options.get("database_graph")) config.update("database_ontology", options.get("database_ontology")) if options.get("database_migrations_dir"): config.update( "database_migrations_dir", Config._parse_migrations_dir(options.get("database_migrations_dir")) ) config.update("database_migrations_dir", config.get("database_migrations_dir")[0]) config.update("log_level", int(options.get("log_level"))) # Ask the password for user if configured if config.get("database_password") == "<<ask_me>>": CLI.msg( '\nPlease inform password to connect to\ virtuoso (DATABASE) "%s@%s"' % (config.get("database_user"), config.get("database_host")) ) passwd = getpass() config.update("database_password", passwd) is_local = config.get("database_host", "").lower() in ["localhost", "127.0.0.1"] # import pdb; pdb.set_trace() if config.get("load_ttl") and config.get("virtuoso_dirs_allowed") is None and not is_local: if config.get("host_password") == "<<ask_me>>": CLI.msg( '\nPlease inform password to connect to\ virtuoso (HOST) "%s@%s"' % (config.get("host_user"), config.get("database_host")) ) passwd = getpass() config.update("host_password", passwd) # If CLI was correctly parsed, execute db-virtuoso. Main(config).execute() except KeyboardInterrupt: CLI.info_and_exit("\nExecution interrupted by user...") except Exception, e: CLI.error_and_exit(unicode(e))