def execute(connector, executor, properties, folder): filters = Filters() excluded_folders = properties.get_property('excluded_folders') for excluded_folder in excluded_folders: ef = File(excluded_folder) ff = FileFolderFilter(ef) filters.add(ff) excluded_extensions = properties.get_property('excluded_extensions') for excluded_extension in excluded_extensions: ef = File("*." + excluded_extension) ff = FileExtensionFilter(ef) filters.add(ff) excluded_files = properties.get_property('excluded_files') for excluded_file in excluded_files: ef = File(excluded_file) ff = FileFilter(ef) filters.add(ff) if folder.is_dir(): files = Files.list(folder) for file in files: if not filters.accept(file) and not file.is_dir(): url = file.get_url() print url executor['script'] = file connector.execute(executor, properties)
def list_filtered(folder, properties): filters = Filters() results = [] excluded_folders = properties.get('excluded_folders') for excluded_folder in excluded_folders: ef = File(excluded_folder) ff = FileFolderFilter(ef) filters.add(ff) excluded_extensions = properties.get('excluded_extensions') for excluded_extension in excluded_extensions: ef = File("*." + excluded_extension) ff = FileExtensionFilter(ef) filters.add(ff) excluded_files = properties.get('excluded_files') for excluded_file in excluded_files: ef = File(excluded_file) ff = FileFilter(ef) filters.add(ff) if folder.is_dir(): files = Files.list(folder) for file in files: if not filters.accept(file) and not file.is_dir(): results.append(file) return results
def get_config_file(properties): current_dir = properties.get_property("current.dir") project_file = properties.get_property("project.file") f = File(os.path.join(current_dir, project_file)) if f.exists(): return f noora_dir = properties.get_property("noora.dir") f = File(os.path.join(noora_dir, project_file)) return f
def testCheckVersion(self): script = File("checkversion.sql") properties = Properties() properties.set_property('name', 'app_prop') properties.set_property('previous', '1.0.0') stream = PreProcessor.parse(script, properties) tmp = File("tmp.sql") f = open(tmp.get_url(), 'w') f.write(stream) f.close() script_reader = open(tmp.get_url()) feedback = File('feedback.log') feedback_writer = open(feedback.get_url(), 'w') statement = "mysql --show-warnings --host=localhost --user=apps --password=apps acme" call = CallFactory.new_call(statement) call['stdin'] = script_reader call['stdout'] = feedback_writer call['stderr'] = feedback_writer result = Shell.execute(call) print "result", result
def testProcessor(self): script = File("drop_tables.sql") properties = Properties() properties.set_property('database', 'orcl') stream = PreProcessor.parse(script, properties) self.assertEqual(stream, 'orcl', "invalid transformation")
def fail_on_invalid_environment(self, connector, executor, environment, properties): plugin_dir = properties.get('plugin.dir') properties['environment'] = environment script = File(os.path.join(plugin_dir, 'mysql', 'update', 'checkenvironment.sql')) executor['script'] = script connector.execute(executor, properties) if "(Code 1329)" in connector.get_result(): raise InvalidEnvironmentException("invalid environment: {}".format(environment))
def fail_on_invalid_environment(self, connector, executor, environment, properties): plugin_dir = properties.get_property('plugin.dir') properties.set_property('environment', environment) script = File( os.path.join(plugin_dir, 'mssql', 'update', 'checkenvironment.sql')) executor['script'] = script
def execute(self, executable, properties): script = executable['script'] cp = Properties() cp.set_property('database', executable['database']) if 'environment' in properties.keys(): cp.set_property('environment', properties.get_property('environment')) if 'previous' in properties.keys(): cp.set_property('previous', properties.get_property('previous')) stream = PreProcessor.parse(script, cp) tmp = File("tmp.sql") f = open(tmp.get_url(), 'w') f.write(stream) f.close() script_reader = open(tmp.get_url()) feedback = File('feedback.log') feedback_writer = open(feedback.get_url(), 'w') statement = "mysql --show-warnings --host=" + executable[ 'host'] + " --user="******" --password="******" " + executable['database'] call = CallFactory.new_call(statement) call['stdin'] = script_reader call['stdout'] = feedback_writer call['stderr'] = feedback_writer result = Shell.execute(call) self.set_result(result)
def execute(self, executable, properties): script = executable['script'] cp = Properties() cp.set_property('database', executable['database']) if 'schema' in executable.keys(): cp.set_property('schema', executable['schema']) if 'environment' in properties.keys(): cp.set_property('environment', properties.get_property('environment')) if 'previous' in properties.keys(): cp.set_property('previous', properties.get_property('previous')) stream = PreProcessor.parse(script, cp) tmp = File("tmp.sql") f = open(tmp.get_url(), 'w') f.write(stream) f.close() #script_reader = open(tmp.get_url()) feedback = File('feedback.log') feedback_writer = open(feedback.get_url(), 'w') statement = "sqlcmd -b -S " + executable['host'] + " -U " + executable[ 'username'] + ' -P ' + executable['password'] + " -d " + executable[ 'database'] + " -i " + tmp.get_url() call = CallFactory.new_call(statement) call['stdout'] = feedback_writer call['stderr'] = feedback_writer result = Shell.execute(call) self.set_result(result)
def fail_on_invalid_environment(self, properties, connector, executor, environment): plugin_dir = properties.get('plugin.dir') properties['environment'] = environment script = File( os.path.join(plugin_dir, 'mssql', 'update', 'checkenvironment.sql')) executor['script'] = script connector.execute(executor, properties)
def list(file=None, recursive=False, exclude=None): result = [] if file.exists(): folder = file.get_url() file_list = os.listdir(folder) file_list.sort() for file_item in file_list: url = os.path.join(folder, file_item) candidate_file = File(url) if candidate_file.is_file() and exclude != "file": result.append(candidate_file) if candidate_file.is_dir() and exclude != "directory": result.append(candidate_file) if candidate_file.is_dir() and recursive == True: recursive_files = Files.list(candidate_file, recursive, exclude) for recursive_file in recursive_files: result.append(recursive_file) return result
def execute(self, properties, arguments): """ Drop a database after checking if schema and environment are valid values. Also check that host is not on the block list. :type properties: system.Properties.Properties :param properties: The project properties :type arguments: dict :param arguments: This dict contains the plugin arguments: * **host**: The hostname to drop on; * **environment**: Environment to drop the database from. """ prepared_args = self._validate_and_prepare(properties, arguments) host = prepared_args['host'] environment = prepared_args['environment'] databases = properties.get('databases') objects = properties.get('drop_objects') # retrieve the user credentials for this database project. users = properties.get('postgresql_users') # try to retrieve the users from the credentials file, when no users are configured in # myproject.json. if not users: # retrieve the name of this database project, introduced in version 1.0.12 profile = PropertyHelper.get_profile(properties) if profile: users = profile.get('postgresql_users') # fail when no users are found. This means that they are not set in myproject.json or # credentials.json Fail.fail_on_no_users(users) for database in databases: print( "dropping database '{db}' on host '{host}' using environment '{env}'" .format(db=database, host=host, env=environment)) executor = PropertyHelper.get_postgres_properties( users, host, database) connector = self.get_connector() for obj in objects: folder = File( os.path.join(properties.get('plugin.dir'), 'postgresql', 'drop', obj)) ConnectionExecutor.execute(connector, executor, properties, folder) print("database '{}' dropped".format(database))
def execute(self, executable, properties): """ Execute the script provided by `executable` on the target server. :type executable: dict :param executable: Should contain the following keys and values: * **host**: The address of the server to connect to; * **port**: Server port to connect to; * **database**: The database name; * **username**: Database username; * **password**: Database user password; * **script**: Path to the script to execute. :type properties: noora.system.Properties.Properties :param properties: A Noora project properties instance """ script = executable['script'] cp = { 'database': executable['database'], 'username': executable['username'].split('@')[0] } if 'environment' in properties.keys(): cp['environment'] = properties.get('environment') if 'previous' in properties.keys(): cp['previous'] = properties.get('previous') stream = PreProcessor.parse(script, cp) tmp = File("tmp.sql") f = open(tmp.get_url(), 'w') f.write(stream) f.close() script_reader = open(tmp.get_url()) feedback = File('feedback.log') feedback_writer = open(feedback.get_url(), 'w') statement = \ "PGPASSWORD={passwd} psql -h {host} -p {port} -U {user} -d {db} " \ "-v ON_ERROR_STOP=1".format( host=executable['host'], port=executable.get('port'), user=executable['username'], passwd=executable['password'], db=executable['database'], ) call = CallFactory.new_call(statement) call['stdin'] = script_reader call['stdout'] = feedback_writer call['stderr'] = feedback_writer result = Shell.execute(call) self.set_result(result)
def execute(connector, executor, properties, folder): """ Execute all applicable files in the specified folder. :type connector: noora.connectors.Connector.Connector :param connector: A connector instance; :type executor: dict :param executor: Settings and credentials for the connector; :type properties: noora.system.Properties.Properties :param properties: Project properties; :param folder: The target folder to execute. """ filters = Filters() excluded_folders = properties.get('excluded_folders') for excluded_folder in excluded_folders: ef = File(excluded_folder) ff = FileFolderFilter(ef) filters.add(ff) excluded_extensions = properties.get('excluded_extensions') for excluded_extension in excluded_extensions: ef = File("*." + excluded_extension) ff = FileExtensionFilter(ef) filters.add(ff) excluded_files = properties.get('excluded_files') for excluded_file in excluded_files: ef = File(excluded_file) ff = FileFilter(ef) filters.add(ff) if folder.is_dir(): files = Files.list(folder) for file in files: if not filters.accept(file) and not file.is_dir(): url = file.get_url() print(url) executor['script'] = file connector.execute(executor, properties)
def execute(self, executable, properties): """ Execute the script provided by `executable` on the target server. :type executable: dict :param executable: Should contain the following keys and values: * **host**: The address of the server to connect to; * **port**: Server port to connect to; * **database**: The database name; * **schema**: The schema to use; * **username**: Database username; * **password**: Database user password; * **script**: Path to the script to execute. :type properties: noora.system.Properties.Properties :param properties: A Noora project properties instance """ script = executable['script'] cp = dict() cp['database'] = executable['database'] if 'schema' in executable.keys(): cp['schema'] = executable['schema'] if 'environment' in properties.keys(): cp['environment'] = properties.get('environment') if 'previous' in properties.keys(): cp['previous'] = properties.get('previous') stream = PreProcessor.parse(script, cp) tmp = File("tmp.sql") f = open(tmp.get_url(), 'w') f.write(stream) f.close() # FIXME: remove? #script_reader = open(tmp.get_url()) feedback = File('feedback.log') feedback_writer = open(feedback.get_url(), 'w') # FIXME: rewrite with format statement = "sqlcmd -b -S {host},{port} -U {user} -P {passwd} -d {db} -i {url}".format( host=executable['host'], port=executable.get('port') or 1433, user=executable['username'], passwd=executable['password'], db=executable['database'], url=tmp.get_url(), ) call = CallFactory.new_call(statement) call['stdout'] = feedback_writer call['stderr'] = feedback_writer result = Shell.execute(call) self.set_result(result)
def fail_on_invalid_version(self, connector, executor, version, properties): plugin_dir = properties.get('plugin.dir') versions = Versions() version_loader = VersionLoader(versions) version_loader.load(properties) versions.sort() v = Version(version) previous = versions.previous(v).get_value() properties['previous'] = previous script = File(os.path.join(plugin_dir, 'mysql', 'update', 'checkversion.sql')) executor['script'] = script connector.execute(executor, properties) if "(Code 1329)" in connector.get_result(): raise InvalidVersionException("invalid version: {}".format(previous))
def fail_on_invalid_version(self, properties, connector, executor, version): plugin_dir = properties.get('plugin.dir') versions = Versions() version_loader = VersionLoader(versions) version_loader.load(properties) versions.sort() v = Version(version) previous = versions.previous(v).get_value() properties['previous'] = previous script = File( os.path.join(plugin_dir, 'mssql', 'update', 'checkversion.sql')) executor['script'] = script connector.execute(executor, properties)
def execute(self, arguments, properties): host = arguments.h Fail.fail_on_no_host(host) self.fail_on_blocked_hosts(host, properties) default_schemes = properties.get_property('schemes') schemes = Ora.nvl(arguments.s, default_schemes) Fail.fail_on_invalid_schema(arguments.s, properties) default_environment = properties.get_property('default_environment') environment = Ora.nvl(arguments.e, default_environment) Fail.fail_on_invalid_environment(arguments.e, properties) database = properties.get_property('database') objects = properties.get_property('drop_objects') # alias = arguments.a # Fail.fail_on_invalid_alias(arguments.a, properties) # # # if an alias is given, only the alias database will be installed, other databases will be ignored. # if alias: # print "using alias :" + alias # databases = [alias] for schema in schemes: print "dropping schema '" + schema + "' in database '" + database + "on host '" + host + "' using environment '" + environment + "'" users = properties.get_property('mssql_users') username = PropertyHelper.get_mssql_user(users, host, schema) password = PropertyHelper.get_mssql_password(users, host, schema) connector = self.get_connector() executor = {} executor['host'] = host executor['database'] = database executor['schema'] = schema executor['username'] = username executor['password'] = password for object in objects: folder = File( os.path.join(self.get_drop_dir(properties), object)) ConnectionExecutor.execute(connector, executor, properties, folder) print "schema '" + schema + "' dropped."
def load(self, properties): alter = File(properties.get_property("alter.dir")) if alter.exists(): files = Files() for version in files.list(alter): self.__versions.add(Version(version.tail())) create = File(properties.get_property("create.dir")) if create.exists(): self.__versions.add( Version(properties.get_property("default_version")))
def testEnvironment(self): noora_dir = os.path.dirname(noora.__file__) current_dir = os.path.abspath('.') properties = dict() properties["noora.dir"] = noora_dir properties["current.dir"] = current_dir executable = {} executable['script'] = File("exec_get_environment.sql") executable['host'] = "elsevierdb4.c07v9zv3jhxs.eu-west-1.rds.amazonaws.com,1433" executable['user'] = "******" executable['password'] = "******" executable['database'] = "ElsevierDB" connector = MssqlConnector() connector.execute(executable, properties)
def load(self, properties): """ Find all versions on the current project and add them to the list. :param properties: An instance of ``noora.system.Properties.Properties``. """ alter = File(properties.get("alter.dir")) if alter.exists(): files = Files() for version in files.list(alter): self.__versions.add(Version(version.tail())) create = File(properties.get("create.dir")) if create.exists(): self.__versions.add(Version(properties.get("default_version")))
def __get_config(self): """ Load a myproject.json file and update some extra parameters if this is an actual project """ current_dir = self.__props.get("current.dir") project_file = self.__props.get("project.file") config_file = File(os.path.join(current_dir, project_file)) if not config_file.exists(): noora_dir = self.__props.get("noora.dir") config_file = File(os.path.join(noora_dir, project_file)) # Read project configuration with open(config_file.get_url()) as fd: data = json.load(fd) return data
def execute(self, properties, arguments): """ Create a new database instance for the initial version. :type properties: system.Properties.Properties :param properties: The project properties :type arguments: dict :param arguments: This dict contains the plugin arguments: * **host**: The hostname where the database is running; * **environment**: The environment to create the database in (optional). """ prepared_args = self._validate_and_prepare(properties, arguments) host = prepared_args['host'] environment = prepared_args['environment'] databases = properties.get('databases') objects = properties.get('create_objects') # retrieve the user credentials for this database project. users = properties.get('postgresql_users') # try to retrieve the users from the credentials file, when no users are configured in # myproject.json. if not users: # retrieve the name of this database project, introduced in version 1.0.12 profile = PropertyHelper.get_profile(properties) if profile: users = profile.get('postgresql_users') # fail when no users are found. This means that they are not set in myproject.json or # credentials.json Fail.fail_on_no_users(users) connector = self.get_connector() create_dir = properties.get('create.dir') for database in databases: print("creating database '{db}' on host '{host}' using environment '{env}'".format( db=database, host=host, env=environment)) executor = PropertyHelper.get_postgres_properties(users, host, database) for obj in objects: # global ddl objects folder = File(os.path.join(create_dir, database, 'ddl', obj)) ConnectionExecutor.execute(connector, executor, properties, folder) # environment specific ddl objects folder = File(os.path.join(create_dir, database, 'ddl', obj, environment)) ConnectionExecutor.execute(connector, executor, properties, folder) # global dat objects folder = File(os.path.join(create_dir, database, 'dat')) ConnectionExecutor.execute(connector, executor, properties, folder) # environment specific dat objects folder = File(os.path.join(create_dir, database, 'dat', environment)) ConnectionExecutor.execute(connector, executor, properties, folder) print("database '{}' created.".format(database))
def execute(self, arguments, properties): project = '' version = None if arguments.v: version = arguments.v current_dir = properties.get_property('current.dir') project_file = properties.get_property('project.file') config_file = File(os.path.join(current_dir, project_file)) if not config_file.exists(): database = raw_input('database : ') project = database + "-db" host = raw_input('host [localhost] : ') host = Ora.nvl(host, "localhost") username = raw_input('username : '******'password : '******'version [1.0.0]: ') version = Ora.nvl(version, "1.0.0") os.mkdir(project) template_dir = os.path.join(properties.get_property('plugin.dir'), 'mysql', 'generate', 'templates') template_file = os.path.join(template_dir, project_file) f = open(template_file) stream = f.read() f.close() stream = stream.replace('{host}', host) stream = stream.replace('{database}', database) stream = stream.replace('{username}', username) stream = stream.replace('{password}', password) stream = stream.replace('{version}', version) config_file = os.path.join(current_dir, project, project_file) f = open(config_file, 'w') f.write(stream) f.close() properties.set_property('alter.dir', os.path.join(current_dir, project, 'alter')) properties.set_property('create.dir', os.path.join(current_dir, project, 'create')) config_file = os.path.join(current_dir, project, project_file) f = open(config_file) data = json.load(f) for key in data.keys(): properties.set_property(key, data[key]) f.close() versions = Versions() version_loader = VersionLoader(versions) version_loader.load(properties) versions.sort() version_guesser = VersionGuesser(properties, versions) next_version = version_guesser.guess(version).to_string() version_dir = version_guesser.to_folder(next_version) # create the version folder os.makedirs(version_dir) databases = properties.get_property('databases') version_database = properties.get_property('version_database') default_version = properties.get_property('default_version') environments = properties.get_property('environments') objects = properties.get_property('create_objects') for database in databases: # create the scheme folder database_dir = os.path.join(version_dir, database) os.mkdir(database_dir) # create the dat folder dat_dir = os.path.join(database_dir, 'dat') os.mkdir(dat_dir) # create the version script in the dat folder if database == version_database: version_file = os.path.join(dat_dir, 'version.sql') f = open(version_file, 'w') if next_version == default_version: stream = properties.get_property('version_insert_statement') else: stream = properties.get_property('version_update_statement') stream = stream.replace('<version>', next_version) f.write(stream) f.close() # sqlScript=self.getSqlVersionStatement(versions, version) # projectHelper.writeFile(datFolder+os.sep+'version.sql', sqlScript) # create the environment folders in the dat folder for environment in environments: os.mkdir(os.path.join(dat_dir, environment)) # create the environment script in the dat folder. if database == version_database and next_version == default_version: environment_file = os.path.join(dat_dir, environment, 'environment.sql') f = open(environment_file, 'w') stream = properties.get_property('environment_insert_statement') stream = stream.replace('<environment>', environment) f.write(stream) f.close() # create the ddl folder ddl_dir = os.path.join(database_dir, 'ddl') os.mkdir(ddl_dir) # create the object folders in the ddl folder for object in objects: os.mkdir(os.path.join(ddl_dir, object)) # create the template code on create. if database == version_database and next_version == default_version: for object in objects: object_dir = os.path.join(template_dir, object) if os.path.exists(object_dir): files = Files.list(File(object_dir)) for file in files: shutil.copyfile(file.get_url(), os.path.join(ddl_dir, object, file.tail())) print "version " + next_version + " created."
def execute(self, arguments, properties): properties.set_property('create.dir', os.path.join(properties.get_property('current.dir'), 'create')) properties.set_property('alter.dir', os.path.join(properties.get_property('current.dir'), 'alter')) version = arguments.v Fail.fail_on_no_version(version) Fail.fail_on_unknown_version(version, properties) default_databases = properties.get_property('databases') databases = Ora.nvl(arguments.d, default_databases) Fail.fail_on_invalid_database(arguments.d, properties) current_dir = properties.get_property('current.dir') component_name = properties.get_property('component_name') target_dir = os.path.join(current_dir, properties.get_property('component_target_folder')) objects = properties.get_property('create_objects') build_dir = App.build_dir(version, properties) # exclude the file 'version.sql', this file is excluded from the dat listing below. component_excluded_files = properties.get_property('component_excluded_files') excluded_files = properties.get_property('excluded_files') excluded_files.extend(component_excluded_files) # create the target folder, if not present. if not File(target_dir).exists(): os.makedirs(target_dir) print "building component with version '" + version + "'" zip_file = os.path.join(target_dir, component_name + '_' + version + '.zip') zip_handle = ZipFile(zip_file, 'w') for database in databases: for object in objects: if not object == 'lib': # global ddl objects folder = File(os.path.join(build_dir, database, 'ddl', object)) zip_dir = File(os.path.join(component_name + '_' + version, 'ddl', object)) files = Files.list_filtered(folder, properties) for file in files: print file.get_url() target_file = File(os.path.join(zip_dir.get_url(), file.tail())) zip_handle.write(file.get_url(), target_file.get_url(), ZIP_DEFLATED) # global dat files folder = File(os.path.join(build_dir, database, 'dat')) zip_dir = File(os.path.join(component_name + '_' + version, 'dat')) files = Files.list_filtered(folder, properties) for file in files: print file.get_url() target_file = File(os.path.join(zip_dir.get_url(), file.tail())) zip_handle.write(file.get_url(), target_file.get_url(), ZIP_DEFLATED) # create the version script in the dat folder version_statement = self.version_statement(version, properties) version_statement = version_statement.replace('<version>', version) version_statement = version_statement.replace('<name>', component_name) f = open('version.sql', 'w') f.write(version_statement) f.close() zip_dir = File(os.path.join(component_name + '_' + version, 'dat')) target_file = File(os.path.join(zip_dir.get_url(), 'version.sql')) zip_handle.write('version.sql', target_file.get_url()) # remove the version.sql file. os.remove('version.sql') # create the checkversion script in the root folder # first retrieve the previous version for the checkversion script versions = Versions() version_loader = VersionLoader(versions) version_loader.load(properties) versions.sort() previous = versions.previous(Version(version)) component_select_statement = properties.get_property("component_select_statement") component_select_statement = component_select_statement.replace('<name>', component_name) component_select_statement = component_select_statement.replace('<previous>', previous.get_value()) print component_select_statement f = open('checkversion.sql', 'w') f.write(component_select_statement) f.close() zip_dir = File(os.path.join(component_name + '_' + version)) target_file = File(os.path.join(zip_dir.get_url(), 'checkversion.sql')) zip_handle.write('checkversion.sql', target_file.get_url()) # remove the version.sql file. os.remove('checkversion.sql') zip_handle.close() print "component with version " + version + " created."
def execute(self, properties, arguments): """ Update database after checking if schema and environment are valid values. Also check that host is not on the block list and that the version to update to is valid :type properties: system.Properties.Properties :param properties: The project properties :type arguments: dict :param arguments: This dict contains the plugin arguments: * **version**: The version to update the database to; * **host**: The hostname that hosts the database to update; * **schema**: Schema to update (optional); * **environment**: Environment to update the database in (optional). """ prepared_args = self._validate_and_prepare(properties, arguments) version = prepared_args['version'] host = prepared_args['host'] environment = prepared_args['environment'] databases = properties.get('databases') objects = properties.get('create_objects') version_database = properties.get('version_database') alter_dir = properties.get('alter.dir') # retrieve the user credentials for this database project. users = properties.get('postgresql_users') # try to retrieve the users from the credentials file, when no users are configured in # myproject.json. if not users: # retrieve the name of this database project, introduced in version 1.0.12 profile = PropertyHelper.get_profile(properties) if profile: users = profile.get('postgresql_users') connector = self.get_connector() for database in databases: print( "updating database '{db}' on host '{host}' using environment '{env}'" .format(db=database, host=host, env=environment)) executor = PropertyHelper.get_mysql_properties( users, host, database) if database == version_database: self.fail_on_invalid_environment(connector, executor, environment, properties) self.fail_on_invalid_version(connector, executor, version, properties) for obj in objects: # global ddl objects folder = File( os.path.join(alter_dir, version, database, 'ddl', obj)) ConnectionExecutor.execute(connector, executor, properties, folder) # environment specific ddl objects folder = File( os.path.join(alter_dir, version, database, 'ddl', obj, environment)) ConnectionExecutor.execute(connector, executor, properties, folder) # global dat objects folder = File(os.path.join(alter_dir, version, database, 'dat')) ConnectionExecutor.execute(connector, executor, properties, folder) # environment specific dat objects folder = File( os.path.join(alter_dir, version, database, 'dat', environment)) ConnectionExecutor.execute(connector, executor, properties, folder) print("database '{}' updated".format(database))
def testMssqlConnector(self): noora_dir = os.path.dirname(noora.__file__) current_dir = os.path.abspath('.') properties = dict() properties["noora.dir"] = noora_dir properties["current.dir"] = current_dir executable = {} executable['script'] = File("drop_application_properties_s.sql") executable['host'] = "elsevierdb4.c07v9zv3jhxs.eu-west-1.rds.amazonaws.com,1433" executable['user'] = "******" executable['password'] = "******" executable['database'] = "ElsevierDB" connector = MssqlConnector() connector.execute(executable, properties) executable = {} executable['script'] = File("application_properties_s.sql") executable['host'] = "elsevierdb4.c07v9zv3jhxs.eu-west-1.rds.amazonaws.com,1433" executable['user'] = "******" executable['password'] = "******" executable['database'] = "ElsevierDB" connector = MssqlConnector() connector.execute(executable, properties) executable = {} executable['script'] = File("drop_application_properties.sql") executable['host'] = "elsevierdb4.c07v9zv3jhxs.eu-west-1.rds.amazonaws.com,1433" executable['user'] = "******" executable['password'] = "******" executable['database'] = "ElsevierDB" connector = MssqlConnector() connector.execute(executable, properties) executable = {} executable['script'] = File("application_properties.sql") executable['host'] = "elsevierdb4.c07v9zv3jhxs.eu-west-1.rds.amazonaws.com,1433" executable['user'] = "******" executable['password'] = "******" executable['database'] = "ElsevierDB" connector = MssqlConnector() connector.execute(executable, properties) executable = {} executable['script'] = File("drop_app_prop.sql") executable['host'] = "elsevierdb4.c07v9zv3jhxs.eu-west-1.rds.amazonaws.com,1433" executable['user'] = "******" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" \ "" executable['password'] = "******" executable['database'] = "ElsevierDB" connector = MssqlConnector() connector.execute(executable, properties) executable = {} executable['script'] = File("app_prop.sql") executable['host'] = "elsevierdb4.c07v9zv3jhxs.eu-west-1.rds.amazonaws.com,1433" executable['user'] = "******" executable['password'] = "******" executable['database'] = "ElsevierDB" connector = MssqlConnector() connector.execute(executable, properties)
def execute(self, properties, arguments): """ Build the package after verifying version is valid, and database, if provided. :param properties: The project properties :param arguments: A dict of { 'version': 'The version to package for' 'database': 'The database to package for (optional)' } """ prepared_args = self._validate_and_prepare(properties, arguments) version = prepared_args['version'] databases = prepared_args['databases'] current_dir = properties.get('current.dir') component_name = properties.get('component_name') target_dir = os.path.join(current_dir, properties.get('component_target_folder')) objects = properties.get('create_objects') if version == properties.get("default_version"): build_dir = properties.get("create.dir") else: build_dir = os.path.join(properties.get("alter.dir"), version) # exclude the file 'version.sql', this file is excluded from the dat listing below. component_excluded_files = properties.get('component_excluded_files') excluded_files = properties.get('excluded_files') excluded_files.extend(component_excluded_files) # create the target folder, if not present. if not File(target_dir).exists(): os.makedirs(target_dir) print("building component with version '{}'".format(version)) zip_file = os.path.join(target_dir, component_name + '_' + version + '.zip') zip_handle = ZipFile(zip_file, 'w') for database in databases: for obj in objects: if not obj == 'lib': # global ddl objects folder = File(os.path.join(build_dir, database, 'ddl', obj)) zip_dir = File(os.path.join(component_name + '_' + version, 'ddl', obj)) files = Files.list_filtered(folder, properties) for file in files: print(file.get_url()) target_file = File(os.path.join(zip_dir.get_url(), file.tail())) zip_handle.write(file.get_url(), target_file.get_url(), ZIP_DEFLATED) # global dat files folder = File(os.path.join(build_dir, database, 'dat')) zip_dir = File(os.path.join(component_name + '_' + version, 'dat')) files = Files.list_filtered(folder, properties) for file in files: print(file.get_url()) target_file = File(os.path.join(zip_dir.get_url(), file.tail())) zip_handle.write(file.get_url(), target_file.get_url(), ZIP_DEFLATED) # create the version script in the dat folder if version == properties.get("default_version"): version_statement = properties.get("component_insert_statement") else: version_statement = properties.get("component_update_statement") version_statement = version_statement.replace('<version>', version) version_statement = version_statement.replace('<name>', component_name) f = open('version.sql', 'w') f.write(version_statement) f.close() zip_dir = File(os.path.join(component_name + '_' + version, 'dat')) target_file = File(os.path.join(zip_dir.get_url(), 'version.sql')) zip_handle.write('version.sql', target_file.get_url()) # remove the version.sql file. os.remove('version.sql') # create the checkversion script in the root folder # first retrieve the previous version for the checkversion script versions = Versions() version_loader = VersionLoader(versions) version_loader.load(properties) versions.sort() previous = versions.previous(Version(version)) component_select_statement = properties.get("component_select_statement") component_select_statement = component_select_statement.replace('<name>', component_name) component_select_statement = component_select_statement.replace( '<previous>', previous.get_value()) print(component_select_statement) f = open('checkversion.sql', 'w') f.write(component_select_statement) f.close() zip_dir = File(os.path.join(component_name + '_' + version)) target_file = File(os.path.join(zip_dir.get_url(), 'checkversion.sql')) zip_handle.write('checkversion.sql', target_file.get_url()) # remove the version.sql file. os.remove('checkversion.sql') zip_handle.close() print("component with version {} created.".format(version))
def fail_on_unknown_version(self, version, properties): alter_dir = properties.get_property('alter.dir') version_dir = File(os.path.join(alter_dir, version)) if not version_dir.exists(): raise UnknownVersionException("unknown version folder", version)
def execute(self, arguments, properties): properties.set_property( 'create.dir', os.path.join(properties.get_property('current.dir'), 'create')) properties.set_property( 'alter.dir', os.path.join(properties.get_property('current.dir'), 'alter')) host = arguments.h Fail.fail_on_no_host(host) version = arguments.v Fail.fail_on_no_version(version) default_databases = properties.get_property('databases') databases = Ora.nvl(arguments.d, default_databases) Fail.fail_on_invalid_database(arguments.d, properties) default_environment = properties.get_property('default_environment') environment = Ora.nvl(arguments.e, default_environment) Fail.fail_on_invalid_environment(arguments.e, properties) objects = properties.get_property('create_objects') version_database = properties.get_property('version_database') alter_dir = properties.get_property('alter.dir') self.fail_on_unknown_version(version, properties) alias = arguments.a database_aliases = properties.get_property('database_aliases') Fail.fail_on_invalid_alias(alias, properties) # if an alias is given, only this database will be installed, other databases will be ignored. if alias: print "using alias :" + alias databases = [alias] connector = self.get_connector() for database in databases: print "updating database '" + database + "' on host '" + host + "' using environment '" + environment + "'" users = properties.get_property('mysql_users') username = PropertyHelper.get_mysql_user(users, host, database) password = PropertyHelper.get_mysql_passwd(users, host, database) executor = {} executor['host'] = host executor['database'] = database executor['username'] = username executor['password'] = password database_folder = PropertyHelper.get_database_folder( database, database_aliases) if database == version_database: self.fail_on_invalid_environment(connector, executor, environment, properties) self.fail_on_invalid_version(connector, executor, version, properties) for object in objects: # global ddl objects folder = File( os.path.join(alter_dir, version, database_folder, 'ddl', object)) ConnectionExecutor.execute(connector, executor, properties, folder) # environment specific ddl objects folder = File( os.path.join(alter_dir, version, database_folder, 'ddl', object, environment)) ConnectionExecutor.execute(connector, executor, properties, folder) # global dat objects folder = File( os.path.join(alter_dir, version, database_folder, 'dat')) ConnectionExecutor.execute(connector, executor, properties, folder) # environment specific dat objects folder = File( os.path.join(alter_dir, version, database_folder, 'dat', environment)) ConnectionExecutor.execute(connector, executor, properties, folder) print "database '" + database + "' updated."