def execute(connector, executor, properties, folder):
filters = Filters()
excluded_folders = properties.get_property('excluded_folders')
for excluded_folder in excluded_folders:
ef = File(excluded_folder)
ff = FileFolderFilter(ef)
filters.add(ff)
excluded_extensions = properties.get_property('excluded_extensions')
for excluded_extension in excluded_extensions:
ef = File("*." + excluded_extension)
ff = FileExtensionFilter(ef)
filters.add(ff)
excluded_files = properties.get_property('excluded_files')
for excluded_file in excluded_files:
ef = File(excluded_file)
ff = FileFilter(ef)
filters.add(ff)
if folder.is_dir():
files = Files.list(folder)
for file in files:
if not filters.accept(file) and not file.is_dir():
url = file.get_url()
print url
executor['script'] = file
connector.execute(executor, properties)
def list_filtered(folder, properties):
filters = Filters()
results = []
excluded_folders = properties.get('excluded_folders')
for excluded_folder in excluded_folders:
ef = File(excluded_folder)
ff = FileFolderFilter(ef)
filters.add(ff)
excluded_extensions = properties.get('excluded_extensions')
for excluded_extension in excluded_extensions:
ef = File("*." + excluded_extension)
ff = FileExtensionFilter(ef)
filters.add(ff)
excluded_files = properties.get('excluded_files')
for excluded_file in excluded_files:
ef = File(excluded_file)
ff = FileFilter(ef)
filters.add(ff)
if folder.is_dir():
files = Files.list(folder)
for file in files:
if not filters.accept(file) and not file.is_dir():
results.append(file)
return results
def get_config_file(properties):
current_dir = properties.get_property("current.dir")
project_file = properties.get_property("project.file")
f = File(os.path.join(current_dir, project_file))
if f.exists():
return f
noora_dir = properties.get_property("noora.dir")
f = File(os.path.join(noora_dir, project_file))
return f
def testCheckVersion(self):
script = File("checkversion.sql")
properties = Properties()
properties.set_property('name', 'app_prop')
properties.set_property('previous', '1.0.0')
stream = PreProcessor.parse(script, properties)
tmp = File("tmp.sql")
f = open(tmp.get_url(), 'w')
f.write(stream)
f.close()
script_reader = open(tmp.get_url())
feedback = File('feedback.log')
feedback_writer = open(feedback.get_url(), 'w')
statement = "mysql --show-warnings --host=localhost --user=apps --password=apps acme"
call = CallFactory.new_call(statement)
call['stdin'] = script_reader
call['stdout'] = feedback_writer
call['stderr'] = feedback_writer
result = Shell.execute(call)
print "result", result
def testProcessor(self):
script = File("drop_tables.sql")
properties = Properties()
properties.set_property('database', 'orcl')
stream = PreProcessor.parse(script, properties)
self.assertEqual(stream, 'orcl', "invalid transformation")
def fail_on_invalid_environment(self, connector, executor, environment, properties):
plugin_dir = properties.get('plugin.dir')
properties['environment'] = environment
script = File(os.path.join(plugin_dir, 'mysql', 'update', 'checkenvironment.sql'))
executor['script'] = script
connector.execute(executor, properties)
if "(Code 1329)" in connector.get_result():
raise InvalidEnvironmentException("invalid environment: {}".format(environment))
def fail_on_invalid_environment(self, connector, executor, environment,
properties):
plugin_dir = properties.get_property('plugin.dir')
properties.set_property('environment', environment)
script = File(
os.path.join(plugin_dir, 'mssql', 'update',
'checkenvironment.sql'))
executor['script'] = script
def execute(self, executable, properties):
script = executable['script']
cp = Properties()
cp.set_property('database', executable['database'])
if 'environment' in properties.keys():
cp.set_property('environment',
properties.get_property('environment'))
if 'previous' in properties.keys():
cp.set_property('previous', properties.get_property('previous'))
stream = PreProcessor.parse(script, cp)
tmp = File("tmp.sql")
f = open(tmp.get_url(), 'w')
f.write(stream)
f.close()
script_reader = open(tmp.get_url())
feedback = File('feedback.log')
feedback_writer = open(feedback.get_url(), 'w')
statement = "mysql --show-warnings --host=" + executable[
'host'] + " --user="******" --password="******" " + executable['database']
call = CallFactory.new_call(statement)
call['stdin'] = script_reader
call['stdout'] = feedback_writer
call['stderr'] = feedback_writer
result = Shell.execute(call)
self.set_result(result)
def execute(self, executable, properties):
script = executable['script']
cp = Properties()
cp.set_property('database', executable['database'])
if 'schema' in executable.keys():
cp.set_property('schema', executable['schema'])
if 'environment' in properties.keys():
cp.set_property('environment',
properties.get_property('environment'))
if 'previous' in properties.keys():
cp.set_property('previous', properties.get_property('previous'))
stream = PreProcessor.parse(script, cp)
tmp = File("tmp.sql")
f = open(tmp.get_url(), 'w')
f.write(stream)
f.close()
#script_reader = open(tmp.get_url())
feedback = File('feedback.log')
feedback_writer = open(feedback.get_url(), 'w')
statement = "sqlcmd -b -S " + executable['host'] + " -U " + executable[
'username'] + ' -P ' + executable['password'] + " -d " + executable[
'database'] + " -i " + tmp.get_url()
call = CallFactory.new_call(statement)
call['stdout'] = feedback_writer
call['stderr'] = feedback_writer
result = Shell.execute(call)
self.set_result(result)
def fail_on_invalid_environment(self, properties, connector, executor,
environment):
plugin_dir = properties.get('plugin.dir')
properties['environment'] = environment
script = File(
os.path.join(plugin_dir, 'mssql', 'update',
'checkenvironment.sql'))
executor['script'] = script
connector.execute(executor, properties)
def list(file=None, recursive=False, exclude=None):
result = []
if file.exists():
folder = file.get_url()
file_list = os.listdir(folder)
file_list.sort()
for file_item in file_list:
url = os.path.join(folder, file_item)
candidate_file = File(url)
if candidate_file.is_file() and exclude != "file":
result.append(candidate_file)
if candidate_file.is_dir() and exclude != "directory":
result.append(candidate_file)
if candidate_file.is_dir() and recursive == True:
recursive_files = Files.list(candidate_file, recursive, exclude)
for recursive_file in recursive_files:
result.append(recursive_file)
return result
def execute(self, properties, arguments):
"""
Drop a database after checking if schema and environment are
valid values. Also check that host is not on the block list.
:type properties: system.Properties.Properties
:param properties: The project properties
:type arguments: dict
:param arguments: This dict contains the plugin arguments:
* **host**: The hostname to drop on;
* **environment**: Environment to drop the database from.
"""
prepared_args = self._validate_and_prepare(properties, arguments)
host = prepared_args['host']
environment = prepared_args['environment']
databases = properties.get('databases')
objects = properties.get('drop_objects')
# retrieve the user credentials for this database project.
users = properties.get('postgresql_users')
# try to retrieve the users from the credentials file, when no users are configured in
# myproject.json.
if not users:
# retrieve the name of this database project, introduced in version 1.0.12
profile = PropertyHelper.get_profile(properties)
if profile:
users = profile.get('postgresql_users')
# fail when no users are found. This means that they are not set in myproject.json or
# credentials.json
Fail.fail_on_no_users(users)
for database in databases:
print(
"dropping database '{db}' on host '{host}' using environment '{env}'"
.format(db=database, host=host, env=environment))
executor = PropertyHelper.get_postgres_properties(
users, host, database)
connector = self.get_connector()
for obj in objects:
folder = File(
os.path.join(properties.get('plugin.dir'), 'postgresql',
'drop', obj))
ConnectionExecutor.execute(connector, executor, properties,
folder)
print("database '{}' dropped".format(database))
def execute(self, executable, properties):
"""
Execute the script provided by `executable` on the target server.
:type executable: dict
:param executable: Should contain the following keys and values:
* **host**: The address of the server to connect to;
* **port**: Server port to connect to;
* **database**: The database name;
* **username**: Database username;
* **password**: Database user password;
* **script**: Path to the script to execute.
:type properties: noora.system.Properties.Properties
:param properties: A Noora project properties instance
"""
script = executable['script']
cp = {
'database': executable['database'],
'username': executable['username'].split('@')[0]
}
if 'environment' in properties.keys():
cp['environment'] = properties.get('environment')
if 'previous' in properties.keys():
cp['previous'] = properties.get('previous')
stream = PreProcessor.parse(script, cp)
tmp = File("tmp.sql")
f = open(tmp.get_url(), 'w')
f.write(stream)
f.close()
script_reader = open(tmp.get_url())
feedback = File('feedback.log')
feedback_writer = open(feedback.get_url(), 'w')
statement = \
"PGPASSWORD={passwd} psql -h {host} -p {port} -U {user} -d {db} " \
"-v ON_ERROR_STOP=1".format(
host=executable['host'],
port=executable.get('port'),
user=executable['username'],
passwd=executable['password'],
db=executable['database'],
)
call = CallFactory.new_call(statement)
call['stdin'] = script_reader
call['stdout'] = feedback_writer
call['stderr'] = feedback_writer
result = Shell.execute(call)
self.set_result(result)
def execute(connector, executor, properties, folder):
"""
Execute all applicable files in the specified folder.
:type connector: noora.connectors.Connector.Connector
:param connector: A connector instance;
:type executor: dict
:param executor: Settings and credentials for the connector;
:type properties: noora.system.Properties.Properties
:param properties: Project properties;
:param folder: The target folder to execute.
"""
filters = Filters()
excluded_folders = properties.get('excluded_folders')
for excluded_folder in excluded_folders:
ef = File(excluded_folder)
ff = FileFolderFilter(ef)
filters.add(ff)
excluded_extensions = properties.get('excluded_extensions')
for excluded_extension in excluded_extensions:
ef = File("*." + excluded_extension)
ff = FileExtensionFilter(ef)
filters.add(ff)
excluded_files = properties.get('excluded_files')
for excluded_file in excluded_files:
ef = File(excluded_file)
ff = FileFilter(ef)
filters.add(ff)
if folder.is_dir():
files = Files.list(folder)
for file in files:
if not filters.accept(file) and not file.is_dir():
url = file.get_url()
print(url)
executor['script'] = file
connector.execute(executor, properties)
def execute(self, executable, properties):
"""
Execute the script provided by `executable` on the target server.
:type executable: dict
:param executable: Should contain the following keys and values:
* **host**: The address of the server to connect to;
* **port**: Server port to connect to;
* **database**: The database name;
* **schema**: The schema to use;
* **username**: Database username;
* **password**: Database user password;
* **script**: Path to the script to execute.
:type properties: noora.system.Properties.Properties
:param properties: A Noora project properties instance
"""
script = executable['script']
cp = dict()
cp['database'] = executable['database']
if 'schema' in executable.keys():
cp['schema'] = executable['schema']
if 'environment' in properties.keys():
cp['environment'] = properties.get('environment')
if 'previous' in properties.keys():
cp['previous'] = properties.get('previous')
stream = PreProcessor.parse(script, cp)
tmp = File("tmp.sql")
f = open(tmp.get_url(), 'w')
f.write(stream)
f.close()
# FIXME: remove?
#script_reader = open(tmp.get_url())
feedback = File('feedback.log')
feedback_writer = open(feedback.get_url(), 'w')
# FIXME: rewrite with format
statement = "sqlcmd -b -S {host},{port} -U {user} -P {passwd} -d {db} -i {url}".format(
host=executable['host'],
port=executable.get('port') or 1433,
user=executable['username'],
passwd=executable['password'],
db=executable['database'],
url=tmp.get_url(),
)
call = CallFactory.new_call(statement)
call['stdout'] = feedback_writer
call['stderr'] = feedback_writer
result = Shell.execute(call)
self.set_result(result)
def fail_on_invalid_version(self, connector, executor, version, properties):
plugin_dir = properties.get('plugin.dir')
versions = Versions()
version_loader = VersionLoader(versions)
version_loader.load(properties)
versions.sort()
v = Version(version)
previous = versions.previous(v).get_value()
properties['previous'] = previous
script = File(os.path.join(plugin_dir, 'mysql', 'update', 'checkversion.sql'))
executor['script'] = script
connector.execute(executor, properties)
if "(Code 1329)" in connector.get_result():
raise InvalidVersionException("invalid version: {}".format(previous))
def fail_on_invalid_version(self, properties, connector, executor,
version):
plugin_dir = properties.get('plugin.dir')
versions = Versions()
version_loader = VersionLoader(versions)
version_loader.load(properties)
versions.sort()
v = Version(version)
previous = versions.previous(v).get_value()
properties['previous'] = previous
script = File(
os.path.join(plugin_dir, 'mssql', 'update', 'checkversion.sql'))
executor['script'] = script
connector.execute(executor, properties)
def execute(self, arguments, properties):
host = arguments.h
Fail.fail_on_no_host(host)
self.fail_on_blocked_hosts(host, properties)
default_schemes = properties.get_property('schemes')
schemes = Ora.nvl(arguments.s, default_schemes)
Fail.fail_on_invalid_schema(arguments.s, properties)
default_environment = properties.get_property('default_environment')
environment = Ora.nvl(arguments.e, default_environment)
Fail.fail_on_invalid_environment(arguments.e, properties)
database = properties.get_property('database')
objects = properties.get_property('drop_objects')
# alias = arguments.a
# Fail.fail_on_invalid_alias(arguments.a, properties)
#
# # if an alias is given, only the alias database will be installed, other databases will be ignored.
# if alias:
# print "using alias :" + alias
# databases = [alias]
for schema in schemes:
print "dropping schema '" + schema + "' in database '" + database + "on host '" + host + "' using environment '" + environment + "'"
users = properties.get_property('mssql_users')
username = PropertyHelper.get_mssql_user(users, host, schema)
password = PropertyHelper.get_mssql_password(users, host, schema)
connector = self.get_connector()
executor = {}
executor['host'] = host
executor['database'] = database
executor['schema'] = schema
executor['username'] = username
executor['password'] = password
for object in objects:
folder = File(
os.path.join(self.get_drop_dir(properties), object))
ConnectionExecutor.execute(connector, executor, properties,
folder)
print "schema '" + schema + "' dropped."
def load(self, properties):
alter = File(properties.get_property("alter.dir"))
if alter.exists():
files = Files()
for version in files.list(alter):
self.__versions.add(Version(version.tail()))
create = File(properties.get_property("create.dir"))
if create.exists():
self.__versions.add(
Version(properties.get_property("default_version")))
def testEnvironment(self):
noora_dir = os.path.dirname(noora.__file__)
current_dir = os.path.abspath('.')
properties = dict()
properties["noora.dir"] = noora_dir
properties["current.dir"] = current_dir
executable = {}
executable['script'] = File("exec_get_environment.sql")
executable['host'] = "elsevierdb4.c07v9zv3jhxs.eu-west-1.rds.amazonaws.com,1433"
executable['user'] = "******"
executable['password'] = "******"
executable['database'] = "ElsevierDB"
connector = MssqlConnector()
connector.execute(executable, properties)
def load(self, properties):
"""
Find all versions on the current project and add them to the list.
:param properties: An instance of ``noora.system.Properties.Properties``.
"""
alter = File(properties.get("alter.dir"))
if alter.exists():
files = Files()
for version in files.list(alter):
self.__versions.add(Version(version.tail()))
create = File(properties.get("create.dir"))
if create.exists():
self.__versions.add(Version(properties.get("default_version")))
def __get_config(self):
"""
Load a myproject.json file and update some extra parameters if this is an actual project
"""
current_dir = self.__props.get("current.dir")
project_file = self.__props.get("project.file")
config_file = File(os.path.join(current_dir, project_file))
if not config_file.exists():
noora_dir = self.__props.get("noora.dir")
config_file = File(os.path.join(noora_dir, project_file))
# Read project configuration
with open(config_file.get_url()) as fd:
data = json.load(fd)
return data
def execute(self, properties, arguments):
"""
Create a new database instance for the initial version.
:type properties: system.Properties.Properties
:param properties: The project properties
:type arguments: dict
:param arguments: This dict contains the plugin arguments:
* **host**: The hostname where the database is running;
* **environment**: The environment to create the database in (optional).
"""
prepared_args = self._validate_and_prepare(properties, arguments)
host = prepared_args['host']
environment = prepared_args['environment']
databases = properties.get('databases')
objects = properties.get('create_objects')
# retrieve the user credentials for this database project.
users = properties.get('postgresql_users')
# try to retrieve the users from the credentials file, when no users are configured in
# myproject.json.
if not users:
# retrieve the name of this database project, introduced in version 1.0.12
profile = PropertyHelper.get_profile(properties)
if profile:
users = profile.get('postgresql_users')
# fail when no users are found. This means that they are not set in myproject.json or
# credentials.json
Fail.fail_on_no_users(users)
connector = self.get_connector()
create_dir = properties.get('create.dir')
for database in databases:
print("creating database '{db}' on host '{host}' using environment '{env}'".format(
db=database, host=host, env=environment))
executor = PropertyHelper.get_postgres_properties(users, host, database)
for obj in objects:
# global ddl objects
folder = File(os.path.join(create_dir, database, 'ddl', obj))
ConnectionExecutor.execute(connector, executor, properties, folder)
# environment specific ddl objects
folder = File(os.path.join(create_dir, database, 'ddl', obj, environment))
ConnectionExecutor.execute(connector, executor, properties, folder)
# global dat objects
folder = File(os.path.join(create_dir, database, 'dat'))
ConnectionExecutor.execute(connector, executor, properties, folder)
# environment specific dat objects
folder = File(os.path.join(create_dir, database, 'dat', environment))
ConnectionExecutor.execute(connector, executor, properties, folder)
print("database '{}' created.".format(database))
def execute(self, arguments, properties):
project = ''
version = None
if arguments.v:
version = arguments.v
current_dir = properties.get_property('current.dir')
project_file = properties.get_property('project.file')
config_file = File(os.path.join(current_dir, project_file))
if not config_file.exists():
database = raw_input('database : ')
project = database + "-db"
host = raw_input('host [localhost] : ')
host = Ora.nvl(host, "localhost")
username = raw_input('username : '******'password : '******'version [1.0.0]: ')
version = Ora.nvl(version, "1.0.0")
os.mkdir(project)
template_dir = os.path.join(properties.get_property('plugin.dir'), 'mysql', 'generate', 'templates')
template_file = os.path.join(template_dir, project_file)
f = open(template_file)
stream = f.read()
f.close()
stream = stream.replace('{host}', host)
stream = stream.replace('{database}', database)
stream = stream.replace('{username}', username)
stream = stream.replace('{password}', password)
stream = stream.replace('{version}', version)
config_file = os.path.join(current_dir, project, project_file)
f = open(config_file, 'w')
f.write(stream)
f.close()
properties.set_property('alter.dir', os.path.join(current_dir, project, 'alter'))
properties.set_property('create.dir', os.path.join(current_dir, project, 'create'))
config_file = os.path.join(current_dir, project, project_file)
f = open(config_file)
data = json.load(f)
for key in data.keys():
properties.set_property(key, data[key])
f.close()
versions = Versions()
version_loader = VersionLoader(versions)
version_loader.load(properties)
versions.sort()
version_guesser = VersionGuesser(properties, versions)
next_version = version_guesser.guess(version).to_string()
version_dir = version_guesser.to_folder(next_version)
# create the version folder
os.makedirs(version_dir)
databases = properties.get_property('databases')
version_database = properties.get_property('version_database')
default_version = properties.get_property('default_version')
environments = properties.get_property('environments')
objects = properties.get_property('create_objects')
for database in databases:
# create the scheme folder
database_dir = os.path.join(version_dir, database)
os.mkdir(database_dir)
# create the dat folder
dat_dir = os.path.join(database_dir, 'dat')
os.mkdir(dat_dir)
# create the version script in the dat folder
if database == version_database:
version_file = os.path.join(dat_dir, 'version.sql')
f = open(version_file, 'w')
if next_version == default_version:
stream = properties.get_property('version_insert_statement')
else:
stream = properties.get_property('version_update_statement')
stream = stream.replace('<version>', next_version)
f.write(stream)
f.close()
# sqlScript=self.getSqlVersionStatement(versions, version)
# projectHelper.writeFile(datFolder+os.sep+'version.sql', sqlScript)
# create the environment folders in the dat folder
for environment in environments:
os.mkdir(os.path.join(dat_dir, environment))
# create the environment script in the dat folder.
if database == version_database and next_version == default_version:
environment_file = os.path.join(dat_dir, environment, 'environment.sql')
f = open(environment_file, 'w')
stream = properties.get_property('environment_insert_statement')
stream = stream.replace('<environment>', environment)
f.write(stream)
f.close()
# create the ddl folder
ddl_dir = os.path.join(database_dir, 'ddl')
os.mkdir(ddl_dir)
# create the object folders in the ddl folder
for object in objects:
os.mkdir(os.path.join(ddl_dir, object))
# create the template code on create.
if database == version_database and next_version == default_version:
for object in objects:
object_dir = os.path.join(template_dir, object)
if os.path.exists(object_dir):
files = Files.list(File(object_dir))
for file in files:
shutil.copyfile(file.get_url(), os.path.join(ddl_dir, object, file.tail()))
print "version " + next_version + " created."
def execute(self, arguments, properties):
properties.set_property('create.dir', os.path.join(properties.get_property('current.dir'), 'create'))
properties.set_property('alter.dir', os.path.join(properties.get_property('current.dir'), 'alter'))
version = arguments.v
Fail.fail_on_no_version(version)
Fail.fail_on_unknown_version(version, properties)
default_databases = properties.get_property('databases')
databases = Ora.nvl(arguments.d, default_databases)
Fail.fail_on_invalid_database(arguments.d, properties)
current_dir = properties.get_property('current.dir')
component_name = properties.get_property('component_name')
target_dir = os.path.join(current_dir, properties.get_property('component_target_folder'))
objects = properties.get_property('create_objects')
build_dir = App.build_dir(version, properties)
# exclude the file 'version.sql', this file is excluded from the dat listing below.
component_excluded_files = properties.get_property('component_excluded_files')
excluded_files = properties.get_property('excluded_files')
excluded_files.extend(component_excluded_files)
# create the target folder, if not present.
if not File(target_dir).exists():
os.makedirs(target_dir)
print "building component with version '" + version + "'"
zip_file = os.path.join(target_dir, component_name + '_' + version + '.zip')
zip_handle = ZipFile(zip_file, 'w')
for database in databases:
for object in objects:
if not object == 'lib':
# global ddl objects
folder = File(os.path.join(build_dir, database, 'ddl', object))
zip_dir = File(os.path.join(component_name + '_' + version, 'ddl', object))
files = Files.list_filtered(folder, properties)
for file in files:
print file.get_url()
target_file = File(os.path.join(zip_dir.get_url(), file.tail()))
zip_handle.write(file.get_url(), target_file.get_url(), ZIP_DEFLATED)
# global dat files
folder = File(os.path.join(build_dir, database, 'dat'))
zip_dir = File(os.path.join(component_name + '_' + version, 'dat'))
files = Files.list_filtered(folder, properties)
for file in files:
print file.get_url()
target_file = File(os.path.join(zip_dir.get_url(), file.tail()))
zip_handle.write(file.get_url(), target_file.get_url(), ZIP_DEFLATED)
# create the version script in the dat folder
version_statement = self.version_statement(version, properties)
version_statement = version_statement.replace('<version>', version)
version_statement = version_statement.replace('<name>', component_name)
f = open('version.sql', 'w')
f.write(version_statement)
f.close()
zip_dir = File(os.path.join(component_name + '_' + version, 'dat'))
target_file = File(os.path.join(zip_dir.get_url(), 'version.sql'))
zip_handle.write('version.sql', target_file.get_url())
# remove the version.sql file.
os.remove('version.sql')
# create the checkversion script in the root folder
# first retrieve the previous version for the checkversion script
versions = Versions()
version_loader = VersionLoader(versions)
version_loader.load(properties)
versions.sort()
previous = versions.previous(Version(version))
component_select_statement = properties.get_property("component_select_statement")
component_select_statement = component_select_statement.replace('<name>', component_name)
component_select_statement = component_select_statement.replace('<previous>', previous.get_value())
print component_select_statement
f = open('checkversion.sql', 'w')
f.write(component_select_statement)
f.close()
zip_dir = File(os.path.join(component_name + '_' + version))
target_file = File(os.path.join(zip_dir.get_url(), 'checkversion.sql'))
zip_handle.write('checkversion.sql', target_file.get_url())
# remove the version.sql file.
os.remove('checkversion.sql')
zip_handle.close()
print "component with version " + version + " created."
def execute(self, properties, arguments):
"""
Update database after checking if schema and environment are
valid values. Also check that host is not on the block list and that
the version to update to is valid
:type properties: system.Properties.Properties
:param properties: The project properties
:type arguments: dict
:param arguments: This dict contains the plugin arguments:
* **version**: The version to update the database to;
* **host**: The hostname that hosts the database to update;
* **schema**: Schema to update (optional);
* **environment**: Environment to update the database in (optional).
"""
prepared_args = self._validate_and_prepare(properties, arguments)
version = prepared_args['version']
host = prepared_args['host']
environment = prepared_args['environment']
databases = properties.get('databases')
objects = properties.get('create_objects')
version_database = properties.get('version_database')
alter_dir = properties.get('alter.dir')
# retrieve the user credentials for this database project.
users = properties.get('postgresql_users')
# try to retrieve the users from the credentials file, when no users are configured in
# myproject.json.
if not users:
# retrieve the name of this database project, introduced in version 1.0.12
profile = PropertyHelper.get_profile(properties)
if profile:
users = profile.get('postgresql_users')
connector = self.get_connector()
for database in databases:
print(
"updating database '{db}' on host '{host}' using environment '{env}'"
.format(db=database, host=host, env=environment))
executor = PropertyHelper.get_mysql_properties(
users, host, database)
if database == version_database:
self.fail_on_invalid_environment(connector, executor,
environment, properties)
self.fail_on_invalid_version(connector, executor, version,
properties)
for obj in objects:
# global ddl objects
folder = File(
os.path.join(alter_dir, version, database, 'ddl', obj))
ConnectionExecutor.execute(connector, executor, properties,
folder)
# environment specific ddl objects
folder = File(
os.path.join(alter_dir, version, database, 'ddl', obj,
environment))
ConnectionExecutor.execute(connector, executor, properties,
folder)
# global dat objects
folder = File(os.path.join(alter_dir, version, database, 'dat'))
ConnectionExecutor.execute(connector, executor, properties, folder)
# environment specific dat objects
folder = File(
os.path.join(alter_dir, version, database, 'dat', environment))
ConnectionExecutor.execute(connector, executor, properties, folder)
print("database '{}' updated".format(database))
def testMssqlConnector(self):
noora_dir = os.path.dirname(noora.__file__)
current_dir = os.path.abspath('.')
properties = dict()
properties["noora.dir"] = noora_dir
properties["current.dir"] = current_dir
executable = {}
executable['script'] = File("drop_application_properties_s.sql")
executable['host'] = "elsevierdb4.c07v9zv3jhxs.eu-west-1.rds.amazonaws.com,1433"
executable['user'] = "******"
executable['password'] = "******"
executable['database'] = "ElsevierDB"
connector = MssqlConnector()
connector.execute(executable, properties)
executable = {}
executable['script'] = File("application_properties_s.sql")
executable['host'] = "elsevierdb4.c07v9zv3jhxs.eu-west-1.rds.amazonaws.com,1433"
executable['user'] = "******"
executable['password'] = "******"
executable['database'] = "ElsevierDB"
connector = MssqlConnector()
connector.execute(executable, properties)
executable = {}
executable['script'] = File("drop_application_properties.sql")
executable['host'] = "elsevierdb4.c07v9zv3jhxs.eu-west-1.rds.amazonaws.com,1433"
executable['user'] = "******"
executable['password'] = "******"
executable['database'] = "ElsevierDB"
connector = MssqlConnector()
connector.execute(executable, properties)
executable = {}
executable['script'] = File("application_properties.sql")
executable['host'] = "elsevierdb4.c07v9zv3jhxs.eu-west-1.rds.amazonaws.com,1433"
executable['user'] = "******"
executable['password'] = "******"
executable['database'] = "ElsevierDB"
connector = MssqlConnector()
connector.execute(executable, properties)
executable = {}
executable['script'] = File("drop_app_prop.sql")
executable['host'] = "elsevierdb4.c07v9zv3jhxs.eu-west-1.rds.amazonaws.com,1433"
executable['user'] = "******" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
"" \
""
executable['password'] = "******"
executable['database'] = "ElsevierDB"
connector = MssqlConnector()
connector.execute(executable, properties)
executable = {}
executable['script'] = File("app_prop.sql")
executable['host'] = "elsevierdb4.c07v9zv3jhxs.eu-west-1.rds.amazonaws.com,1433"
executable['user'] = "******"
executable['password'] = "******"
executable['database'] = "ElsevierDB"
connector = MssqlConnector()
connector.execute(executable, properties)
def execute(self, properties, arguments):
"""
Build the package after verifying version is valid, and database,
if provided.
:param properties: The project properties
:param arguments: A dict of {
'version': 'The version to package for'
'database': 'The database to package for (optional)'
}
"""
prepared_args = self._validate_and_prepare(properties, arguments)
version = prepared_args['version']
databases = prepared_args['databases']
current_dir = properties.get('current.dir')
component_name = properties.get('component_name')
target_dir = os.path.join(current_dir, properties.get('component_target_folder'))
objects = properties.get('create_objects')
if version == properties.get("default_version"):
build_dir = properties.get("create.dir")
else:
build_dir = os.path.join(properties.get("alter.dir"), version)
# exclude the file 'version.sql', this file is excluded from the dat listing below.
component_excluded_files = properties.get('component_excluded_files')
excluded_files = properties.get('excluded_files')
excluded_files.extend(component_excluded_files)
# create the target folder, if not present.
if not File(target_dir).exists():
os.makedirs(target_dir)
print("building component with version '{}'".format(version))
zip_file = os.path.join(target_dir, component_name + '_' + version + '.zip')
zip_handle = ZipFile(zip_file, 'w')
for database in databases:
for obj in objects:
if not obj == 'lib':
# global ddl objects
folder = File(os.path.join(build_dir, database, 'ddl', obj))
zip_dir = File(os.path.join(component_name + '_' + version, 'ddl', obj))
files = Files.list_filtered(folder, properties)
for file in files:
print(file.get_url())
target_file = File(os.path.join(zip_dir.get_url(), file.tail()))
zip_handle.write(file.get_url(), target_file.get_url(), ZIP_DEFLATED)
# global dat files
folder = File(os.path.join(build_dir, database, 'dat'))
zip_dir = File(os.path.join(component_name + '_' + version, 'dat'))
files = Files.list_filtered(folder, properties)
for file in files:
print(file.get_url())
target_file = File(os.path.join(zip_dir.get_url(), file.tail()))
zip_handle.write(file.get_url(), target_file.get_url(), ZIP_DEFLATED)
# create the version script in the dat folder
if version == properties.get("default_version"):
version_statement = properties.get("component_insert_statement")
else:
version_statement = properties.get("component_update_statement")
version_statement = version_statement.replace('<version>', version)
version_statement = version_statement.replace('<name>', component_name)
f = open('version.sql', 'w')
f.write(version_statement)
f.close()
zip_dir = File(os.path.join(component_name + '_' + version, 'dat'))
target_file = File(os.path.join(zip_dir.get_url(), 'version.sql'))
zip_handle.write('version.sql', target_file.get_url())
# remove the version.sql file.
os.remove('version.sql')
# create the checkversion script in the root folder
# first retrieve the previous version for the checkversion script
versions = Versions()
version_loader = VersionLoader(versions)
version_loader.load(properties)
versions.sort()
previous = versions.previous(Version(version))
component_select_statement = properties.get("component_select_statement")
component_select_statement = component_select_statement.replace('<name>', component_name)
component_select_statement = component_select_statement.replace(
'<previous>', previous.get_value())
print(component_select_statement)
f = open('checkversion.sql', 'w')
f.write(component_select_statement)
f.close()
zip_dir = File(os.path.join(component_name + '_' + version))
target_file = File(os.path.join(zip_dir.get_url(), 'checkversion.sql'))
zip_handle.write('checkversion.sql', target_file.get_url())
# remove the version.sql file.
os.remove('checkversion.sql')
zip_handle.close()
print("component with version {} created.".format(version))
def fail_on_unknown_version(self, version, properties):
alter_dir = properties.get_property('alter.dir')
version_dir = File(os.path.join(alter_dir, version))
if not version_dir.exists():
raise UnknownVersionException("unknown version folder", version)
def execute(self, arguments, properties):
properties.set_property(
'create.dir',
os.path.join(properties.get_property('current.dir'), 'create'))
properties.set_property(
'alter.dir',
os.path.join(properties.get_property('current.dir'), 'alter'))
host = arguments.h
Fail.fail_on_no_host(host)
version = arguments.v
Fail.fail_on_no_version(version)
default_databases = properties.get_property('databases')
databases = Ora.nvl(arguments.d, default_databases)
Fail.fail_on_invalid_database(arguments.d, properties)
default_environment = properties.get_property('default_environment')
environment = Ora.nvl(arguments.e, default_environment)
Fail.fail_on_invalid_environment(arguments.e, properties)
objects = properties.get_property('create_objects')
version_database = properties.get_property('version_database')
alter_dir = properties.get_property('alter.dir')
self.fail_on_unknown_version(version, properties)
alias = arguments.a
database_aliases = properties.get_property('database_aliases')
Fail.fail_on_invalid_alias(alias, properties)
# if an alias is given, only this database will be installed, other databases will be ignored.
if alias:
print "using alias :" + alias
databases = [alias]
connector = self.get_connector()
for database in databases:
print "updating database '" + database + "' on host '" + host + "' using environment '" + environment + "'"
users = properties.get_property('mysql_users')
username = PropertyHelper.get_mysql_user(users, host, database)
password = PropertyHelper.get_mysql_passwd(users, host, database)
executor = {}
executor['host'] = host
executor['database'] = database
executor['username'] = username
executor['password'] = password
database_folder = PropertyHelper.get_database_folder(
database, database_aliases)
if database == version_database:
self.fail_on_invalid_environment(connector, executor,
environment, properties)
self.fail_on_invalid_version(connector, executor, version,
properties)
for object in objects:
# global ddl objects
folder = File(
os.path.join(alter_dir, version, database_folder, 'ddl',
object))
ConnectionExecutor.execute(connector, executor, properties,
folder)
# environment specific ddl objects
folder = File(
os.path.join(alter_dir, version, database_folder, 'ddl',
object, environment))
ConnectionExecutor.execute(connector, executor, properties,
folder)
# global dat objects
folder = File(
os.path.join(alter_dir, version, database_folder, 'dat'))
ConnectionExecutor.execute(connector, executor, properties, folder)
# environment specific dat objects
folder = File(
os.path.join(alter_dir, version, database_folder, 'dat',
environment))
ConnectionExecutor.execute(connector, executor, properties, folder)
print "database '" + database + "' updated."
