class EnvList:
"""
Can list all the envs and apps.
"""
logger = Logger("EnvList")
def __init__(self, script_settings):
self.script_settings = script_settings
def list_envs(self):
self.logger.info('[%s]' % ', '.join(map(str, self.get_list_of_envs())))
def get_list_of_envs(self):
envs = os.listdir(self.script_settings.get_env_configs_dir())
envs.remove("global-configuration.cfg")
return envs
def list_projects(self, env):
projects = os.listdir(
os.path.join(self.script_settings.get_env_configs_dir(), env))
return [
project.replace(".cfg", "") for project in projects
if project.endswith(".cfg")
]
def print_envs_with_projects(self):
for env in self.list_envs():
print env
for project in self.list_projects(env):
print " |_" + project
class EnvList:
""" Can list all the envs and apps. """
logger = Logger("EnvList")
def __init__(self, global_configs_dir):
""" :param str global_configs_dir: the environments/ dir """
self.global_configs_dir = global_configs_dir
def list_envs(self):
self.logger.info('[%s]' % ', '.join(map(str, self.get_list_of_envs())))
def get_list_of_envs(self):
envs = os.listdir(self.global_configs_dir
) # dir contains folders with envs configuration
global_configuration_file = 'global-configuration.cfg'
if global_configuration_file in envs:
envs.remove(global_configuration_file)
return envs
def list_projects(self, env):
projects = os.listdir(os.path.join(self.global_configs_dir, env))
if ScriptSettings.ENV_CONFIG_FILE_NAME in projects:
projects.remove(ScriptSettings.ENV_CONFIG_FILE_NAME)
return [
project.replace(".cfg", "") for project in projects
if project.endswith(".cfg")
]
def print_envs_with_projects(self):
for env in self.get_list_of_envs():
print env
for project in self.list_projects(env):
print " |_" + project
class DeployCommand:
NEXUS_URL = 'http://repo.jtalks.org/content/repositories/deployment-pipeline/deployment-pipeline/'
logger = Logger('DeployCommand')
def __init__(self, jtalks_artifacts, old_nexus, tomcat, sanity_test,
backuper, scriptsettings):
"""
:param jtalks.ScriptSettings.ScriptSettings scriptsettings: config files to be deployed along with the app
:param jtalks.Nexus.JtalksArtifacts jtalks_artifacts: downloads JTalks artifacts from Nexus
:param jtalks.OldNexus.Nexus old_nexus: used to download artifacts if it's old and is kept in old repo
:param jtalks.Tomcat.Tomcat tomcat: manages tomcat
:param jtalks.sanity.SanityTest.SanityTest sanity_test: runs the tests after an app is deployed
:param jtalks.backup.Backuper.Backuper backuper: backs up DB, tomcat
"""
self.scriptsettings = scriptsettings
self.tomcat = tomcat
self.jtalks_artifacts = jtalks_artifacts
self.sanity_test = sanity_test
self.old_nexus = old_nexus
self.backuper = backuper
def deploy(self, project, build, app_final_name, plugins=[]):
self.__validate_params_and_raise__(project, build)
plugin_files = []
try:
gav, filename = self.jtalks_artifacts.download_war(project, build)
plugin_files = self.jtalks_artifacts.download_plugins(
project, gav.version, plugins)
except BuildNotFoundException:
if project == 'jcommune' and len(plugins) != 0:
self.logger.warn(
'Looks like a pretty old build is requested, plugins will not be installed for it '
'even though they were requested - at those times we did not upload plugins to '
'binary storage so they are not saved. To get plugins you would either need to find'
' a required revision and build them from source. Sorry for that. But hey, this is'
' a very old build of JCommune, use a newer version!')
filename = self.old_nexus.download_war(project)
self.tomcat.stop()
self.backuper.back_up_dir(self.tomcat.tomcat_location)
self.backuper.back_up_db()
self.tomcat.move_to_webapps(filename, app_final_name)
if project == 'jcommune':
self.jtalks_artifacts.deploy_plugins(
self.scriptsettings.get_plugins_dir(), plugin_files)
self.scriptsettings.deploy_configs()
self.tomcat.start()
self.sanity_test.check_app_started_correctly()
def __validate_params_and_raise__(self, project, build):
if build is None:
self.logger.error(
'Build number was not specified, see [{0}] to get list of builds',
self.NEXUS_URL)
raise RuntimeError
if project not in ['jcommune', 'poulpe', 'antarcticle']:
self.logger.error(
'A correct project should be specified: [poulpe, jcommune, antarcticle]. Actual: [{0}]',
project)
raise RuntimeError
class Main:
def __init__(self):
self.logger = Logger("Main")
def main(self, args, options):
command = args[0]
if command == "version":
print __version__
exit(0)
script_settings = ScriptSettings(options)
app_context = ApplicationContext(script_settings)
if script_settings.grab_envs == "true":
app_context.environment_config_grabber().grab_jtalks_configs()
# recreating them after configs were updated
script_settings = ScriptSettings(options)
app_context = ApplicationContext(script_settings)
try:
if command == "deploy":
LibVersion().log_lib_versions()
app_context.deploy_command().deploy(
script_settings.project,
script_settings.build,
script_settings.get_app_final_name(),
script_settings.get_plugins(),
)
elif command == "upload-to-nexus":
LibVersion().log_lib_versions()
app_context.old_nexus().upload_war("pom.xml")
elif command == "list-envs":
app_context.env_list().list_envs()
elif command == "load-db-from-backup":
LibVersion().log_lib_versions()
app_context.load_db_from_backup().load()
else:
error = (
"Command was not recognized, you can use: deploy, list-envs, load-db-from-backup. "
"Also see jtalks -h"
)
self.logger.error(error)
raise RuntimeError(error)
except:
self.logger.error("Program finished with errors")
if options.debug:
print ("Root cause: %s" % traceback.format_exc())
sys.exit(1)
class SanityTest:
"""
Tests that application was deployed correctly without errors. For these purposes it opens some pages and determines
whether they return HTML. If let's say they return HTTP 500, then the test failed. It has to break CI builds.
"""
HOST = "127.0.0.1"
logger = Logger("SanityTest")
port = None
app_name = None
def __init__(self, tomcat_port, app_name, sanity_test_timeout_sec=120, sleep_sec=30):
"""
@param tomcat_port - an HTTP port to access the web server where application is
@param sleep_sec - the amount of time tests ignore error responses as deployment failure. This is needed because
first tomcat may not start quickly and therefore the response Connection Refused will be immediate. Thus when we
send requests, first we should treat error messages as possible responses. After this sleep time error responses
are considered as failed deployment.
"""
self.port = int(tomcat_port)
self.app_name = app_name
self.sanity_test_timeout_sec = sanity_test_timeout_sec
self.sleep_sec = sleep_sec
if app_name == "ROOT":
self.app_name = ""
def check_app_started_correctly(self):
request_address = "http://{0}:{1}/{2}".format(self.HOST, self.port, self.app_name)
tests_sleep_end = datetime.datetime.now() + datetime.timedelta(seconds=self.sleep_sec)
response = None
attempt_counter = 0
while tests_sleep_end > datetime.datetime.now():
attempt_counter += 1
self.logger.info(
"[Attempt #{0}] Running sanity tests to check whether application started correctly and responds back..",
attempt_counter)
self.logger.info("[Attempt #{0}] Connecting to {1}", attempt_counter, request_address)
try:
response = requests.get(request_address, timeout=self.sanity_test_timeout_sec)
except ConnectionError:
self.logger.info("Sleeping for 5 sec..")
sleep(5) # so that we don't connect too often
continue
if response.status_code in [200, 201]:
break
else:
self.logger.info("Error response, got {0} HTTP status. App server might still be booting.",
response.status_code)
if not response or response.status_code not in [200, 201]:
self.logger.error('After {0} no successful response was received from the app. Finishing by timeout {1}',
attempt_counter, self.sanity_test_timeout_sec)
if response:
self.logger.error("Last time while accessing main page, app answered with error: [{0} {1} {2}]",
response.status_code, response.reason, response.text)
else:
self.logger.error("App Server did not even get up!")
raise SanityCheckFailedException("Sanity check failed")
self.logger.info("Sanity check passed: [{0} {1}]", response.status_code, response.reason)
class EnvironmentConfigGrabber:
logger = Logger("EnvironmentConfigGrabber")
def __init__(self, env_configs_root, temp_dir):
self.env_configs_root = env_configs_root
self.temp_dir = temp_dir
self.clone_repo_to = temp_dir + 'environments'
self.grabbed_configs_location = self.clone_repo_to + "/configs"
def grab_jtalks_configs(self):
try:
self.__remove_previous_git_folder__()
self.__create_jtalks_temp_dir__()
repo.Repo.clone_from('[email protected]:environments',
self.clone_repo_to)
self.__copy_grabbed_configs_into_work_dir__()
except GitCommandError:
self.logger.warn(
"You don't have access to JTalks repo with environment configs. You may want to use your own "
+
"local environment. Create a folder with env name in configs directory. If you think you "
+
"need access to JTalks internal envs (including UAT, DEV, PREPROD, PROD envs), "
+ "you should write a request to [email protected].")
def __remove_previous_git_folder__(self):
try:
if os.path.exists(self.clone_repo_to):
self.logger.info("Removing {0} directory if it was there",
self.clone_repo_to)
shutil.rmtree(self.clone_repo_to)
except OSError as e:
if e.errno is not 2: #No such file or directory
self.logger.warn(e.message)
def __copy_grabbed_configs_into_work_dir__(self):
grabbed_dirs_and_files = os.listdir(self.grabbed_configs_location)
for next_grabbed_file_or_dir in grabbed_dirs_and_files:
destination_file = self.env_configs_root + next_grabbed_file_or_dir
if os.path.exists(destination_file):
self.logger.info(
"{0} will be overwritten by newer version from git repo",
destination_file)
self.__delete_file_or_dir__(destination_file)
shutil.move(
self.grabbed_configs_location + "/" + next_grabbed_file_or_dir,
destination_file)
def __delete_file_or_dir__(self, destination_file):
if os.path.isfile(destination_file):
os.remove(destination_file)
else:
shutil.rmtree(destination_file)
def __create_jtalks_temp_dir__(self):
if not os.path.exists(self.temp_dir):
os.mkdir(self.temp_dir)
class DbOperations:
""" Base class for working with database """
logger = Logger("DbBase")
def __init__(self, dbsettings):
""" :param jtalks.ScripSettings.DbSettings dbsettings: settings """
self.dbsettings = dbsettings
def connect_to_database(self):
""" Connects to the specified database """
self.logger.info("Connecting to [{0}] with user [{1}]",
self.dbsettings.host, self.dbsettings.user)
self.connection = MySQLdb.connect(host=self.dbsettings.host,
user=self.dbsettings.user,
passwd=self.dbsettings.password)
self.cursor = self.connection.cursor()
def close_connection(self):
""" Closes open connection to the database """
self.connection.close()
def recreate_database(self):
""" Deletes the specified database and create it again """
self.logger.info("Dropping and creating database from scratch: [{0}]",
self.dbsettings.name)
self.cursor.execute('DROP DATABASE IF EXISTS ' + self.dbsettings.name)
self.cursor.execute('CREATE DATABASE ' + self.dbsettings.name)
def backup_database(self, backup_path):
""" Backups database to given backupPath """
self.logger.info("Backing up database [{0}] to [{1}] using user [{2}]",
self.dbsettings.name, backup_path,
self.dbsettings.user)
dump_command = "mysqldump -u'{0}'".format(self.dbsettings.user)
if self.dbsettings.password:
dump_command += " -p'{0}'".format(self.dbsettings.password)
dump_command += " '{0}' > '{1}/{0}.sql'".format(
self.dbsettings.name, backup_path)
self.logger.info("Dumping DB: [{0}]",
dump_command.replace(self.dbsettings.password, "***"))
os.popen(
dump_command.format(self.dbsettings.user, self.dbsettings.password,
self.dbsettings.name, backup_path)).read()
self.logger.info("Database backed up [{0}]".format(
self.dbsettings.name))
def restore_database_from_file(self, backupPath):
""" Restores database from file specified in backupPath """
self.logger.info("Loading a db dump from [{0}/{1}.sql]", backupPath,
self.dbsettings.name)
self.recreate_database()
os.popen("mysql -u'{0}' -p'{1}' '{2}' < '{3}/{2}.sql'".format(
self.dbsettings.user, self.dbsettings.password,
self.dbsettings.name, backupPath)).read()
self.logger.info("Database restored [{0}]".format(
self.dbsettings.name))
class SSH:
env = None
config = None
sftpTransport = None
sftpClient = None
sftpHost = None
sftpPort = None
sftpUser = None
sftpPass = None
sftpBackupArchive = None
sftpBackupFileName = None
logger = Logger("SSH")
def __init__(self, script_settings):
self.env = script_settings.env
self.config = ConfigParser.ConfigParser()
self.config.read(script_settings.get_env_configs_dir() + self.env +
"/ssh.cfg")
self.sftpHost = self.config.get('sftp', 'sftp_host')
self.sftpPort = self.config.get('sftp', 'sftp_port')
self.sftpUser = self.config.get('sftp', 'sftp_user')
self.sftpPass = self.config.get('sftp', 'sftp_pass')
self.sftpBackupArchive = self.config.get('sftp', 'sftp_backup_archive')
self.sftpBackupFileName = self.config.get('sftp',
'sftp_backup_filename')
def download_backup_prod_db(self):
self.sftp_connection()
attrList = self.sftpClient.listdir_attr('.')
attrTimeTmp = 0
dirWithLastUpdate = None
for attr in attrList:
if attrTimeTmp < attr.st_mtime:
attrTimeTmp = attr.st_mtime
dirWithLastUpdate = attr.filename
self.sftpClient.get(dirWithLastUpdate + '/' + self.sftpBackupArchive,
'./' + self.sftpBackupArchive)
os.popen('bunzip2 -d ' + self.sftpBackupArchive)
self.sftp_connection_close()
def sftp_connection_close(self):
self.sftpClient.close()
self.sftpTransport.close()
def sftp_connection(self):
self.logger.info("Getting a DB backup from {0}", self.sftpHost)
self.sftpTransport = paramiko.Transport(
(self.sftpHost, int(self.sftpPort)))
self.sftpTransport.connect(username=self.sftpUser,
password=self.sftpPass)
self.sftpClient = paramiko.SFTPClient.from_transport(
self.sftpTransport)
def remove_backup_prod_db_file(self):
os.popen('rm -rf ' + self.sftpBackupFileName)
class Main:
def __init__(self):
self.logger = Logger('Main')
def main(self, args, options):
command = args[0]
if command == 'version':
print __version__
exit(0)
script_settings = ScriptSettings(options)
app_context = ApplicationContext(script_settings)
if script_settings.grab_envs == "true":
app_context.environment_config_grabber().grab_jtalks_configs()
# recreating them after configs were updated
script_settings = ScriptSettings(options)
app_context = ApplicationContext(script_settings)
try:
if command == 'deploy':
LibVersion().log_lib_versions()
app_context.deploy_command().deploy(
script_settings.project, script_settings.build,
script_settings.get_app_final_name(),
script_settings.get_plugins())
elif command == "upload-to-nexus":
LibVersion().log_lib_versions()
app_context.old_nexus().upload_war('pom.xml')
elif command == "list-envs":
app_context.env_list().list_envs()
elif command == 'load-db-from-backup':
LibVersion().log_lib_versions()
app_context.load_db_from_backup().load()
else:
error = 'Command was not recognized, you can use: deploy, list-envs, load-db-from-backup. ' \
'Also see jtalks -h'
self.logger.error(error)
raise RuntimeError(error)
except:
self.logger.error("Program finished with errors")
if options.debug:
print("Root cause: %s" % traceback.format_exc())
sys.exit(1)
class LibVersion:
logger = Logger("LibVersions")
def log_lib_versions(self):
self.logger.info("python={0}", sys.version_info)
self.__log_lib_version("requests")
self.__log_lib_version("GitPython")
self.__log_lib_version("mock")
def __log_lib_version(self, libname):
self.logger.info("{0}={1}", libname,
pkg_resources.get_distribution(libname).version)
class Nexus:
"""
A class to work with Nexus (upload, download, search). Note, that for all the operations we need a build number
to be passed into the constructor. Other properties may or may not be required (some of them can be determined,
like project name and version by pom.xml).
"""
base_url = "http://repo.jtalks.org/content/repositories/deployment-pipeline/"
logger = Logger("OldNexus")
def __init__(self, build_number):
self.build_number = build_number
def upload_war(self, pom_file_location):
"""
Uploads a war to the Nexus. The path to pom.xml is acceptaced as an argument, by this path
we also determean where war file is placed.
"""
pom = PomFile(pom_file_location)
artifact_version = pom.version()
artifact_id = pom.artifact_id()
maven_deploy_command = (
"mvn deploy:deploy-file -Durl={2} " +
"-DrepositoryId=deployment-pipeline -DgroupId=deployment-pipeline -DartifactId={0} -Dpackaging=war "
+ "-Dfile={0}-view/{0}-web-view/target/{0}.war -Dversion={1}"
).format(artifact_id, artifact_version, self.base_url)
print maven_deploy_command
return_code = os.system(maven_deploy_command)
if return_code != 0:
self.logger.error("Maven returned error code: " + str(return_code))
raise Exception("Maven returned error code: " + str(return_code))
def download_war(self, project):
self.logger.info("Looking up build #{0} for {1} project",
self.build_number, project)
war_url = self.get_war_url(project, self.build_number)
self.logger.info("Downloading artifact: [{0}]", war_url)
urllib.urlretrieve(war_url, project + ".war")
return project + '.war'
def get_war_url(self, project, build_number):
group_id = "deployment-pipeline/"
artifact_version_url = NexusPageWithVersions().parse(
self.base_url + group_id + project).version(build_number)
# get version by URL (last part is something like /jcommune/12.3.123/)
artifact_version = artifact_version_url.rpartition(project +
"/")[2].replace(
"/", "")
return artifact_version_url + '{0}-{1}.war'.format(
project, artifact_version)
class TomcatServerXml:
"""
Parses $TOMCAT_HOME/conf/server.xml file and returns attributes and tag values.
"""
logger = Logger("TomcatServerXml")
HTTP_PROTOCOL = "HTTP/1.1"
tree = None
def __init__(self, xml_element):
"""
Creates a file from xml element object
@param xml_element server.xml already parsed into xml.etree.Element
"""
self.tree = xml_element
@staticmethod
def fromfile(filename):
"""
Parses the specified server.xml
"""
Logger("TomcatServerXml").info("Parsing [{0}] to obtain information about Tomcat", filename)
return TomcatServerXml(ElementTree.parse(filename))
@staticmethod
def fromstring(file_content):
"""
Creates object from string instead of from file
"""
return TomcatServerXml(ElementTree.fromstring(file_content))
def http_port(self):
"""
Parses server.xml and obtains a Connector with protocol="HTTP/1.1". Returns its port.
"""
http_connectors = self.tree.findall("./Service/Connector")
http_connector = None
for connector in http_connectors:
if connector.get("protocol") == self.HTTP_PROTOCOL:
http_connector = connector
break
if http_connector is None:
error_message = "$TOMCAT_HOME/conf/server.xml didn't contain Connector with HTTP/1.1 protocol"
self.logger.error(error_message)
raise WrongConfigException(error_message)
return http_connector.get("port")
class Backuper:
"""
Responsible for backing up artifacts like war files, config files, as well as DB backups.
Stores backups in the folder and allows to restore those backups from it.
"""
logger = Logger("Backuper")
BACKUP_FOLDER_DATE_FORMAT = "%Y_%m_%dT%H_%M_%S"
def __init__(self, root_backup_folder, db_operations):
"""
:param str root_backup_folder: a directory to put backups to, it may contain backups from different envs,
thus a folder for each env will be created
:param jtalks.db.DbOperations.DbOperations db_operations: instance of DbOperations
"""
self.backup_folder = self.create_folder_to_backup(root_backup_folder)
self.db_operations = db_operations
def create_folder_to_backup(self, backup_folder):
"""
We don't just put backups into a backup folder, we're creating a new folder there
with current date so that our previous backups are kept there. E.g.:
'/var/backups/prod/20130302T195924'
:param str backup_folder: the root folder to create folders with datetime names for each backup
"""
now = datetime.now().strftime(self.BACKUP_FOLDER_DATE_FORMAT)
final_backup_folder = os.path.join(backup_folder, now)
if not os.path.exists(final_backup_folder):
self.logger.info("Creating a folder to store back ups: [{0}]", final_backup_folder)
os.makedirs(final_backup_folder)
return final_backup_folder
def back_up_dir(self, folder_path):
if os.path.exists(folder_path):
head, folder_name = os.path.split(folder_path)
backup_dst = os.path.join(self.backup_folder, folder_name)
self.logger.info('Backing up [{0}] to [{1}]', folder_path, backup_dst)
shutil.copytree(folder_path, backup_dst)
else:
backup_dst = None
self.logger.info("There was no previous folder [{0}], so nothing to backup", folder_path)
return backup_dst
def back_up_db(self):
self.db_operations.backup_database(self.backup_folder)
class DbSettings:
"""
Class keeping connection settings to the database
"""
logger = Logger("DbSettings")
def __init__(self, project, config_file_location):
"""
Creates connection settings object with given project name
"""
self.dbHost = None
self.dbUser = None
self.dbPass = None
self.dbName = None
self.dbPort = None
self.project = project.upper()
self.parse_config(config_file_location)
def parse_config(self, config_file_path):
"""
Parses given config file and gets information about connection settings.
"""
if not os.path.exists(config_file_path):
self.logger.error("Config file not found: [{0}]", config_file_path)
raise ValueError("Config file not found: " + config_file_path)
configs_doc = parse(config_file_path)
env_elements = configs_doc.getElementsByTagName('Environment')
for element in env_elements:
name = element.getAttribute("name")
value = element.getAttribute("value")
if name == (self.project + "_DB_USER"):
self.dbUser = value
elif name == (self.project + "_DB_PASSWORD"):
self.dbPass = value
elif name == (self.project + "_DB_URL"):
jdbc_pattern = 'mysql://(.*?):?(\d*)/(.*?)\?'
(host, port,
database) = re.compile(jdbc_pattern).findall(value)[0]
self.dbHost = host
self.dbName = database
self.dbPort = port
class Nexus:
"""
A class to work with Nexus (upload, download, search). Note, that for all the operations we need a build number
to be passed into the constructor. Other properties may or may not be required (some of them can be determined,
like project name and version by pom.xml).
"""
logger = Logger("Nexus")
def __init__(self, nexus='http://repo.jtalks.org/content/repositories/'):
self.nexus_url = nexus
def download(self, repo, gav, tofile_path):
"""
:param repo - str
:param gav - Gav
:param tofile_path str
"""
url = self.nexus_url + repo + '/' + gav.to_repo_path()
self.logger.info('Downloading artifact: [{0}]', url)
urllib.urlretrieve(url, tofile_path)
class DeployToTomcatFacade:
NEXUS_URL = "http://repo.jtalks.org/content/repositories/deployment-pipeline/deployment-pipeline/"
logger = Logger("DeployToTomcatFacade")
def __init__(self, application_context):
self.app_context = application_context
def deploy(self):
script_settings = self.app_context.script_settings
self.__raise_if_settings_not_specified__(script_settings)
if script_settings.grab_envs == "true":
self.app_context.environment_config_grabber().grab_jtalks_configs()
self.app_context.nexus().download_war(project=self.app_context.script_settings.project)
self.app_context.tomcat().deploy_war()
self.app_context.sanity_test().check_app_started_correctly()
def __raise_if_settings_not_specified__(self, script_settings):
if script_settings.build is None:
self.logger.error("Build number was not specified, see [{0}] to get list of builds", self.NEXUS_URL)
raise RuntimeError
if script_settings.project not in ["jcommune", "poulpe", "antarcticle"]:
self.logger.error("A correct project should be specified: [poulpe, jcommune, antarcticle]")
raise RuntimeError
def fromfile(filename):
"""
Parses the specified server.xml
"""
Logger("TomcatServerXml").info("Parsing [{0}] to obtain information about Tomcat", filename)
return TomcatServerXml(ElementTree.parse(filename))
class Tomcat:
""" Class for deploying and backing up Tomcat applications """
logger = Logger("Tomcat")
def __init__(self, tomcat_location):
""" :param str tomcat_location: location of the tomcat root dir """
self.tomcat_location = tomcat_location
if self.tomcat_location and os.path.exists(tomcat_location):
self.logger.info('Tomcat location: [{0}]', tomcat_location)
else:
self.logger.warn(
'Tomcat location was not set or it does not exist: [{0}]',
tomcat_location)
def stop(self):
""" Stops the Tomcat server if it is running """
if not os.path.exists(self.tomcat_location):
raise TomcatNotFoundException(
'Could not found tomcat: [{0}]'.format(self.tomcat_location))
stop_command = 'pkill -9 -f {0}'.format(
os.path.abspath(self.tomcat_location))
self.logger.info('Killing tomcat [{0}]', stop_command)
# dunno why but return code always equals to SIGNAL (-9 in this case), didn't figure out how to
# distinguish errors from this
subprocess.call([stop_command], shell=True, stdout=PIPE, stderr=PIPE)
def start(self):
""" Starts the Tomcat server """
if not os.path.exists(self.tomcat_location):
raise TomcatNotFoundException(
'Could not found tomcat: [{0}]'.format(self.tomcat_location))
startup_file = self.tomcat_location + "/bin/startup.sh"
self.logger.info("Starting Tomcat [{0}]", startup_file)
pipe = subprocess.Popen(['/bin/bash', startup_file],
shell=False,
stdout=PIPE,
stderr=PIPE)
out, err = pipe.communicate()
if pipe.returncode != 0:
error = 'Could not start Tomcat, return code: {0}'.format(
pipe.returncode)
self.logger.error(error)
self.logger.error(out)
self.logger.error(err)
raise CouldNotStartTomcatException(error)
def move_to_webapps(self, src_filepath, appname):
"""
Moves application war-file to 'webapps' Tomcat sub-folder
:param str src_filepath: to get artifact from
:param str appname: the name of the webapp to be deployed
"""
webapps_location = os.path.join(self.tomcat_location, 'webapps')
final_app_location = os.path.join(webapps_location, appname)
self.logger.info('Putting new war file to Tomcat: [{0}]',
final_app_location)
if not os.path.exists(webapps_location):
error = 'Tomcat webapps folder was not found in [{0}], configuration must have been wrong. ' \
'Please configure correct Tomcat location. Current location contains: {1}' \
.format(self.tomcat_location, os.listdir(self.tomcat_location))
self.logger.error(error)
raise TomcatNotFoundException(error)
self._remove_previous_app(final_app_location)
shutil.move(src_filepath, final_app_location + '.war')
return final_app_location + '.war'
def _remove_previous_app(self, app_location):
if os.path.exists(app_location):
self.logger.info("Removing previous app: [{0}]", app_location)
shutil.rmtree(app_location)
else:
self.logger.info(
"Previous application was not found in [{0}], thus nothing to remove",
app_location)
war_location = app_location + ".war"
if os.path.exists(war_location):
self.logger.info("Removing previous war file: [{0}]", war_location)
os.remove(war_location)
class ScriptSettings:
logger = Logger("ScriptSettings")
TEMP_DIR_NAME = 'temp'
BACKUPS_DIR_NAME = 'backups'
ENVS_DIR_NAME = 'environments'
GLOBAL_ENV_CONFIG_FILE_NAME = 'global-configuration.cfg'
ENV_CONFIG_FILE_NAME = 'environment-configuration.cfg'
def __init__(self,
options_passed_to_script,
workdir=os.path.expanduser('~/.jtalks')):
""" :param optparse.Values options_passed_to_script: thins that are passed with -f or --flags """
self.env = options_passed_to_script.env
self.build = options_passed_to_script.build
self.project = options_passed_to_script.project
self.grab_envs = options_passed_to_script.grab_envs
self.sanity_test_timeout_sec = int(
options_passed_to_script.sanity_test_timeout_sec)
self.package_version = __version__
self.work_dir = workdir
self.temp_dir = os.path.join(self.work_dir, self.TEMP_DIR_NAME)
self.backups_dir = os.path.join(self.work_dir, self.BACKUPS_DIR_NAME)
self.global_configs_dir = os.path.join(self.work_dir,
self.ENVS_DIR_NAME)
self.env_configs_dir = os.path.join(self.global_configs_dir, self.env)
self.global_config_path = os.path.join(
self.global_configs_dir, self.GLOBAL_ENV_CONFIG_FILE_NAME)
self.env_config_path = os.path.join(self.env_configs_dir,
self.ENV_CONFIG_FILE_NAME)
self.project_config_path = os.path.join(self.env_configs_dir,
self.project + '.cfg')
self.props = self._read_properties()
def log_settings(self):
self.logger.info(
'Script Settings: project=[{0}], env=[{1}], build number=[{2}], sanity test timeout=[{3}], '
'package version=[{4}]', self.project, self.env, self.build,
self.sanity_test_timeout_sec, self.package_version)
self.logger.info("Environment configuration: [{0}]",
self.env_configs_dir)
def create_work_dir_if_absent(self):
self._create_dir_if_absent(self.work_dir)
self._create_dir_if_absent(self.env_configs_dir)
self._create_dir_if_absent(self.backups_dir)
def _create_dir_if_absent(self, directory):
if not os.path.exists(directory):
self.logger.info("Creating directory [{0}]", directory)
os.makedirs(directory)
def get_tomcat_port(self):
return int(self.props.get('tomcat_http_port', 0))
def get_tomcat_location(self):
""" Gets value of the tomcat home from [project].cfg file related to particular env and project """
return self.props.get('tomcat_location', None)
def get_app_final_name(self):
""" Gets the name of the application to be deployed (even if it's Poulpe, it can be deployed as ROOT.war). """
return self.props.get('app_final_name', self.project)
def get_plugins(self):
if 'app_plugins' in self.props and self.props['app_plugins']:
return self.props['app_plugins'].split(',')
else:
return []
def get_plugins_dir(self):
if 'app_plugins_dir' in self.props and self.props['app_plugins_dir']:
return self.props['app_plugins_dir']
return None
def get_app_file_mapping(self):
"""
:return dict: mapping of the src file name (located either in environments/ or in environments/env folder)
without folder part and the destination file path (where to put those files on the server)
"""
file_mapping = {}
app_files_section = 'app-files_'
for key in self.props.keys():
if key.startswith(app_files_section):
src_filename = key.replace(app_files_section, '')
dst_filepath = self.props[key]
file_mapping[src_filename] = dst_filepath
return file_mapping
def get_db_settings(self):
""" :return DbSettings: db settings """
return DbSettings.build(self.props)
def deploy_configs(self):
mapping = self.get_app_file_mapping()
for key in mapping:
src_filepath = os.path.join(self.env_configs_dir, key)
if not os.path.exists(src_filepath):
src_filepath = os.path.join(self.global_configs_dir, key)
if not os.path.exists(src_filepath):
self.logger.info(
'File {0} did not exist, skipping its deployment', key)
continue
dst_filepath = mapping[key]
self.logger.info('Putting file [{0}]', dst_filepath)
if not os.path.exists(os.path.dirname(dst_filepath)):
self.logger.info('Creating dir [{0}] to place the file',
os.path.dirname(dst_filepath))
os.makedirs(os.path.dirname(dst_filepath))
dst_file = file(dst_filepath, 'w')
for line in open(src_filepath).readlines():
dst_file.write(self._resolve_placeholder(self.props, line))
dst_file.close()
def _read_properties(self):
"""
Reads props from global, env and project configs, then returns them as map with `section_option=value`.
Values may contain placeholders referencing other options and special options like `${project}` & `${env}`
"""
configs = [
os.path.abspath(self.global_config_path),
os.path.abspath(self.env_config_path),
os.path.abspath(self.project_config_path)
]
props = {}
config = ConfigParser()
for config_file in configs:
abs_path = os.path.abspath(config_file)
if os.path.exists(abs_path):
self.logger.info('Reading config file [{0}]', abs_path)
config.read(abs_path)
else:
self.logger.info(
'Could not read config file as it does not exist: [{0}]',
abs_path)
for section in config.sections():
for option in config.options(section):
props[section + '_' + option] = config.get(section, option)
with_replaced_placeholders = {}
for key in props.keys():
with_replaced_placeholders[key] = self._resolve_placeholder(
props, props[key])
return with_replaced_placeholders
def _resolve_placeholder(self, props, value, n_of_trial=0):
if n_of_trial > 10:
self.logger.warn(
'Could not resolve all placeholders in property value: {0}. Leaving it as is.',
value)
return value
value = value.replace("${env}",
self.env).replace("${project}", self.project)
if value.find('${') != -1:
for key in props.keys():
value = value.replace('${' + key + '}', props[key])
if value.find('${') != -1:
value = self._resolve_placeholder(props, value, n_of_trial + 1)
return value
def test_deploy_plugins_must_not_clean_prev_plugins_if_not_jc_is_deployed(self):
logger = Logger("plugins_must_not_clean_prev_plugins_if_not_jc_is_deployed")
jcsettings = self._scriptsettings(build=2843)
antsettings = self._scriptsettings(build=574, project="antarcticle")
try:
logger.info("Given JCommune was deployed with one of its plugins")
self._deploy(jcsettings)
logger.info("When deploying Antarcticle")
self._deploy(antsettings)
except:
logger.error("Oops, error happened during deployment")
self._read_log_if_available("/home/jtalks/tomcat/logs/catalina.out")
self._read_log_if_available("/home/jtalks/tomcat/logs/jcommune.log")
raise
plugins = os.listdir("/home/jtalks/.jtalks/plugins/system-test")
logger.info("Then plugins stayed from prev jcommune deployment: [{0}]", ", ".join(plugins))
self.assertNotEqual(0, len(plugins), "Actual plugins: " + ", ".join(plugins))
logger.info("And precisely the same plugin was left (QnA)")
self.assertEqual(["questions-n-answers-plugin.jar"], plugins)
class ScriptSettings:
SCRIPT_TEMD_DIR = '/tmp/jtalks-cicd/'
script_work_dir = "" + os.path.expanduser("~/.jtalks/")
backups_dir = script_work_dir + "backups/"
ENV_CONFIGS_DIR = script_work_dir + "environments/"
GLOBAL_CONFIG_LOCATION = ENV_CONFIGS_DIR + "global-configuration.cfg"
logger = Logger("ScriptSettings")
def __init__(self,
build,
project=None,
env=None,
grab_envs=None,
work_dir=None,
sanity_test_timeout_sec=120,
package_version=None):
"""
@param grab_envs - whether or not we should clone JTalks predefined environment configuration from private git
repo
@param work_dir - standard is ~/.jtalks, but it may be useful to override this value, e.g. during tests
@param sanity_test_timeout_sec - how much time do sanity tests wait for the application to respond until they
consider deployment as failed
"""
self.env = env
self.build = build
self.project = project
self.grab_envs = grab_envs
self.script_work_dir = work_dir
self.sanity_test_timeout_sec = sanity_test_timeout_sec
self.package_version = package_version
def log_settings(self):
self.logger.info(
"Script Settings: project=[{0}], env=[{1}], build number=[{2}], sanity test timeout=[{3}], package version=[{4}]",
self.project, self.env, self.build, self.sanity_test_timeout_sec,
self.package_version)
self.logger.info("Environment configuration: [{0}]",
self.ENV_CONFIGS_DIR)
def create_work_dir_if_absent(self):
self.__create_dir_if_absent__(self.script_work_dir)
self.__create_dir_if_absent__(self.get_env_configs_dir())
self.__create_dir_if_absent__(self.get_backup_folder())
def __create_dir_if_absent__(self, directory):
if not os.path.exists(directory):
self.logger.info("Creating directory [{0}]", directory)
os.mkdir(directory)
def get_tomcat_location(self):
"""
Gets value of the tomcat home from [project].cfg file related to particular env and project
"""
return self.__get_property('tomcat', 'location')
def get_app_final_name(self):
"""
Gets the name of the application to be deployed (even if it's Poulpe, it can be deployed as ROOT.war).
"""
return self.__get_property('app', 'final_name')
def get_temp_dir(self):
"""
Script can save there some temp files.
"""
return self.SCRIPT_TEMD_DIR
def get_tomcat_port(self):
"""
This is not actually a pre-configured parameter, it parses $TOMCAT_HOME/conf/server.xml to find out the HTTP
port it's going to be listening. This is needed e.g. for sanity tests to.
"""
server_xml = os.path.join(self.get_tomcat_location(), "conf",
"server.xml")
raise RuntimeError("tomcat port parsing is not implemented yet")
def get_backup_folder(self):
return self.backups_dir
def get_env_configs_dir(self):
return self.ENV_CONFIGS_DIR
def get_global_config_location(self):
return self.GLOBAL_CONFIG_LOCATION
def __get_property(self, section, prop_name):
"""
Finds property first in project configuration, then environment configuration and if there is no such property
there, then it looks is it up in global configuration.
@param section - a section joins several properties under it
@param prop_name - a particular property name from specified section
@returns None if there is no such property found in any config
"""
value = self.__get_project_property(section, prop_name)
if value == None:
value = self.__get_env_property(section, prop_name)
if value == None:
value = self.__get_global_prop(section, prop_name)
if value == None:
self.logger.error("Property [{0}] was not found in any configs",
prop_name)
raise ValueError
return self.__replace_placeholders(value)
def __get_project_property(self, section, prop_name):
"""
Finds property value in project configuration. This overrides env and global configuration.
"""
config = ConfigParser()
config.read(
os.path.join(self.ENV_CONFIGS_DIR, self.env,
self.project + ".cfg"))
return self.__get_value_from_config(config, section, prop_name)
def __get_env_property(self, section, prop_name):
"""
Finds property value in configs/${env}/${env}.cfg configuration file. This overrides global configuration, but
still can be overriden by project configs. These configs are shared between apps of the same environment. E.g.
if we have Poulpe and JCommune on UAT env, and there is a file configs/uat/uat.cfg, then these properties are
shared between those Poulpe and JCommune.
"""
config = ConfigParser()
config.read(
os.path.join(self.ENV_CONFIGS_DIR, self.env,
"environment-configuration.cfg"))
return self.__get_value_from_config(config, section, prop_name)
def __get_global_prop(self, section, prop_name):
"""
Finds a property value in configs/global-configuration.cfg.
"""
config = ConfigParser()
config.read(self.GLOBAL_CONFIG_LOCATION)
return self.__get_value_from_config(config, section, prop_name)
def __get_value_from_config(self, config, section, prop_name):
try:
return config.get(section, prop_name)
except NoSectionError:
return None
def __replace_placeholders(self, prop_value):
"""
Replaces placeholder for env and project that were possibly set in config files.
"""
return prop_value.replace("${env}",
self.env).replace("${project}", self.project)
def get_sanity_test_timeout_sec(self):
"""
How much time do sanity tests wait for the application response until they consider that the app didn't start
"""
self.sanity_test_timeout_sec
class DbOperations:
"""
Base class for working with database
"""
logger = Logger("DbBase")
env = None
db_settings = None
def __init__(self, env, db_settings):
"""
Args:
env: environment name e.g. dev, uat
db_settings: database connection settings of type DbSettings, which will be used for all
database related operations
"""
self.env = env
self.db_settings = db_settings
def connect_to_database(self):
"""
Connects to the specified database
"""
self.logger.info("Connecting to [{0}] with user [{1}]",
self.db_settings.dbHost, self.db_settings.dbUser)
self.connection = MySQLdb.connect(host=self.db_settings.dbHost,
user=self.db_settings.dbUser,
passwd=self.db_settings.dbPass)
self.cursor = self.connection.cursor()
def close_connection(self):
"""
Closes open connection to the database
"""
self.connection.close()
def recreate_database(self):
"""
Deletes the specified database and create it again
"""
self.logger.info("Dropping and creating database from scratch: [{0}]",
self.db_settings.dbName)
self.cursor.execute('DROP DATABASE IF EXISTS ' +
self.db_settings.dbName)
self.cursor.execute('CREATE DATABASE ' + self.db_settings.dbName)
def backup_database(self, backupPath):
"""
Backups database to given backupPath
"""
self.logger.info("Backing up database [{0}] to [{1}] using user [{2}]",
self.db_settings.dbName, backupPath,
self.db_settings.dbUser)
os.popen("mysqldump -u{0} -p{1} {2} > {3}/{2}.sql"
.format(self.db_settings.dbUser, self.db_settings.dbPass, self.db_settings.dbName, backupPath)) \
.read()
self.logger.info("Database backed up: environment=[{0}]".format(
self.env))
def restore_database_from_file(self, backupPath):
"""
Restores database from file specified in backupPath
"""
self.logger.info("Loading a db dump from [{0}/{1}.sql]", backupPath,
self.db_settings.dbName)
self.recreate_database()
os.popen('mysql -u{0} -p{1} {2} < {3}/{2}.sql'
.format(self.db_settings.dbUser,self.db_settings.dbPass, self.db_settings.dbName, backupPath))\
.read()
self.logger.info("Database restored: environment=[{0}]".format(
self.env))
class DB:
"""
A class to work with DB (upload, download, search). Note, that for all the operations we need a connection to
database which created by connect_to_database method. To connection need name of environments, which contained config file with properties to database.
"""
env = None
config = None
dbHost = None
dbUser = None
dbPass = None
dbName = None
dbDefiner = None
cursor = None
connection = None
jcName = None
jcDescription = None
jcUrl = None
jcNotify = None
poulpeAdminPass = None
logger = Logger("DB")
def __init__(self, script_settings):
self.env = script_settings.env
self.config = ConfigParser.ConfigParser()
self.config.read(script_settings.get_env_configs_dir() + self.env +
"/db.cfg")
self.dbHost = self.config.get('db', 'host')
self.dbUser = self.config.get('db', 'user')
self.dbPass = self.config.get('db', 'pass')
self.dbName = self.config.get('db', 'name')
self.dbDefiner = self.config.get('db', 'definer')
self.jcName = self.config.get('properties', 'jc_name')
self.jcDescription = self.config.get('properties', 'jc_description')
self.jcUrl = self.config.get('properties', 'jc_url')
self.jcNotify = self.config.get('properties', 'jc_notify')
self.poulpeAdminPass = self.config.get('properties',
'poulpe_admin_pass')
def connect_to_database(self):
"""
Create connection to database.
"""
self.connection = MySQLdb.connect(host=self.dbHost,
user=self.dbUser,
passwd=self.dbPass)
self.cursor = self.connection.cursor()
def close_connection(self):
"""
Close connection to database.
"""
self.connection.close()
def recreate_database(self):
"""
Method removed database (whith name, which contains in dbName), and create new(empty)
database whita same name.
"""
self.logger.info("Recreating database {0}", self.dbName)
self.cursor.execute('DROP DATABASE IF EXISTS ' + self.dbName)
self.cursor.execute('CREATE DATABASE ' + self.dbName)
def restore_database_from_file(self, backupPath):
"""
This method get path to filename (.sql) with backup of database. And restoring backup to database whith name,
which contains in dbName.
"""
self.logger.info("Loading a db dump from [{0}]", backupPath)
self.recreate_database()
self.fix_definer_in_backup(backupPath)
os.popen('mysql -u ' + self.dbUser + ' --password='******' ' + self.dbName + ' < ' + backupPath).read()
self.logger.info("Database restored: environment=[{0}]".format(
self.env))
def fix_definer_in_backup(self, backupPath):
"""
For MySQL.
If database contains VIEWS then created backup of database (by mysqldump) file contains lines with permissions (to
user from origin database). When we restoring database to another server(it server don't have it user) we get a ERROR. To fix this problem, before restore we need replace all entries (with DEFINER='{username}') to MySQL constant CURRENT_USER. This constant indicates that the law should give to the user on whose behalf is restored.
"""
os.popen('sed -i \'s/DEFINER=' + self.dbDefiner +
'/DEFINER=CURRENT_USER/g\' ' + backupPath).read()
self.logger.info("Database definer fixed: environment=[{0}]".format(
self.env))
def update_properties_to_preprod(self):
"""
This method update application properties in database. Need to call after restore.
"""
self.cursor.execute('USE ' + self.dbName)
self.logger.info("Changing forum name to [{0}]", self.jcName)
self.cursor.execute('UPDATE COMPONENTS SET NAME="' + self.jcName +
'", DESCRIPTION="' + self.jcDescription +
'" where COMPONENT_TYPE="FORUM"')
self.logger.info("Switching off mail notifications")
self.cursor.execute(
'UPDATE PROPERTIES SET VALUE="' + self.jcNotify +
'" where NAME="jcommune.sending_notifications_enabled"')
self.cursor.execute('UPDATE PROPERTIES SET VALUE="' + self.jcUrl +
'" where NAME="jcommune.url_address"')
self.cursor.execute('UPDATE USERS SET PASSWORD=MD5(' +
self.poulpeAdminPass + ') WHERE USERNAME="******"')
self.connection.commit()
def __init__(self):
self.logger = Logger('Main')
class DB:
"""
A class to work with DB (upload, download, search). Note, that for all the operations we need a connection to
database which created by connect_to_database method. To connection need name of environments, which contained config file with properties to database.
"""
env = None
config = None
dbHost = None
dbUser = None
dbPass = None
dbName = None
cursor = None
connection = None
jcName = None
jcDescription = None
jcUrl = None
jcNotify = None
poulpeAdminPass = None
logger = Logger("DB")
def __init__(self, dbsettings, scriptsettings):
self.dbHost = dbsettings.host
self.dbUser = dbsettings.user
self.dbPass = dbsettings.password
self.dbName = dbsettings.name
self.jcName = scriptsettings.props('forum_name')
self.jcDescription = scriptsettings.props('forum_description')
self.jcUrl = scriptsettings.props('forum_url')
self.jcNotify = scriptsettings.props('forum_notify')
self.poulpeAdminPass = scriptsettings.props('forum_poulpe_admin_pass')
def connect_to_database(self):
"""
Create connection to database.
"""
self.connection = MySQLdb.connect(host=self.dbHost,
user=self.dbUser,
passwd=self.dbPass)
self.cursor = self.connection.cursor()
def close_connection(self):
"""
Close connection to database.
"""
self.connection.close()
def recreate_database(self):
"""
Method removed database (whith name, which contains in dbName), and create new(empty)
database whita same name.
"""
self.logger.info("Recreating database {0}", self.dbName)
self.cursor.execute('DROP DATABASE IF EXISTS ' + self.dbName)
self.cursor.execute('CREATE DATABASE ' + self.dbName)
def restore_database_from_file(self, backupPath):
"""
This method get path to filename (.sql) with backup of database. And restoring backup to database whith name,
which contains in dbName.
"""
self.logger.info("Loading a db dump from [{0}]", backupPath)
self.recreate_database()
self.fix_definer_in_backup(backupPath)
os.popen('mysql -u ' + self.dbUser + ' --password='******' ' + self.dbName + ' < ' + backupPath).read()
self.logger.info("Database restored: environment=[{0}]".format(
self.env))
def fix_definer_in_backup(self, backupPath):
"""
For MySQL.
If database contains VIEWS then created backup of database (by mysqldump) file contains lines with permissions
(to user from original database). When we restoring database on another server (the server that doesn't have
that user) we get an ERROR. To fix this problem, before restore we need remove all entries
(with DEFINER='{username}') and the view will be created and will be invoked by the user that created the DB.
"""
os.popen("sed -i 's/DEFINER=[^*]*\*/\*/g' " + backupPath).read()
self.logger.info("Database definer fixed: environment=[{0}]".format(
self.env))
def update_properties_to_preprod(self):
"""
This method update application properties in database. Need to call after restore.
"""
self.cursor.execute('USE ' + self.dbName)
self.logger.info("Changing forum name to [{0}]", self.jcName)
self.cursor.execute('UPDATE COMPONENTS SET NAME="' + self.jcName +
'", DESCRIPTION="' + self.jcDescription +
'" where COMPONENT_TYPE="FORUM"')
self.logger.info("Switching off mail notifications")
self.cursor.execute('UPDATE PROPERTIES SET VALUE="' + self.jcUrl +
'" where NAME="jcommune.url_address"')
self.cursor.execute('UPDATE USERS SET PASSWORD=MD5(' +
self.poulpeAdminPass + ') WHERE USERNAME="******"')
self.connection.commit()
class Backuper:
"""
Responsible for backing up artifacts like war files, config files, as well as DB backups.
Stores backups in the folder and allows to restore those backups from it.
@author stanislav bashkirtsev
"""
logger = Logger("Backuper")
BACKUP_FOLDER_DATE_FORMAT = "%Y_%m_%dT%H_%M_%S"
def __init__(self, backup_folder, script_settings, db_operations):
"""
@param backup_folder a directory to put backups to, it may contain backups from different envs,
thus a folder for each env will be created
@param script_settings is instance of ScriptSettings class
@param db_operations instance of DbOperations
"""
if not backup_folder.endswith("/"):
raise ValueError(
"Folder name should finish with '/'. Actual value: " +
backup_folder)
self.root_backup_folder = backup_folder
self.script_settings = script_settings
self.db_operations = db_operations
def backup(self):
"""
Does all operations for project backup:
- backup the application database
- backup application war-file
- backup tomcat and ehcache configuration files
"""
folder_to_put_backups = self.create_folder_to_backup(
self.root_backup_folder, self.script_settings)
self.backup_tomcat(folder_to_put_backups)
self.backup_db(folder_to_put_backups)
def clean_old_backups(self, backups_to_keep):
"""
In order not to keep a lot of useless backups that take space, we're purging old ones.
@param backups_to_keep amount of backups to keep in backup folder without removal
"""
all_backups = self.get_list_of_backups()
def create_folder_to_backup(self, backup_folder, script_settings):
"""
We don't just put backups into a backup folder, we're creating a new folder there
with current date so that our previous backups are kept there. E.g.:
'/var/backups/prod/20130302T195924'
@param backups_folder - basic folder to put backups there, it will be concatenated with env and current time
@param script_settings to figure out what's the env and what's the project we're going to backup
"""
now = datetime.now().strftime(self.BACKUP_FOLDER_DATE_FORMAT)
final_backup_folder = "{0}/{1}".format(
self.get_project_backup_folder(), now)
os.makedirs(final_backup_folder)
self.logger.info("Backing up old resources to [{0}]",
final_backup_folder)
return final_backup_folder
def backup_tomcat(self, backup_folder):
"""
To be safe that we didn't forget anything, we're doing a backup for the whole tomcat directory.
@param backup_folder a directory to put Tomcat to, should be created by the time this method is invoked
"""
tomcat_location = self.script_settings.get_tomcat_location()
if os.path.exists(tomcat_location):
shutil.copytree(tomcat_location, backup_folder + "/tomcat")
else:
self.logger.info(
"There was no previous tomcat folder [{0}], so nothing to backup",
tomcat_location)
def backup_db(self, backup_folder):
self.db_operations.backup_database(backup_folder)
def get_list_of_backups(self):
"""
Gets the list of backups saved for
"""
return os.listdir(self.root_backup_folder)
def get_project_backup_folder(self):
return "{0}{1}/{2}".format(self.root_backup_folder,
self.script_settings.env,
self.script_settings.project)
class JtalksArtifacts:
logger = Logger('JtalksArtifacts')
def __init__(self, repo='builds'):
self.repo = repo
def download_war(self, project, build):
gav = Gav(project + '-web-view',
'org.jtalks.' + project,
version='',
extension='')
nexus = Nexus()
version_page_url = gav.to_url(nexus.nexus_url, self.repo)
version = NexusPageWithVersions().parse(version_page_url).version(
build)
gav.version = version
gav.extension = 'war'
nexus.download(self.repo, gav, project + '.war')
return gav, project + '.war'
def download_plugins(self, project, version, artifact_ids=()):
""" -> [str] """
files = []
for plugin in artifact_ids:
gav, filename = self.download_plugin(project, version, plugin)
files.append(filename)
return files
def download_plugin(self, project, version, artifact_id):
gav = Gav(artifact_id, 'org.jtalks.' + project, version)
tofile_path = artifact_id + '.' + gav.extension
Nexus().download(self.repo, gav, tofile_path)
return gav, tofile_path
def deploy_plugins(self, to_dir, plugin_files=[]):
"""
Puts plugins to the config to the specified folder on the FS. Cleans previous plugins of there are any.
:param str to_dir: directory to put the plugins to. Will be created if it's absent.
:param [str] plugin_files: file names to put to the target dir
"""
if to_dir:
if not os.path.exists(to_dir):
self.logger.info('Plugin dir did not exist, creating: [{0}]',
to_dir)
os.makedirs(to_dir)
for filename in os.listdir(to_dir): # rm previous plugins
if filename.endswith('.jar'):
plugin_path = os.path.join(to_dir, filename)
self.logger.info('Removing previous plugins: [{0}]',
plugin_path)
os.remove(plugin_path)
for plugin in plugin_files:
self.logger.info('Adding plugin [{0}] to [{1}]', plugin,
to_dir)
shutil.move(plugin, to_dir)
elif len(plugin_files) != 0:
self.logger.warn(
'Plugin dir was not specified in env configs while there are plugins specified '
'to be deployed: [{0}]. Skipping plugin deployment',
','.join(plugin_files))
return
def __init__(self):
self.logger = Logger("Main")
class Tomcat:
"""
Class for deploying and backing up Tomcat applications
"""
logger = Logger("Tomcat")
def __init__(self, backuper, script_settings):
self.backuper = backuper
self.script_settings = script_settings
def deploy_war(self):
"""
Stops the Tomcat server, backups all necessary application data, copies
new application data to Tomcat directories
"""
self.logger.info("Deploying {0} to {1}", self.script_settings.project,
self.script_settings.get_tomcat_location())
self.stop()
self.backuper.backup()
self.move_war_to_webapps()
self.put_configs_to_conf()
self.start()
def stop(self):
"""
Stops the Tomcat server if it is running
"""
stop_command = "pkill -9 -f {0}".format(
self.script_settings.get_tomcat_location())
self.logger.info("Killing tomcat [{0}]", stop_command)
#dunno why but retcode always equals to SIGNAL (-9 in this case), didn't figure out how to
#distinguish errors from this
retcode = subprocess.call([stop_command],
shell=True,
stdout=PIPE,
stderr=PIPE)
def move_war_to_webapps(self):
"""
Moves application war-file to 'webapps' Tomcat subfolder
"""
final_app_location = self.get_web_apps_location(
) + "/" + self.script_settings.get_app_final_name()
self.remove_previous_app(final_app_location)
self.logger.info("Putting new war file to Tomcat: [{0}]",
final_app_location)
shutil.move(self.script_settings.project + ".war",
final_app_location + ".war")
def remove_previous_app(self, app_location):
if os.path.exists(app_location):
self.logger.info("Removing previous app: [{0}]", app_location)
shutil.rmtree(app_location)
else:
self.logger.info(
"Previous application was not found in [{0}], thus nothing to remove",
app_location)
war_location = app_location + ".war"
if os.path.exists(war_location):
self.logger.info("Removing previous war file: [{0}]", war_location)
os.remove(war_location)
def get_config_file_location(self):
return os.path.join(self.script_settings.get_env_configs_dir(),
self.script_settings.env, self.get_config_name())
def put_configs_to_conf(self):
"""
Copies configuration files for application and ehcache to Tomcat directories
"""
final_conf_location = self.get_config_folder_location(
) + "/" + self.script_settings.get_app_final_name() + ".xml"
conf_file_location = self.get_config_file_location()
ehcache_config_file_location = os.path.join(
self.script_settings.get_env_configs_dir(),
self.script_settings.env, self.get_ehcache_config_name())
self.logger.info("Putting [{0}] into [{1}]", conf_file_location,
final_conf_location)
shutil.copyfile(conf_file_location, final_conf_location)
if os.path.exists(ehcache_config_file_location):
shutil.copy(ehcache_config_file_location,
self.script_settings.get_tomcat_location() + "/conf")
def start(self):
"""
Starts the Tomcat server
"""
startup_file = self.script_settings.get_tomcat_location(
) + "/bin/startup.sh"
self.logger.info("Starting Tomcat [{0}]", startup_file)
subprocess.call(startup_file, shell=True, stdout=PIPE, stderr=PIPE)
def get_ehcache_config_name(self):
"""
Returns name of the Ehcache configuration file
"""
return self.script_settings.project + ".ehcache.xml"
def get_config_name(self):
"""
Returns name of the Tomcat configuration file
"""
return self.script_settings.project + ".xml"
def get_config_folder_location(self):
"""
Returns configuration folder for Tomcat
"""
return os.path.join(self.script_settings.get_tomcat_location(), "conf",
"Catalina", "localhost")
def get_web_apps_location(self):
"""
Returns path to web applications directory of Tomcat
"""
return self.script_settings.get_tomcat_location() + "/webapps"
def test_deploy_plugins_must_not_clean_prev_plugins_if_not_jc_is_deployed(
self):
logger = Logger(
'plugins_must_not_clean_prev_plugins_if_not_jc_is_deployed')
jcsettings = self._scriptsettings(build=2843)
antsettings = self._scriptsettings(build=574, project='antarcticle')
try:
logger.info('Given JCommune was deployed with one of its plugins')
self._deploy(jcsettings)
logger.info('When deploying Antarcticle')
self._deploy(antsettings)
except:
logger.error('Oops, error happened during deployment')
self._read_log_if_available(
'/home/jtalks/tomcat/logs/catalina.out')
self._read_log_if_available(
'/home/jtalks/tomcat/logs/jcommune.log')
raise
plugins = os.listdir('/home/jtalks/.jtalks/plugins/system-test')
logger.info('Then plugins stayed from prev jcommune deployment: [{0}]',
', '.join(plugins))
self.assertNotEqual(0, len(plugins),
'Actual plugins: ' + ', '.join(plugins))
logger.info('And precisely the same plugin was left (QnA)')
self.assertEqual(['questions-n-answers-plugin.jar'], plugins)
