def run_deployment_script(self, custom_deploy_script):
method = 'run_deployment_script'
commons.print_msg(Cloud.clazz, method, 'begin')
cmd = "./" + custom_deploy_script
execute_custom_script = subprocess.Popen(cmd.split(),
shell=False,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
while execute_custom_script.poll() is None:
line = execute_custom_script.stdout.readline().decode(
'utf-8').strip(' \r\n')
commons.print_msg(Cloud.clazz, method, line)
execute_custom_script_output, execute_custom_script_error = execute_custom_script.communicate(
timeout=120)
for line in execute_custom_script_output.splitlines():
commons.print_msg(Cloud.clazz, method, line.decode("utf-8"))
if execute_custom_script.returncode != 0:
commons.print_msg(
Cloud.clazz, method,
"Failed calling {command}. Return code of {rtn}".format(
command=cmd, rtn=execute_custom_script.returncode),
'ERROR')
return False
commons.print_msg(Cloud.clazz, method, 'end')
return True
def find_deployable(self, file_ext, dir_to_look_in):
method = 'find_deployable'
commons.print_msg(Cloud.clazz, method, 'begin')
commons.print_msg(
Cloud.clazz, method,
"Looking for a {ext} in {dir}".format(ext=file_ext,
dir=dir_to_look_in))
deployable_files = commons.get_files_of_type_from_directory(
file_ext.lower(), dir_to_look_in)
if len(deployable_files) > 1:
commons.print_msg(
Cloud.clazz, method,
"Found more than 1 artifact in {}".format(dir_to_look_in),
'ERROR')
# raise IOError('Found more than 1 artifact')
exit(1)
elif len(deployable_files) == 0:
commons.print_msg(
Cloud.clazz, method,
"Could not find file of type {ext} in {dir}".format(
ext=file_ext, dir=dir_to_look_in), 'ERROR')
# raise IOError('Found 0 artifacts')
exit(1)
commons.print_msg(Cloud.clazz, method, 'end')
return deployable_files[0]
def _get_stopped_apps(self):
method = '_get_stopped_apps'
commons.print_msg(CloudFoundry.clazz, method, 'begin')
cmd = "{path}cf apps | grep {proj}*-v\d*\.\d*\.\d* | grep stopped | awk '{{print $1}}'".format(
path=CloudFoundry.path_to_cf,
proj=self.config.project_name)
stopped_apps = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) # nosec
get_stopped_apps_failed = False
try:
CloudFoundry.stopped_apps, errs = stopped_apps.communicate(timeout=60)
for line in CloudFoundry.stopped_apps.splitlines():
commons.print_msg(CloudFoundry.clazz, method, "App Already Stopped: {}".format(line.decode('utf-8')))
if stopped_apps.returncode != 0:
commons.print_msg(CloudFoundry.clazz, method, "Failed calling {command}. Return code of {rtn}".format(
command=cmd, rtn=stopped_apps.returncode), 'ERROR')
get_stopped_apps_failed = True
except TimeoutExpired:
commons.print_msg(CloudFoundry.clazz, method, "Timed out calling {}".format(cmd), 'ERROR')
get_stopped_apps_failed = True
if get_stopped_apps_failed:
stopped_apps.kill()
os.system('stty sane')
self._cf_logout()
exit(1)
commons.print_msg(CloudFoundry.clazz, method, 'end')
def _cf_logout(self):
method = '_cf_logout'
commons.print_msg(CloudFoundry.clazz, method, 'begin')
cmd = "{}cf logout".format(CloudFoundry.path_to_cf)
cf_logout = subprocess.Popen(cmd.split(), shell=False, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
logout_failed = False
try:
cf_logout_output, errs = cf_logout.communicate(timeout=30)
for line in cf_logout_output.splitlines():
commons.print_msg(CloudFoundry.clazz, method, line.decode('utf-8'))
if cf_logout.returncode != 0:
commons.print_msg(CloudFoundry.clazz, method, "Failed calling {command}. Return code of {rtn}"
.format(command=cmd,
rtn=cf_logout.returncode),
'ERROR')
logout_failed = True
except TimeoutExpired:
commons.print_msg(CloudFoundry.clazz, method, "Timed out calling {}".format(cmd), 'ERROR')
logout_failed = True
if logout_failed:
cf_logout.kill()
os.system('stty sane')
exit(1)
commons.print_msg(CloudFoundry.clazz, method, 'end')
def __init__(self, config_override=None):
method = '__init__'
commons.print_msg(SonarQube.clazz, method, 'begin')
if config_override is not None:
self.config = config_override
commons.print_msg(SonarQube.clazz, method, 'end')
def get_artifact_url(self):
method = "get_artifact_url"
if len(self.artifactory_extensions) > 1:
commons.print_msg(
Artifactory.clazz, method,
"You have more then one extension declared. Use 'get_urls_of "
"artifacts' to get more then one url")
return self._get_artifact_url(self.artifactory_extensions[0])
def _check_artifact_permissions(self, remove_resp, method):
if remove_resp.status_code == 403:
commons.print_msg(
Artifactory.clazz, method,
"Failed publishing to artifactory: {response}. \nArtifact version must "
"be updated before publishing".format(
response=remove_resp.text), "ERROR")
exit(1)
def __init__(self):
method = '__init__'
self.endpoint = self.config.settings.get('metrics', 'endpoint')
commons.print_msg(self.clazz, method, "Metrics Endpoint {}".format(self.endpoint))
self.prefix = self.config.settings.get('metrics', 'prefix')
commons.print_msg(self.clazz, method, "Metrics Prefix {}".format(self.prefix))
def __init__(self, config_override=None):
method = '__init__'
commons.print_msg(Slack.clazz, method, 'begin')
Slack.slack_url = os.getenv('SLACK_WEBHOOK_URL')
if config_override is not None:
self.config = config_override
commons.print_msg(Slack.clazz, method, 'end')
def _verify_required_attributes(self):
method = '_verfify_required_attributes'
if not os.getenv('GCAPPENGINE_USER_JSON'):
commons.print_msg(
GCAppEngine.clazz, method,
'Credentials not loaded. Please define '
'environment variable '
'\'GCAPPENGINE_USER_JSON\'', 'ERROR')
exit(1)
def tag_stories_in_commit(self, story_list):
method = 'tag_stories_in_commit'
commons.print_msg(Tracker.clazz, method, 'begin')
for story in story_list:
label = self.config.project_name + '-' + self.config.version_number
self._add_label_to_tracker(story, label)
commons.print_msg(Tracker.clazz, method, 'end')
def _ship_it_artifactory(self, name):
method = '_ship_it_artifactory'
commons.print_msg(ZipIt.clazz, method, 'begin')
file_with_path = name.split('/')
ar = Artifactory()
ar.publish(file_with_path[-1], name)
commons.print_msg(ZipIt.clazz, method, 'end')
def _refresh_tags(self):
method = '_refresh_tags'
commons.print_msg(GitHub.clazz, method, 'getting latest tags')
pull_tags_cmd = "git pull --tags"
pull_tags = subprocess.Popen(pull_tags_cmd.split(), shell=False, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
pull_tags_outputs, pull_tags_errs = pull_tags.communicate(timeout=300)
for tag_line in pull_tags_outputs.splitlines():
commons.print_msg(GitHub.clazz, method, tag_line.decode("utf-8"))
def __init__(self, mode, name, contents):
method = '__init__'
commons.print_msg(ZipIt.clazz, method, 'begin')
ZipIt.file_name = name
if mode == 'artifactory':
ZipIt.zip_contents = contents
self._zip_it(name, contents)
self._ship_it_artifactory(name)
commons.print_msg(ZipIt.clazz, method, 'end')
def _verify_required_attributes(self):
method = '_verify_required_attributes'
try:
# noinspection PyStatementEffect
self.config.json_config['github']
GitHub.url = self.config.json_config['github']['URL']
GitHub.org = self.config.json_config['github']['org']
GitHub.repo = self.config.json_config['github']['repo']
except KeyError as e:
commons.print_msg(GitHub.clazz, method, "The build config associated with github is missing, {}."
.format(e), 'ERROR')
exit(1)
def __init__(self, config_override=None):
method = '__init__'
commons.print_msg(GCAppEngine.clazz, method, 'begin')
if config_override is not None:
self.config = config_override
if os.environ.get('WORKSPACE'): # for Jenkins
GCAppEngine.path_to_google_sdk = os.environ.get('WORKSPACE') + '/'
else:
GCAppEngine.path_to_google_sdk = ""
commons.print_msg(GCAppEngine.clazz, method, 'end')
def tag_stories_in_commit(self, story_list):
method = 'tag_stories_in_commit'
commons.print_msg(Jira.clazz, method, 'begin')
self._create_version(self.config.project_name + '-' +
self.config.version_number)
for story in story_list:
label = self.config.project_name + '-' + self.config.version_number
self._add_version_to_jira_story(story, label)
commons.print_msg(Jira.clazz, method, 'end')
def __init__(self, config_override=None):
method = '__init__'
commons.print_msg(CloudFoundry.clazz, method, 'begin')
if config_override is not None:
self.config = config_override
if os.environ.get('WORKSPACE'): # for Jenkins
CloudFoundry.path_to_cf = os.environ.get('WORKSPACE') + '/'
else:
CloudFoundry.path_to_cf = ""
commons.print_msg(CloudFoundry.clazz, method, 'end')
def _determine_app_yml(self):
method = '_determine_app_yml'
commons.print_msg(GCAppEngine.clazz, method, 'begin')
if os.path.isfile("app-{}.yml".format(self.config.build_env)):
app_yaml = "app-{}.yml".format(self.config.build_env)
elif os.path.isfile("{dir}/app-{env}.yml".format(
dir=self.config.push_location, env=self.config.build_env)):
app_yaml = "{dir}/app-{env}.yml".format(
dir=self.config.push_location, env=self.config.build_env)
elif os.path.isfile("app-{}.yaml".format(self.config.build_env)):
app_yaml = "app-{}.yaml".format(self.config.build_env)
elif os.path.isfile("{dir}/app-{env}.yaml".format(
dir=self.config.push_location, env=self.config.build_env)):
app_yaml = "{dir}/app-{env}.yaml".format(
dir=self.config.push_location, env=self.config.build_env)
else:
commons.print_msg(
GCAppEngine.clazz, method,
"Failed to find app_yaml file app-{}.yml/yaml".format(
self.config.build_env), 'ERROR')
exit(1)
# noinspection PyUnboundLocalVariable
commons.print_msg(GCAppEngine.clazz, method,
"Using app_yaml {}".format(app_yaml))
commons.print_msg(GCAppEngine.clazz, method, 'end')
return app_yaml
def download_cf_cli(self):
method = '_download_cf_cli'
commons.print_msg(CloudFoundry.clazz, method, 'begin')
cmd = "where" if platform.system() == "Windows" else "which"
rtn = subprocess.call([cmd, 'cf'])
if rtn == 0:
commons.print_msg(CloudFoundry.clazz, method,
'cf cli already installed')
else:
commons.print_msg(
CloudFoundry.clazz, method,
"cf CLI was not installed on this image. "
"Downloading CF CLI from {}".format(
self.config.settings.get('cloudfoundry',
'cli_download_path')))
urllib.request.urlretrieve(
self.config.settings.get('cloudfoundry',
'cli_download_path'), # nosec
'./cf-linux-amd64.tgz')
tar = tarfile.open('./cf-linux-amd64.tgz')
CloudFoundry.path_to_cf = "./"
tar.extractall()
tar.close()
commons.print_msg(CloudFoundry.clazz, method, 'end')
def _get_artifact_url(self):
method = "_get_artifact_url"
commons.print_msg(GitHub.clazz, method, "begin")
if GitHub.token is not None:
headers = {'Content-type': cicommons.content_json, 'Accept': cicommons.content_json, 'Authorization': ('token ' + GitHub.token)}
else:
headers = {'Content-type': cicommons.content_json, 'Accept': cicommons.content_json}
tag_information_url = GitHub.url.replace('\\', '/').rstrip('/') + '/' + self.org + '/' + self.repo + \
'/releases/tags/' + self.config.version_number
commons.print_msg(GitHub.clazz, method, ("Retrieving Github information from " + tag_information_url))
resp = requests.get(tag_information_url, headers=headers)
if resp.status_code != 200:
commons.print_msg(GitHub.clazz, method, ("Failed to access github tag information at " + tag_information_url + "\r\n Response: " + resp.text), "ERROR")
exit(1)
else:
commons.print_msg(GitHub.clazz, method, resp.text)
json_data = json.loads(resp.text)
artifact_to_download = json_data['tarball_url']
return artifact_to_download
def scan_code(self):
method = 'scan_code'
commons.print_msg(SonarQube.clazz, method, 'begin')
retries = 0
keep_retrying = True
while keep_retrying:
try:
sleep_timer = 0
if self.config.settings.has_section('project') and self.config.settings.has_option('project',
'retry_sleep_interval'):
sleep_timer = int(self.config.settings.get('project', 'retry_sleep_interval'))
if retries > 0:
time.sleep(sleep_timer * retries)
self._submit_scan()
keep_retrying = False
except Exception:
retries += 1
if retries > 3:
commons.print_msg(SonarQube.clazz, method, 'Could not connect to Sonar. Maximum number of retries '
'reached.', "ERROR")
keep_retrying = False
exit(1)
else:
commons.print_msg(SonarQube.clazz, method, "Attempting retry number {}".format(retries), "WARN")
commons.print_msg(SonarQube.clazz, method, 'end')
def get_all_semver_tags(self, need_snapshot=0, need_release=0, need_tag=None, need_base=False):
method = "get_all_semver_tags"
all_tags_output = self.get_all_tags_and_shas_from_github(need_snapshot=need_snapshot, need_release=need_release, need_tag=need_tag, need_base=need_base)
all_tags = all_tags_output#.splitlines()
tag_data = []
for tag, _ in all_tags:
try:
tag_data.append(self.convert_semver_string_to_semver_tag_array(tag))
except Exception:
commons.print_msg(GitHub.clazz, method, "This tag didn't parse right skipping: {} ".format(tag))
tag_data.sort(reverse=True)
GitHub.all_tags_sorted = tag_data
return tag_data
def _determine_push_location(self):
method = '_determine_push_location'
commons.print_msg(CloudFoundry.clazz, method, 'begin')
if self.config.artifact_extension is None or self.config.artifact_extension in (
'zip', 'tar', 'tar.gz'):
# deployed from github directly or it's a zip, tar, tar.gz file
return "-p " + self.config.push_location
elif self.config.artifact_extension == 'docker':
# -p flag not supported for Docker deployments
return ""
else:
return "-p {dir}/{file}".format(dir=self.config.push_location,
file=self.find_deployable(
self.config.artifact_extension,
self.config.push_location))
def write_metric(self, task, action):
method = 'write_metric'
commons.print_msg(self.clazz, method, 'begin')
# try:
# message = "{0}.{1}.{2}.{3}.count {4} {5}\n".format(self.prefix, task, action, self.config.project_name,
# 1, int(time()))
# # resp = post(self.endpoint, message)
# #
# # if resp.status_code == 200:
# # commons.print_msg(self.clazz, method, "Metrics Write {}".format(message))
# # else:
# # commons.print_msg(self.clazz, method, 'Metrics Write Failed', 'ERROR')
#
# except socket.error as e:
# commons.print_msg(self.clazz, method, "Metrics Write Failed ()".format(e), 'ERROR')
commons.print_msg(self.clazz, method, 'end')
def __init__(self, config_override=None, verify_repo=True):
method = '__init__'
commons.print_msg(GitHub.clazz, method, 'begin')
# check if we provided an override
if config_override is not None:
self.config = config_override
if verify_repo is True:
self._load_github_token()
#self._refresh_tags()
self._verify_required_attributes()
self._verify_repo_existence(GitHub.url, GitHub.org, GitHub.repo)
commons.print_msg(GitHub.clazz, method, 'end')
def download_artifact(self, artifact_url, download_path):
"""
Download the artifact from artifactory. Really just a save a url to a file method.
:param artifact_url: obviously, the artifact url
:param download_path: Where you want the file to go
:return: nothing, exceptions raised if it fails
"""
method = "download_artifact"
try:
with open(download_path, 'wb') as handle:
response = requests.get(artifact_url, stream=True)
if not response.ok:
response.raise_for_status()
for block in response.iter_content(1024):
handle.write(block)
except Exception as e:
commons.print_msg(Artifactory.clazz, method, 'Failed to download {}'.format(artifact_url), 'ERROR')
commons.print_msg(Artifactory.clazz, method, "URLError is {msg}".format(msg=e))
raise ArtifactDownloadException(e)
def __init__(self, config_override=None):
method = '__init__'
commons.print_msg(ServiceNow.clazz, method, 'begin')
if config_override is not None:
self.config = config_override
try:
# below line is to maintain backwards compatibility since stanza was renamed
servicenow_json_config = self.config.json_config['servicenow'] if 'servicenow' in self.config.json_config else \
self.config.json_config['servicemanagement']["servicenow"]
except KeyError as e:
commons.print_msg(
ServiceNow.clazz, method,
"The build config associated with servicemanagement is missing key {}"
.format(str(e)), 'ERROR')
exit(1)
# Check for servicenow url first in buildConfig, second try settings.ini
try:
# noinspection PyUnboundLocalVariable
ServiceNow.servicenow_url = servicenow_json_config['url']
except:
if self.config.settings.has_section(
'servicenow') and self.config.settings.has_option(
'servicenow', 'url'):
ServiceNow.servicenow_url = self.config.settings.get(
'servicenow', 'url')
else:
commons.print_msg(
ServiceNow.clazz, method,
'No service now url found in buildConfig or settings.ini.',
'ERROR')
exit(1)
def _get_manual_deploy_links(self):
method = '_get_manual_deploy_links'
manual_deploy_environment_links = {}
# Look for manual deploy links in the current environment stanza
if 'manualDeployEnvs' in self.config.build_env_info:
manual_deploy_links = self.config.build_env_info['manualDeployEnvs']
commons.print_msg(Slack.clazz, method, "Publishing build links: {}".format(manual_deploy_links))
# For each manual deploy environment, lookup the corresponding link for that environment stanza
for manually_deploy_to_env in manual_deploy_links:
try:
manual_deploy_environment_links[manually_deploy_to_env] = self.config.json_config['environments'][manually_deploy_to_env]['manualDeployLink']
if "?" in manual_deploy_environment_links:
manual_deploy_environment_links[manually_deploy_to_env] = manual_deploy_environment_links[manually_deploy_to_env] + "&"
else:
manual_deploy_environment_links[manually_deploy_to_env] = manual_deploy_environment_links[manually_deploy_to_env] + "?"
manual_deploy_environment_links[manually_deploy_to_env] = manual_deploy_environment_links[manually_deploy_to_env] + "VERSION=" + urllib.parse.quote_plus(self.config.version_number)
except KeyError as e:
commons.print_msg(Slack.clazz, method, "Could not find manual deploy link: {}".format(e),
'ERROR')
else:
commons.print_msg(Slack.clazz, method, 'No manual build links specified')
return manual_deploy_environment_links
def deploy(self, force_deploy=False, manifest=None):
method = 'deploy'
commons.print_msg(CloudFoundry.clazz, method, 'begin')
self._verify_required_attributes()
self.download_cf_cli()
self._cf_login_check()
self._cf_login()
self._check_cf_version()
self._get_stopped_apps()
self._get_started_apps(force_deploy)
if manifest is None:
manifest = self._determine_manifests()
self._cf_push(manifest)
if not os.getenv("AUTO_STOP"):
self._stop_old_app_servers()
if not force_deploy:
# don't delete if force bc we want to ensure that there is always 1 non-started instance
# for backup and force_deploy is used when you need to redeploy/replace an instance
# that is currently running
self._unmap_delete_previous_versions()
commons.print_msg(CloudFoundry.clazz, method, 'DEPLOYMENT SUCCESSFUL')
commons.print_msg(CloudFoundry.clazz, method, 'end')
