def find_deployable(self, file_ext, dir_to_look_in):
method = 'find_deployable'
commons.printMSG(Cloud.clazz, method, 'begin')
commons.printMSG(
Cloud.clazz, method,
"Looking for a {ext} in {dir}".format(ext=file_ext,
dir=dir_to_look_in))
deployable_files = commons.get_files_of_type_from_directory(
file_ext.lower(), dir_to_look_in)
if len(deployable_files) > 1:
commons.printMSG(
Cloud.clazz, method,
"Found more than 1 artifact in {}".format(dir_to_look_in),
'ERROR')
# raise IOError('Found more than 1 artifact')
exit(1)
elif len(deployable_files) == 0:
commons.printMSG(
Cloud.clazz, method,
"Could not find file of type {ext} in {dir}".format(
ext=file_ext, dir=dir_to_look_in), 'ERROR')
# raise IOError('Found 0 artifacts')
exit(1)
commons.printMSG(Cloud.clazz, method, 'end')
return deployable_files[0]
def run_deployment_script(self, custom_deploy_script):
method = 'run_deployment_script'
commons.printMSG(Cloud.clazz, method, 'begin')
cmd = "./" + custom_deploy_script
execute_custom_script = subprocess.Popen(cmd,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
while execute_custom_script.poll() is None:
line = execute_custom_script.stdout.readline().decode(
'utf-8').strip(' \r\n')
commons.printMSG(Cloud.clazz, method, line)
execute_custom_script_output, execute_custom_script_error = execute_custom_script.communicate(
timeout=120)
for line in execute_custom_script_output.splitlines():
commons.printMSG(Cloud.clazz, method, line.decode("utf-8"))
if execute_custom_script.returncode != 0:
commons.printMSG(
Cloud.clazz, method,
"Failed calling {command}. Return code of {rtn}".format(
command=cmd, rtn=execute_custom_script.returncode),
'ERROR')
return False
commons.printMSG(Cloud.clazz, method, 'end')
return True
def load_plugin(plugin):
clazz = 'plugin_loader'
method = 'load_plugin'
current_plugin = imp.load_module(plugin['name'], plugin["module_hdl"],
plugin["path_name"],
plugin["description"])
plugin_members = inspect.getmembers(current_plugin)
plugin_methods = inspect.getmembers(current_plugin, inspect.isfunction)
if 'run_action' not in tuple(
x[0] for x in plugin_methods) or 'register_parser' not in tuple(
x[0] for x in plugin_methods):
commons.printMSG(
clazz, method,
"Failed to find method run_action() and/or register_parser() in plugin {"
"}.".format(plugin), 'ERROR')
exit(1)
if 'parser' not in tuple(x[0] for x in plugin_members):
commons.printMSG(
clazz, method,
"Failed to find variable 'parser' in plugin {}.".format(plugin),
'ERROR')
exit(1)
return current_plugin
def _verify_required_attributes(self):
method = '_verfify_required_attributes'
if not os.getenv('GCAPPENGINE_USER_JSON'):
commons.printMSG(GCAppEngine.clazz, method, 'Credentials not loaded. Please define ''environment variable '
'\'GCAPPENGINE_USER_JSON\'', 'ERROR')
exit(1)
def get_plugins():
clazz = 'plugin_loader'
method = 'load_plugin'
plugins = []
possible_plugins = os.listdir(
os.path.join(os.path.dirname(__file__), plugin_folder))
for i in possible_plugins:
location = os.path.join(
os.path.join(os.path.dirname(__file__), plugin_folder), i)
if not os.path.isdir(location) or '__pycache__' in location:
continue
if not MainModule + ".py" in os.listdir(location): # no .py file
commons.printMSG(
clazz, method,
"Failed to load plugin {}. Missing __init__ method".format(i),
'ERROR')
continue
module_hdl, path_name, description = imp.find_module(
MainModule, [location])
plugins.append({
"name": i,
"module_hdl": module_hdl,
"path_name": path_name,
"description": description
})
module_hdl.close()
return plugins
def __init__(self, config_override=None):
method = '__init__'
commons.printMSG(SonarQube.clazz, method, 'begin')
if config_override is not None:
self.config = config_override
commons.printMSG(SonarQube.clazz, method, 'end')
def __init__(self):
method = '__init__'
self.endpoint = self.config.settings.get('metrics', 'endpoint')
commons.printMSG(self.clazz, method,
"Metrics Endpoint {}".format(self.endpoint))
self.prefix = self.config.settings.get('metrics', 'prefix')
commons.printMSG(self.clazz, method,
"Metrics Prefix {}".format(self.prefix))
def __init__(self, config_override=None):
method = '__init__'
commons.printMSG(Slack.clazz, method, 'begin')
Slack.slack_url = os.getenv('SLACK_WEBHOOK_URL')
if config_override is not None:
self.config = config_override
commons.printMSG(Slack.clazz, method, 'end')
def _ship_it_artifactory(self, name):
method = '_ship_it_artifactory'
commons.printMSG(ZipIt.clazz, method, 'begin')
file_with_path = name.split('/')
ar = ArtiFactory()
ar.publish(file_with_path[-1], name)
commons.printMSG(ZipIt.clazz, method, 'end')
def tag_stories_in_commit(self, story_list):
method = 'tag_stories_in_commit'
commons.printMSG(Tracker.clazz, method, 'begin')
for story in story_list:
label = self.config.project_name + '-' + self.config.version_number
self._add_label_to_tracker(story, label)
commons.printMSG(Tracker.clazz, method, 'end')
def _verify_required_attributes(self):
method = '_verify_required_attributes'
try:
self.config.json_config['github']
GitHub.url = self.config.json_config['github']['URL']
GitHub.org = self.config.json_config['github']['org']
GitHub.repo = self.config.json_config['github']['repo']
except KeyError as e:
commons.printMSG(GitHub.clazz, method, "The build config associated with github is missing, {}."
.format(e), 'ERROR')
exit(1)
def _refresh_tags(self):
method = '_refresh_tags'
commons.printMSG(GitHub.clazz, method, 'getting latest tags')
# TODO this didn't seem to work on Jenkins
pull_tags_cmd = 'git pull --tags'
pull_tags = subprocess.Popen(pull_tags_cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
pull_tags_outputs, pull_tags_errs = pull_tags.communicate(timeout=300)
for tag_line in pull_tags_outputs.splitlines():
commons.printMSG(GitHub.clazz, method, tag_line.decode("utf-8"))
def __init__(self, mode, name, contents):
method = '__init__'
commons.printMSG(ZipIt.clazz, method, 'begin')
ZipIt.file_name = name
if mode == 'artifactory':
ZipIt.zip_contents = contents
self._zip_it(name, contents)
self._ship_it_artifactory(name)
commons.printMSG(ZipIt.clazz, method, 'end')
def __init__(self, config_override=None):
method = '__init__'
commons.printMSG(CloudFoundry.clazz, method, 'begin')
if config_override is not None:
self.config = config_override
if os.environ.get('WORKSPACE'): # for Jenkins
CloudFoundry.path_to_cf = os.environ.get('WORKSPACE') + '/'
else:
CloudFoundry.path_to_cf = ""
commons.printMSG(CloudFoundry.clazz, method, 'end')
def __init__(self, config_override=None):
method = '__init__'
commons.printMSG(GCAppEngine.clazz, method, 'begin')
if config_override is not None:
self.config = config_override
if os.environ.get('WORKSPACE'): # for Jenkins
GCAppEngine.path_to_google_sdk = os.environ.get('WORKSPACE') + '/'
else:
GCAppEngine.path_to_google_sdk = ""
commons.printMSG(GCAppEngine.clazz, method, 'end')
def download_cf_cli(self):
method = '_download_cf_cli'
commons.printMSG(CloudFoundry.clazz, method, 'begin')
cmd = "where" if platform.system() == "Windows" else "which"
rtn = subprocess.call([cmd, 'cf'])
if rtn == 0:
commons.printMSG(CloudFoundry.clazz, method,
'cf cli already installed')
else:
commons.printMSG(
CloudFoundry.clazz, method,
"cf CLI was not installed on this image. "
"Downloading CF CLI from {}".format(
self.config.settings.get('cloudfoundry',
'cli_download_path')))
urllib.request.urlretrieve(
self.config.settings.get('cloudfoundry', 'cli_download_path'),
'./cf-linux-amd64.tgz')
tar = tarfile.open('./cf-linux-amd64.tgz')
CloudFoundry.path_to_cf = "./"
tar.extractall()
tar.close()
commons.printMSG(CloudFoundry.clazz, method, 'end')
def _get_artifact_url(self):
method = "_get_artifact_url"
commons.printMSG(GitHub.clazz, method, "begin")
if GitHub.token is not None:
headers = {'Content-type': cicommons.content_json, 'Accept': cicommons.content_json, 'Authorization': ('token ' + GitHub.token)}
else:
headers = {'Content-type': cicommons.content_json, 'Accept': cicommons.content_json}
tag_information_url = GitHub.url.replace('\\', '/').rstrip('/') + '/' + self.org + '/' + self.repo + \
'/releases/tags/' + self.config.version_number
commons.printMSG(GitHub.clazz, method, ("Retrieving Github information from " + tag_information_url))
resp = requests.get(tag_information_url, headers=headers, verify=False)
if resp.status_code != 200:
commons.printMSG(GitHub.clazz, method, ("Failed to access github tag information at " + tag_information_url + "\r\n Response: " + resp.text), "ERROR")
exit(1)
else:
commons.printMSG(GitHub.clazz, method, resp.text)
json_data = json.loads(resp.text)
artifact_to_download = json_data['tarball_url']
return artifact_to_download
def _download_google_sdk(self):
method = '_download_google_sdk'
commons.printMSG(GCAppEngine.clazz, method, 'begin')
ctx = ssl.create_default_context()
ctx.check_hostname = False
ctx.verify_mode = ssl.CERT_NONE
cmd = "where" if platform.system() == "Windows" else "which"
rtn = subprocess.call([cmd, 'gcloud'])
gcloud_location = self.config.settings.get('googlecloud', 'cloud_sdk_path') + self.config.settings.get('googlecloud', 'gcloud_version')
if rtn == 0:
commons.printMSG(GCAppEngine.clazz, method, 'gcloud already installed')
else:
commons.printMSG(GCAppEngine.clazz, method, "gcloud CLI was not installed on this image. "
"Downloading Google Cloud SDK from {}".format(
gcloud_location))
with urllib.request.urlopen(gcloud_location, context=ctx) as u, open(self.config.settings.get('googlecloud', 'gcloud_version'), 'wb') as f:
f.write(u.read())
tar = tarfile.open('./' + self.config.settings.get('googlecloud', 'gcloud_version'))
GCAppEngine.path_to_google_sdk = 'google-cloud-sdk/bin/'
tar.extractall()
tar.close()
commons.printMSG(GCAppEngine.clazz, method, 'end')
def _download_google_sdk(self):
method = '_download_google_sdk'
commons.printMSG(GCAppEngine.clazz, method, 'begin')
cmd = "where" if platform.system() == "Windows" else "which"
rtn = subprocess.call([cmd, 'gcloud'])
if rtn == 0:
commons.printMSG(GCAppEngine.clazz, method,
'gcloud already installed')
else:
commons.printMSG(
GCAppEngine.clazz, method,
"gcloud CLI was not installed on this image. "
"Downloading Google Cloud SDK from {}".format(
self.config.settings.get('googlecloud', 'cloud_sdk_path')))
urllib.request.urlretrieve(
self.config.settings.get('googlecloud', 'cloud_sdk_path'),
'./google-cloud-sdk-144.0.0-linux-x86_64.tar.gz')
tar = tarfile.open(
'./google-cloud-sdk-144.0.0-linux-x86_64.tar.gz')
GCAppEngine.path_to_google_sdk = 'google-cloud-sdk/bin/'
tar.extractall()
tar.close()
commons.printMSG(GCAppEngine.clazz, method, 'end')
def __init__(self, config_override=None):
method = '__init__'
commons.printMSG(Tracker.clazz, method, 'begin')
if config_override is not None:
self.config = config_override
Tracker.token = os.getenv('TRACKER_TOKEN')
if not Tracker.token:
commons.printMSG(Tracker.clazz, method, 'No tracker token found in environment. Did you define '
'environment variable \'TRACKER_TOKEN\'?', 'ERROR')
exit(1)
try:
# below line is to maintain backwards compatibility since stanza was renamed
tracker_json_config = self.config.json_config['tracker'] if 'tracker' in self.config.json_config else \
self.config.json_config['projectTracking']["tracker"]
Tracker.project_id = str(tracker_json_config['projectId'])
except KeyError as e:
commons.printMSG(Tracker.clazz,
method,
"The build config associated with projectTracking is missing key {}".format(str(e)), 'ERROR')
exit(1)
# Check for tracker url first in buildConfig, second try settings.ini
try:
Tracker.tracker_url = tracker_json_config['url']
except:
if self.config.settings.has_section('tracker') and self.config.settings.has_option('tracker', 'url'):
Tracker.tracker_url = self.config.settings.get('tracker', 'url')
else:
commons.printMSG(Tracker.clazz, method, 'No tracker url found in buildConfig or settings.ini.', 'ERROR')
exit(1)
def _determine_app_yml(self):
method = '_determine_app_yml'
commons.printMSG(GCAppEngine.clazz, method, 'begin')
if os.path.isfile("app-{}.yml".format(self.config.build_env)):
app_yaml = "app-{}.yml".format(self.config.build_env)
elif os.path.isfile("{dir}/app-{env}.yml".format(
dir=self.config.push_location, env=self.config.build_env)):
app_yaml = "{dir}/app-{env}.yml".format(
dir=self.config.push_location, env=self.config.build_env)
elif os.path.isfile("app-{}.yaml".format(self.config.build_env)):
app_yaml = "app-{}.yaml".format(self.config.build_env)
elif os.path.isfile("{dir}/app-{env}.yaml".format(
dir=self.config.push_location, env=self.config.build_env)):
app_yaml = "{dir}/app-{env}.yaml".format(
dir=self.config.push_location, env=self.config.build_env)
else:
commons.printMSG(
GCAppEngine.clazz, method,
"Failed to find app_yaml file app-{}.yml/yaml".format(
self.config.build_env), 'ERROR')
exit(1)
commons.printMSG(GCAppEngine.clazz, method,
"Using app_yaml {}".format(app_yaml))
commons.printMSG(GCAppEngine.clazz, method, 'end')
return app_yaml
def get_all_semver_tags(self, need_snapshot=0, need_release=0, need_tag=None, need_base=False):
method = "get_all_semver_tags"
all_tags_output = self.get_all_tags_and_shas_from_github(need_snapshot=need_snapshot, need_release=need_release, need_tag=need_tag, need_base=need_base)
all_tags = all_tags_output#.splitlines()
tag_data = []
for tag, _ in all_tags:
try:
tag_data.append(self.convert_semver_string_to_semver_tag_array(tag))
except Exception as e:
commons.printMSG(GitHub.clazz, method, "This tag didn't parse right skipping: {} ".format(tag))
tag_data.sort(reverse=True)
GitHub.all_tags_sorted = tag_data
return tag_data
def __init__(self, config_override=None, verify_repo=True):
method = '__init__'
commons.printMSG(GitHub.clazz, method, 'begin')
# check if we provided an override
if config_override is not None:
self.config = config_override
if verify_repo is True:
self._load_github_token()
#self._refresh_tags()
self._verify_required_attributes()
self._verify_repo_existence(GitHub.url, GitHub.org, GitHub.repo)
commons.printMSG(GitHub.clazz, method, 'end')
def _get_stopped_apps(self):
method = '_get_stopped_apps'
commons.printMSG(CloudFoundry.clazz, method, 'begin')
cmd = "{path}cf apps | grep {proj}*-v\d*\.\d*\.\d* | grep stopped | awk '{{print $1}}'".format(
path=CloudFoundry.path_to_cf, proj=self.config.project_name)
stopped_apps = subprocess.Popen(cmd,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
get_stopped_apps_failed = False
try:
CloudFoundry.stopped_apps, errs = stopped_apps.communicate(
timeout=60)
for line in CloudFoundry.stopped_apps.splitlines():
commons.printMSG(
CloudFoundry.clazz, method,
"App Already Stopped: {}".format(line.decode('utf-8')))
if stopped_apps.returncode != 0:
commons.printMSG(
CloudFoundry.clazz, method,
"Failed calling {command}. Return code of {rtn}".format(
command=cmd, rtn=stopped_apps.returncode), 'ERROR')
get_stopped_apps_failed = True
except TimeoutExpired:
commons.printMSG(CloudFoundry.clazz, method,
"Timed out calling {}".format(cmd), 'ERROR')
get_stopped_apps_failed = True
if get_stopped_apps_failed:
stopped_apps.kill()
os.system('stty sane')
self._cf_logout()
exit(1)
commons.printMSG(CloudFoundry.clazz, method, 'end')
def download_artifact(self, artifact_url, download_path):
"""
Download the artifact from artifactory. Really just a save a url to a file method.
:param artifact_url: obviously, the artifact url
:param download_path: Where you want the file to go
:return: nothing, exceptions raised if it fails
"""
method = "download_artifact"
try:
with open(download_path, 'wb') as handle:
response = requests.get(artifact_url, stream=True)
if not response.ok:
response.raise_for_status()
for block in response.iter_content(1024):
handle.write(block)
except Exception as e:
commons.printMSG(ArtiFactory.clazz, method, 'Failed to download {}'.format(artifact_url), 'ERROR')
commons.printMSG("URLError is {msg}".format(msg=e.message))
raise ArtifactDownloadException(e.message)
def connect_error_dispatcher():
clazz = 'aggregator'
method = 'connect_error_dispatcher'
# Load dispatchers for communicating error messages to slack (or somewhere else)
SIGNAL = 'publish-error-signal'
if 'slack' in BuildConfig.json_config:
commons.printMSG(
clazz, method,
'Detected slack in buildConfig. Connecting error dispatcher to slack.'
)
dispatcher.connect(Slack.publish_error,
signal=SIGNAL,
sender=dispatcher.Any)
elif BuildConfig.settings.has_section('slack'):
commons.printMSG(
clazz, method,
'Detected slack in global settings.ini. Connecting error dispatcher to slack.'
)
dispatcher.connect(Slack.publish_error,
signal=SIGNAL,
sender=dispatcher.Any)
else:
commons.printMSG(
clazz, method,
'No event dispatcher detected. The only place errors will show up is in this '
'log.', 'WARN')
def determine_semantic_version_bump(self, story_details):
method = 'determine_semantic_version_bump'
commons.printMSG(Tracker.clazz, method, 'begin')
bump_type = None
for i, story in enumerate(story_details):
for j, label in enumerate(story.get('labels')):
if label.get('name') == 'major':
return 'major'
if story.get('story_type') == 'feature' or story.get('story_type') == 'chore' or story.get('story_type') == 'release':
bump_type = 'minor'
elif story.get('story_type') == 'bug' and bump_type is None:
bump_type = 'bug'
# This fall-through rule is needed because if there are no tracker
# stories present in the commits, we need to default to something,
# else calculate_next_semver will throw an error about getting 'None'
if bump_type is None:
bump_type = 'minor'
commons.printMSG(Tracker.clazz, method, "bump type: {}".format(bump_type))
commons.printMSG(Tracker.clazz, method, 'end')
return bump_type
def __init__(self, config_override=None):
method = '__init__'
commons.printMSG(ArtiFactory.clazz, method, 'begin')
if config_override is not None:
self.config = config_override
try:
# below line is to maintain backwards compatibility since stanza was renamed
if 'artifactoryConfig' in self.config.json_config:
artifactory_json_config = self.config.json_config['artifactoryConfig']
else:
artifactory_json_config = self.config.json_config['artifact']
ArtiFactory.artifactory_domain = artifactory_json_config['artifactoryDomain']
ArtiFactory.artifactory_group = artifactory_json_config['artifactoryGroup']
if self.config.build_env_info['artifactCategory'] == 'release':
ArtiFactory.repo_key = artifactory_json_config['artifactoryRepoKey']
else:
ArtiFactory.repo_key = artifactory_json_config['artifactoryRepoKeySnapshot']
if self.config.artifact_extensions:
for extensions in self.config.artifact_extensions:
self.artifactory_extensions.append(extensions)
else:
self.artifactory_extensions.append(self.config.artifact_extension)
except KeyError as e:
commons.printMSG(ArtiFactory.clazz,
method,
"The build config associated with artifactory is missing key {}".format(str(e)), 'ERROR')
exit(1)
commons.printMSG(ArtiFactory.clazz, method, 'end')
def deploy(self, force_deploy=False, manifest=None):
method = 'deploy'
commons.printMSG(CloudFoundry.clazz, method, 'begin')
self._verify_required_attributes()
self.download_cf_cli()
self._cf_login_check()
self._cf_login()
self._check_cf_version()
self._get_stopped_apps()
self._get_started_apps(force_deploy)
if manifest is None:
manifest = self._determine_manifests()
self._cf_push(manifest)
if not os.getenv("AUTO_STOP"):
self._stop_old_app_servers()
if not force_deploy:
# don't delete if force bc we want to ensure that there is always 1 non-started instance
# for backup and force_deploy is used when you need to redeploy/replace an instance
# that is currently running
self._unmap_delete_previous_versions()
commons.printMSG(CloudFoundry.clazz, method, 'DEPLOYMENT SUCCESSFUL')
commons.printMSG(CloudFoundry.clazz, method, 'end')
def _cf_logout(self):
method = '_cf_logout'
commons.printMSG(CloudFoundry.clazz, method, 'begin')
cmd = "{}cf logout".format(CloudFoundry.path_to_cf)
cf_logout = subprocess.Popen(cmd,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
logout_failed = False
try:
cf_logout_output, errs = cf_logout.communicate(timeout=30)
for line in cf_logout_output.splitlines():
commons.printMSG(CloudFoundry.clazz, method,
line.decode('utf-8'))
if cf_logout.returncode != 0:
commons.printMSG(
CloudFoundry.clazz, method,
"Failed calling {command}. Return code of {"
"rtn}".format(command=cmd,
rtn=cf_logout.returncode), 'ERROR')
logout_failed = True
except TimeoutExpired:
commons.printMSG(CloudFoundry.clazz, method,
"Timed out calling {}".format(cmd), 'ERROR')
logout_failed = True
if logout_failed:
cf_logout.kill()
os.system('stty sane')
exit(1)
commons.printMSG(CloudFoundry.clazz, method, 'end')