def load_deepomake_file(loaded_files, blacklist, service_providers,
service_dependencies, file):
if file in loaded_files:
return
if file in blacklist:
return
# Load YAML and check version
try:
with open(file, 'r') as stream:
data = yaml.load(stream)
except yaml.parser.ParserError as e:
raise DMakeException(str(e))
if 'dmake_version' not in data:
raise DMakeException("Missing field 'dmake_version' in %s" % file)
version = str(data['dmake_version'])
if version not in ['0.1']:
raise DMakeException("Incorrect version '%s'" %
str(data['dmake_version']))
# Load appropriate version (TODO: versionning)
if version == '0.1':
deepomake = DeepoMakeFile(file, data)
loaded_files[file] = deepomake
# Blacklist should be on child file because they are loaded this way
for bl in deepomake.blacklist:
blacklist.append(bl)
for link in deepomake.docker_links:
add_service_provider(
service_providers,
'links/%s/%s' % (deepomake.get_app_name(), link.link_name), file)
# Unroll docker image references
if common.is_string(deepomake.docker):
ref = deepomake.docker
load_deepomake_file(loaded_files, blacklist, service_providers,
service_dependencies, ref)
deepomake.__fields__['docker'] = loaded_files[ref].docker
else:
if common.is_string(deepomake.docker.root_image):
ref = deepomake.docker.root_image
load_deepomake_file(loaded_files, blacklist, service_providers,
service_dependencies, ref)
deepomake.docker.__fields__['root_image'] = loaded_files[
ref].docker.get_docker_base_image_name_tag()
else:
deepomake.docker.__fields__[
'root_image'] = deepomake.docker.root_image.full_name()
# If a base image is declared
root_image = deepomake.docker.root_image
base_image = deepomake.docker.get_docker_base_image_name_tag()
if root_image != base_image:
add_service_provider(service_providers, base_image, file)
service_dependencies[('base', base_image)] = [('base', root_image)]
class DeepoMakeFileSerializer(YAML2PipelineSerializer):
dmake_version = FieldSerializer("string",
help_text="The deepomake version.",
example="0.1")
app_name = FieldSerializer("string",
help_text="The application name.",
example="my_app",
no_slash_no_space=True)
blacklist = FieldSerializer(
"array",
child="path",
default=[],
help_text="List of deepomake files to blacklist.",
child_path_only=True,
example=['some/sub/deepomake.yml'])
env = EnvSerializer(
help_text="Environment variables to embed in built docker images.")
docker = FieldSerializer([
FieldSerializer(
"path",
help_text=
"to another deepomake file (which will be added to dependencies) that declares a docker field, in which case it replaces this file's docker field."
),
DockerSerializer()
],
help_text=
"The environment in which to build and deploy.")
docker_links = FieldSerializer(
"array",
child=DockerLinkSerializer(),
default=[],
help_text=
"List of link to create, they are shared across the whole application, so potentially across multiple deepomake files."
)
build_tests_commands = FieldSerializer(
"array",
default=[],
child=FieldSerializer(["string", "array"],
child="string",
post_validation=lambda x: [x]
if common.is_string(x) else x),
help_text=
"Command list (or list of lists, in which case each list of commands will be executed in paralell) to build.",
example=["cmake .", "make"])
build_services_commands = FieldSerializer(
"array",
default=[],
child=FieldSerializer(["string", "array"],
child="string",
post_validation=lambda x: [x]
if common.is_string(x) else x),
help_text=
"Command list (or list of lists, in which case each list of commands will be executed in paralell) to build.",
example=["cmake .", "make"])
services = FieldSerializer("array",
child=ServicesSerializer(),
default=[],
help_text="Service list.")
def generate_command_bash(file, cmds):
file.write('set -e\n')
for cmd, kwargs in cmds:
if cmd == "stage":
file.write("echo %s\n" % kwargs['name'])
elif cmd == "sh":
commands = kwargs['shell']
if common.is_string(commands):
commands = [commands]
for c in commands:
file.write("%s\n" % c)
elif cmd == "read_sh":
file.write("export %s=`%s`\n" % (kwargs['var'], kwargs['shell']))
if kwargs['fail_if_empty']:
file.write("if [ -z \"${%s}\" ]; then exit 1; fi\n" %
kwargs['var'])
elif cmd == "env":
file.write('export %s="%s"\n' %
(kwargs['var'], kwargs['value'].replace('"', '\\"')))
elif cmd == "git_tag":
file.write('git tag --force %s\n' % kwargs['tag'])
file.write('git push --force --tags || echo %s\n' %
tag_push_error_msg)
elif cmd == "junit":
pass # Should be configured with GUI
elif cmd == "cobertura":
pass # Should be configured with GUI
elif cmd == "publishHTML":
pass # Should be configured with GUI
elif cmd == "build":
pass # Should be configured with GUI
else:
raise DMakeException("Unknown command %s" % cmd)
def _validate_(self, path, data):
if data is None:
if self.__optional__:
return None
data = {}
for name, serializer in self.__fields__.items():
try:
serializer._validate_(path,
data[name] if name in data else None)
except ValidationError as e:
raise ValidationError("Error with field '%s': %s" %
(name, str(e)))
for key in data:
if not common.is_string(key):
raise ValidationError("Expected a field name, got: '%s'" %
str(key))
if key not in self.__fields__:
raise ValidationError("Unexpected field '%s'" % key)
self.__has_value__ = True
return self
class DeployStageSerializer(YAML2PipelineSerializer):
description = FieldSerializer("string",
example="Deployment on AWS and via SSH",
help_text="Deploy stage description.")
branches = FieldSerializer(
["string", "array"],
child="string",
default=['stag'],
post_validation=lambda x: [x] if common.is_string(x) else x,
help_text=
"Branch list for which this stag is active, '*' can be used to match any branch. Can also be a simple string."
)
env = FieldSerializer(
"dict",
child="string",
default={},
example={
'AWS_ACCESS_KEY_ID': '1234',
'AWS_SECRET_ACCESS_KEY': 'abcd'
},
help_text="Additionnal environment variables for deployment.")
aws_beanstalk = AWSBeanStalkDeploySerializer(
optional=True, help_text="Deploy via Elastic Beanstalk")
ssh = SSHDeploySerializer(optional=True, help_text="Deploy via SSH")
def generate_command_pipeline(file, cmds):
file.write('{ ->\n')
if common.build_description is not None:
file.write("currentBuild.description = '%s'\n" %
common.build_description.replace("'", "\\'"))
file.write('node {\n')
file.write('try {\n')
# Check crendentials
file.write(
"try {withCredentials([[$class: 'UsernamePasswordMultiBinding', credentialsId: 'dmake-http', usernameVariable: '_', passwordVariable: '__']]) {}}\n"
)
file.write(
"""catch(error) {\nsh('echo "WARNING: Jenkins credentials \\'dmake-http\\' are not defined: you won\\'t be able to deploy only the part of your app that have changed."')\n}\n"""
)
for cmd, kwargs in cmds:
if cmd == "stage":
name = kwargs['name'].replace("'", "\\'")
if kwargs['concurrency'] is not None:
file.write("stage concurrency: %s, name: '%s'\n" %
(str(kwargs['concurrency']), name))
else:
file.write("stage '%s'\n" % name)
elif cmd == "sh":
commands = kwargs['shell']
if common.is_string(commands):
commands = [commands]
commands = [
c.replace("'", "\\'").replace("$", "\\$") for c in commands
]
if len(commands) == 0:
return
if len(commands) == 1:
file.write("sh('%s')\n" % commands[0])
else:
file.write('parallel (\n')
commands_list = []
for c in enumerate(commands):
commands_list.append("cmd%d: { sh('%s') }" % c)
file.write(',\n'.join(commands_list))
file.write(')\n')
elif cmd == "read_sh":
file_output = os.path.join(common.root_dir, ".dmake",
"output_%d" % kwargs['id'])
file.write("sh('%s > %s')\n" % (kwargs['shell'], file_output))
file.write("env.%s = readFile '%s'\n" %
(kwargs['var'], file_output))
if kwargs['fail_if_empty']:
file.write("sh('if [ -z \"${%s}\" ]; then exit 1; fi')\n" %
kwargs['var'])
elif cmd == "env":
file.write('env.%s = "%s"\n' % (kwargs['var'], kwargs['value']))
elif cmd == "git_tag":
url = common.repo_url
if url is not None and (url.startswith('https://')
or url.startswith('http://')):
i = url.find(':')
prefix = url[:i]
host = url[(i + 3):]
file.write("sh('git tag --force %s')\n" % kwargs['tag'])
file.write('try {\n')
file.write(
"withCredentials([[$class: 'UsernamePasswordMultiBinding', credentialsId: 'dmake-http', usernameVariable: 'GIT_USERNAME', passwordVariable: 'GIT_PASSWORD']]) {\n"
)
file.write("sh('env')\n")
file.write('try {\n')
file.write(
"sh('git push %s://${GIT_USERNAME}:${GIT_PASSWORD}@%s --force --tags')\n"
% (prefix, host))
file.write("""} catch(error) {\nsh('echo "%s"')\n}\n""" %
tag_push_error_msg.replace("'", "\\'"))
file.write("}\n")
file.write(
"""} catch(error) {\nsh('echo "Credentials \\'dmake-http\\' are not defined, skipping tag pushing."')\n}\n"""
)
elif cmd == "junit":
file.write("junit '%s'\n" % kwargs['report'])
elif cmd == "cobertura":
pass
elif cmd == "publishHTML":
file.write(
"publishHTML(target: [allowMissing: false, alwaysLinkToLastBuild: true, keepAll: false, reportDir: '%s', reportFiles: '%s', reportName: '%s'])\n"
% (kwargs['directory'], kwargs['index'],
kwargs['title'].replace("'", "\'")))
elif cmd == "build":
parameters = []
for var, value in kwargs['parameters'].items():
value = common.eval_str_in_env(value)
parameters.append(
"string(name: '%s', value: '%s')" %
(var.replace("'", "\\'"), value.replace("'", "\\'")))
parameters = ','.join(parameters)
file.write(
"build job: '%s', parameters: [%s], propagate: %s, wait: %s\n"
% (kwargs['job'].replace("'", "\\'"), parameters,
"true" if kwargs['propagate'] else "false",
"true" if kwargs['wait'] else "false"))
else:
raise DMakeException("Unknown command %s" % cmd)
file.write('}\n')
file.write('finally {\n')
file.write('sh("deepomake_clean %s")\n' % common.tmp_dir)
file.write('sh("sudo chown jenkins:jenkins * -R")\n')
file.write('}\n}}\n')
def _validate_type_(self, path, data_type, data):
if data_type == "bool":
if not isinstance(data, bool):
raise WrongType("Expecting bool")
return data
elif data_type == "int":
if isinstance(data, int) or isinstance(data, float):
data = int(data)
if not isinstance(data, int):
raise WrongType("Expecting int")
return str(data)
elif data_type == "string":
if isinstance(data, int) or isinstance(data, float):
data = str(data)
if not common.is_string(data):
raise WrongType("Expecting string")
if not self.blank and data == "":
raise WrongType("Expecting non-blank string")
if self.no_slash_no_space:
for c in ['/', ' ']:
if data.find(c) >= 0:
raise ValidationError("Character '%s' not allowed" % c)
return data
elif data_type == "path" or data_type == "dir":
if not common.is_string(data):
raise WrongType("Expecting string")
if len(data) > 0 and data[0] == '/':
data = data[1:]
full_path = data
if self.child_path_only:
if full_path.startswith(path):
data = data[len(path)]
if len(data) > 0 and data[0] == '/':
data = data[1:]
else:
raise WrongType(
"Path must be sub-paths to deepomake file for this field."
)
else:
full_path = os.path.join(path, data)
if not self.child_path_only:
data = full_path
data = os.path.normpath(data)
if data.startswith("../"):
raise WrongType(
"Trying to access a parent directory is forbidden")
if self.check_path:
if data_type == "path":
if not os.path.isfile(full_path):
raise WrongType("Could not find file: %s" % data)
if self.executable and not os.access(full_path, os.X_OK):
raise WrongType("The file must be executable: %s" %
full_path)
else:
if not os.path.isdir(full_path):
raise WrongType("Could not find directory: %s" % data)
return data
elif data_type == "array":
if not isinstance(data, list):
raise WrongType("Expecting array")
valid_data = []
for d in data:
child = copy.deepcopy(self.child)
valid_data.append(child._validate_(path, d))
return valid_data
elif data_type == "dict":
if not isinstance(data, dict):
raise WrongType("Expecting dict")
valid_data = {}
for k, d in data.items():
child = copy.deepcopy(self.child)
try:
valid_data[k] = child._validate_(path, d)
except ValidationError as e:
raise ValidationError("Error with field '%s': %s" %
(k, str(e)))
return valid_data
else:
raise DMakeException("Unkown data type: %s" % data_type)