def load_deepomake_file(loaded_files, blacklist, service_providers, service_dependencies, file): if file in loaded_files: return if file in blacklist: return # Load YAML and check version try: with open(file, 'r') as stream: data = yaml.load(stream) except yaml.parser.ParserError as e: raise DMakeException(str(e)) if 'dmake_version' not in data: raise DMakeException("Missing field 'dmake_version' in %s" % file) version = str(data['dmake_version']) if version not in ['0.1']: raise DMakeException("Incorrect version '%s'" % str(data['dmake_version'])) # Load appropriate version (TODO: versionning) if version == '0.1': deepomake = DeepoMakeFile(file, data) loaded_files[file] = deepomake # Blacklist should be on child file because they are loaded this way for bl in deepomake.blacklist: blacklist.append(bl) for link in deepomake.docker_links: add_service_provider( service_providers, 'links/%s/%s' % (deepomake.get_app_name(), link.link_name), file) # Unroll docker image references if common.is_string(deepomake.docker): ref = deepomake.docker load_deepomake_file(loaded_files, blacklist, service_providers, service_dependencies, ref) deepomake.__fields__['docker'] = loaded_files[ref].docker else: if common.is_string(deepomake.docker.root_image): ref = deepomake.docker.root_image load_deepomake_file(loaded_files, blacklist, service_providers, service_dependencies, ref) deepomake.docker.__fields__['root_image'] = loaded_files[ ref].docker.get_docker_base_image_name_tag() else: deepomake.docker.__fields__[ 'root_image'] = deepomake.docker.root_image.full_name() # If a base image is declared root_image = deepomake.docker.root_image base_image = deepomake.docker.get_docker_base_image_name_tag() if root_image != base_image: add_service_provider(service_providers, base_image, file) service_dependencies[('base', base_image)] = [('base', root_image)]
class DeepoMakeFileSerializer(YAML2PipelineSerializer): dmake_version = FieldSerializer("string", help_text="The deepomake version.", example="0.1") app_name = FieldSerializer("string", help_text="The application name.", example="my_app", no_slash_no_space=True) blacklist = FieldSerializer( "array", child="path", default=[], help_text="List of deepomake files to blacklist.", child_path_only=True, example=['some/sub/deepomake.yml']) env = EnvSerializer( help_text="Environment variables to embed in built docker images.") docker = FieldSerializer([ FieldSerializer( "path", help_text= "to another deepomake file (which will be added to dependencies) that declares a docker field, in which case it replaces this file's docker field." ), DockerSerializer() ], help_text= "The environment in which to build and deploy.") docker_links = FieldSerializer( "array", child=DockerLinkSerializer(), default=[], help_text= "List of link to create, they are shared across the whole application, so potentially across multiple deepomake files." ) build_tests_commands = FieldSerializer( "array", default=[], child=FieldSerializer(["string", "array"], child="string", post_validation=lambda x: [x] if common.is_string(x) else x), help_text= "Command list (or list of lists, in which case each list of commands will be executed in paralell) to build.", example=["cmake .", "make"]) build_services_commands = FieldSerializer( "array", default=[], child=FieldSerializer(["string", "array"], child="string", post_validation=lambda x: [x] if common.is_string(x) else x), help_text= "Command list (or list of lists, in which case each list of commands will be executed in paralell) to build.", example=["cmake .", "make"]) services = FieldSerializer("array", child=ServicesSerializer(), default=[], help_text="Service list.")
def generate_command_bash(file, cmds): file.write('set -e\n') for cmd, kwargs in cmds: if cmd == "stage": file.write("echo %s\n" % kwargs['name']) elif cmd == "sh": commands = kwargs['shell'] if common.is_string(commands): commands = [commands] for c in commands: file.write("%s\n" % c) elif cmd == "read_sh": file.write("export %s=`%s`\n" % (kwargs['var'], kwargs['shell'])) if kwargs['fail_if_empty']: file.write("if [ -z \"${%s}\" ]; then exit 1; fi\n" % kwargs['var']) elif cmd == "env": file.write('export %s="%s"\n' % (kwargs['var'], kwargs['value'].replace('"', '\\"'))) elif cmd == "git_tag": file.write('git tag --force %s\n' % kwargs['tag']) file.write('git push --force --tags || echo %s\n' % tag_push_error_msg) elif cmd == "junit": pass # Should be configured with GUI elif cmd == "cobertura": pass # Should be configured with GUI elif cmd == "publishHTML": pass # Should be configured with GUI elif cmd == "build": pass # Should be configured with GUI else: raise DMakeException("Unknown command %s" % cmd)
def _validate_(self, path, data): if data is None: if self.__optional__: return None data = {} for name, serializer in self.__fields__.items(): try: serializer._validate_(path, data[name] if name in data else None) except ValidationError as e: raise ValidationError("Error with field '%s': %s" % (name, str(e))) for key in data: if not common.is_string(key): raise ValidationError("Expected a field name, got: '%s'" % str(key)) if key not in self.__fields__: raise ValidationError("Unexpected field '%s'" % key) self.__has_value__ = True return self
class DeployStageSerializer(YAML2PipelineSerializer): description = FieldSerializer("string", example="Deployment on AWS and via SSH", help_text="Deploy stage description.") branches = FieldSerializer( ["string", "array"], child="string", default=['stag'], post_validation=lambda x: [x] if common.is_string(x) else x, help_text= "Branch list for which this stag is active, '*' can be used to match any branch. Can also be a simple string." ) env = FieldSerializer( "dict", child="string", default={}, example={ 'AWS_ACCESS_KEY_ID': '1234', 'AWS_SECRET_ACCESS_KEY': 'abcd' }, help_text="Additionnal environment variables for deployment.") aws_beanstalk = AWSBeanStalkDeploySerializer( optional=True, help_text="Deploy via Elastic Beanstalk") ssh = SSHDeploySerializer(optional=True, help_text="Deploy via SSH")
def generate_command_pipeline(file, cmds): file.write('{ ->\n') if common.build_description is not None: file.write("currentBuild.description = '%s'\n" % common.build_description.replace("'", "\\'")) file.write('node {\n') file.write('try {\n') # Check crendentials file.write( "try {withCredentials([[$class: 'UsernamePasswordMultiBinding', credentialsId: 'dmake-http', usernameVariable: '_', passwordVariable: '__']]) {}}\n" ) file.write( """catch(error) {\nsh('echo "WARNING: Jenkins credentials \\'dmake-http\\' are not defined: you won\\'t be able to deploy only the part of your app that have changed."')\n}\n""" ) for cmd, kwargs in cmds: if cmd == "stage": name = kwargs['name'].replace("'", "\\'") if kwargs['concurrency'] is not None: file.write("stage concurrency: %s, name: '%s'\n" % (str(kwargs['concurrency']), name)) else: file.write("stage '%s'\n" % name) elif cmd == "sh": commands = kwargs['shell'] if common.is_string(commands): commands = [commands] commands = [ c.replace("'", "\\'").replace("$", "\\$") for c in commands ] if len(commands) == 0: return if len(commands) == 1: file.write("sh('%s')\n" % commands[0]) else: file.write('parallel (\n') commands_list = [] for c in enumerate(commands): commands_list.append("cmd%d: { sh('%s') }" % c) file.write(',\n'.join(commands_list)) file.write(')\n') elif cmd == "read_sh": file_output = os.path.join(common.root_dir, ".dmake", "output_%d" % kwargs['id']) file.write("sh('%s > %s')\n" % (kwargs['shell'], file_output)) file.write("env.%s = readFile '%s'\n" % (kwargs['var'], file_output)) if kwargs['fail_if_empty']: file.write("sh('if [ -z \"${%s}\" ]; then exit 1; fi')\n" % kwargs['var']) elif cmd == "env": file.write('env.%s = "%s"\n' % (kwargs['var'], kwargs['value'])) elif cmd == "git_tag": url = common.repo_url if url is not None and (url.startswith('https://') or url.startswith('http://')): i = url.find(':') prefix = url[:i] host = url[(i + 3):] file.write("sh('git tag --force %s')\n" % kwargs['tag']) file.write('try {\n') file.write( "withCredentials([[$class: 'UsernamePasswordMultiBinding', credentialsId: 'dmake-http', usernameVariable: 'GIT_USERNAME', passwordVariable: 'GIT_PASSWORD']]) {\n" ) file.write("sh('env')\n") file.write('try {\n') file.write( "sh('git push %s://${GIT_USERNAME}:${GIT_PASSWORD}@%s --force --tags')\n" % (prefix, host)) file.write("""} catch(error) {\nsh('echo "%s"')\n}\n""" % tag_push_error_msg.replace("'", "\\'")) file.write("}\n") file.write( """} catch(error) {\nsh('echo "Credentials \\'dmake-http\\' are not defined, skipping tag pushing."')\n}\n""" ) elif cmd == "junit": file.write("junit '%s'\n" % kwargs['report']) elif cmd == "cobertura": pass elif cmd == "publishHTML": file.write( "publishHTML(target: [allowMissing: false, alwaysLinkToLastBuild: true, keepAll: false, reportDir: '%s', reportFiles: '%s', reportName: '%s'])\n" % (kwargs['directory'], kwargs['index'], kwargs['title'].replace("'", "\'"))) elif cmd == "build": parameters = [] for var, value in kwargs['parameters'].items(): value = common.eval_str_in_env(value) parameters.append( "string(name: '%s', value: '%s')" % (var.replace("'", "\\'"), value.replace("'", "\\'"))) parameters = ','.join(parameters) file.write( "build job: '%s', parameters: [%s], propagate: %s, wait: %s\n" % (kwargs['job'].replace("'", "\\'"), parameters, "true" if kwargs['propagate'] else "false", "true" if kwargs['wait'] else "false")) else: raise DMakeException("Unknown command %s" % cmd) file.write('}\n') file.write('finally {\n') file.write('sh("deepomake_clean %s")\n' % common.tmp_dir) file.write('sh("sudo chown jenkins:jenkins * -R")\n') file.write('}\n}}\n')
def _validate_type_(self, path, data_type, data): if data_type == "bool": if not isinstance(data, bool): raise WrongType("Expecting bool") return data elif data_type == "int": if isinstance(data, int) or isinstance(data, float): data = int(data) if not isinstance(data, int): raise WrongType("Expecting int") return str(data) elif data_type == "string": if isinstance(data, int) or isinstance(data, float): data = str(data) if not common.is_string(data): raise WrongType("Expecting string") if not self.blank and data == "": raise WrongType("Expecting non-blank string") if self.no_slash_no_space: for c in ['/', ' ']: if data.find(c) >= 0: raise ValidationError("Character '%s' not allowed" % c) return data elif data_type == "path" or data_type == "dir": if not common.is_string(data): raise WrongType("Expecting string") if len(data) > 0 and data[0] == '/': data = data[1:] full_path = data if self.child_path_only: if full_path.startswith(path): data = data[len(path)] if len(data) > 0 and data[0] == '/': data = data[1:] else: raise WrongType( "Path must be sub-paths to deepomake file for this field." ) else: full_path = os.path.join(path, data) if not self.child_path_only: data = full_path data = os.path.normpath(data) if data.startswith("../"): raise WrongType( "Trying to access a parent directory is forbidden") if self.check_path: if data_type == "path": if not os.path.isfile(full_path): raise WrongType("Could not find file: %s" % data) if self.executable and not os.access(full_path, os.X_OK): raise WrongType("The file must be executable: %s" % full_path) else: if not os.path.isdir(full_path): raise WrongType("Could not find directory: %s" % data) return data elif data_type == "array": if not isinstance(data, list): raise WrongType("Expecting array") valid_data = [] for d in data: child = copy.deepcopy(self.child) valid_data.append(child._validate_(path, d)) return valid_data elif data_type == "dict": if not isinstance(data, dict): raise WrongType("Expecting dict") valid_data = {} for k, d in data.items(): child = copy.deepcopy(self.child) try: valid_data[k] = child._validate_(path, d) except ValidationError as e: raise ValidationError("Error with field '%s': %s" % (k, str(e))) return valid_data else: raise DMakeException("Unkown data type: %s" % data_type)