def get_modules_changed(path, ref='HEAD'):
    '''Get modules changed from git diff-index {ref}
    :param path: String path of git repo
    :param ref: branch or remote/branch or sha to compare
    :return: List of paths of modules changed
    '''
    git_run_obj = GitRun(os.path.join(path, '.git'))
    if ref != 'HEAD':
        fetch_ref = ref
        if ':' not in fetch_ref:
            # to force create branch
            fetch_ref += ':' + fetch_ref
        git_run_obj.run(['fetch'] + fetch_ref.split('/', 1))
    items_changed = git_run_obj.get_items_changed(ref)
    folders_changed = set([
        item_changed.split('/')[0]
        for item_changed in items_changed
        if '/' in item_changed]
    )
    modules = set(get_modules(path))
    modules_changed = list(modules & folders_changed)
    modules_changed_path = [
        os.path.join(path, module_changed)
        for module_changed in modules_changed]
    return modules_changed_path
예제 #2
0
def get_modules_changed(path, ref='HEAD'):
    '''Get modules changed from git diff-index {ref}
    :param path: String path of git repo
    :param ref: branch or remote/branch or sha to compare
    :return: List of paths of modules changed
    '''
    git_run_obj = GitRun(os.path.join(path, '.git'))
    if ref != 'HEAD':
        fetch_ref = ref
        if ':' not in fetch_ref:
            # to force create branch
            fetch_ref += ':' + fetch_ref
        git_run_obj.run(['fetch'] + fetch_ref.split('/', 1))
    items_changed = git_run_obj.get_items_changed(ref)
    folders_changed = set([
        item_changed.split('/')[0]
        for item_changed in items_changed
        if '/' in item_changed]
    )
    modules = set(get_modules(path))
    modules_changed = list(modules & folders_changed)
    modules_changed_path = [
        os.path.join(path, module_changed)
        for module_changed in modules_changed]
    return modules_changed_path
def version_validate(version, dir):
    if not version and dir:
        repo_path = os.path.join(dir, '.git')
        branch_name = GitRun(repo_path).get_branch_name()
        version = (branch_name.replace('_', '-').split('-')[:1]
                   if branch_name else False)
        version = version[0] if version else None
    if not version:
        print(
            travis_helpers.yellow('Undefined environment variable'
                                  ' `VERSION`.\nSet `VERSION` for '
                                  'compatibility with guidelines by version.'))
    return version
예제 #4
0
def version_validate(version, dir):
    if not version and dir:
        repo_path = os.path.join(dir, '.git')
        branch_name = GitRun(repo_path).get_branch_name()
        version = (branch_name.replace('_', '-').split('-')[:1]
                   if branch_name else False)
        version = version[0] if version else None
    if not version:
        print(travis_helpers.yellow(
            'Undefined environment variable'
            ' `VERSION`.\nSet `VERSION` for '
            'compatibility with guidelines by version.'))
    return version
예제 #5
0
 def __init__(self, git_project, revision,
              command_format='docker', docker_user=None,
              root_path=None, default_docker_image=None,
              remotes=None, exclude_after_success=None,
              run_extra_args=None, include_cleanup=None,
              build_extra_args=''):
     """
     Method Constructor
     @fname_dockerfile: str name of file dockerfile to save.
     @format_cmd: str name of format of command.
                  bash: Make a bash script.
                  docker: Make a dockerfile script.
     """
     if root_path is None:
         self.root_path = os.path.join(
             gettempdir(),
             os.path.splitext(os.path.basename(__file__))[0],
         )
     else:
         self.root_path = os.path.expandvars(
             os.path.expanduser(root_path)
         )
     self.default_docker_image = default_docker_image
     self.git_project = git_project
     self.revision = revision
     git_path = self.get_repo_path(self.root_path)
     self.git_obj = GitRun(git_project, git_path)
     self.travis_data = self.load_travis_file(revision)
     self.sha = self.git_obj.get_sha(revision)
     self.exclude_after_success = exclude_after_success
     self.run_extra_args = run_extra_args
     self.include_cleanup = include_cleanup
     self.build_extra_args = build_extra_args
     if not self.travis_data:
         raise Exception(
             "Make sure you have access to repository"
             " " + git_project + " and that the file .travis.yml"
             " exists in branch or revision " + revision
         )
     self.travis2docker_section = [
         # ('build_image', 'build_image'),
         ('python', 'python'),
         ('env', 'env'),
         ('install', 'run'),
         ('script', 'script'),
         ('after_success', 'script'),
     ]
     self.travis2docker_section_dict = dict(self.travis2docker_section)
     self.scripts_root_path = self.get_script_path(self.root_path)
     self.env_regex_str2 = r"\s(?=\w+=)"
     env_regex_str = r"(?P<var>[\w]*)[ ]*[\=][ ]*[\"\']{0,1}" + \
         r"(?P<value>[\w\.\-\_/\$\{\}\:,\(\)\#\* ]*)[\"\']{0,1}"
     export_regex_str = r"^(?P<export>export|EXPORT)( )+" + env_regex_str
     self.env_regex = re.compile(env_regex_str, re.M)
     self.export_regex = re.compile(export_regex_str, re.M)
     self.extra_env_from_run = ""
     self.command_format = command_format
     self.docker_user = docker_user or 'root'
     self.remotes = remotes or []
예제 #6
0
class GitRmAllLog(object):

    def __init__(self, git_url,
                 root_path=None):
        self.git_url = git_url
        if root_path is None:
            root_path = gettempdir()
        else:
            root_path = os.path.expandvars(
                os.path.expanduser(root_path)
            )
        self.git_run_obj = GitRun(git_url, None)
        self.git_run_obj.path = os.path.join(
            root_path,
            self.git_run_obj.owner,
            self.git_run_obj.repo
        )

    def rm_all_log(self, items):
        self.git_run_obj.update()
        refs = self.git_run_obj.get_ref_data(['refs/heads']).keys()
        for ref in refs:
            self.git_run_obj.checkout_bare(ref)
            self.git_run_obj.run([
                "filter-branch", "-f", "--tree-filter",
                "rm -rf %s" % items, "HEAD"
            ])
예제 #7
0
 def __init__(self):
     self._git = GitRun(os.path.join(os.getcwd(), '.git'), True)
     self.branch = os.environ.get("TRAVIS_BRANCH",
                                  self._git.get_branch_name())
     remote = self._git.run(["ls-remote", "--get-url", "origin"])
     name = remote.replace(':', '/')
     name = re.sub('.+@', '', name)
     name = re.sub('.git$', '', name)
     name = re.sub('^https://', '', name)
     name = re.sub('^http://', '', name)
     match = re.search(
         r'(?P<host>[^/]+)/(?P<owner>[^/]+)/(?P<repo>[^/]+)', name)
     if match:
         name = ("%(host)s:%(owner)s/%(repo)s (%(branch)s)" %
                 dict(match.groupdict(), branch=self.branch))
     self.repo_name = name
     self.wl_api = WeblateApi()
     self.gh_api = GitHubApi()
     self._travis_home = os.environ.get("HOME", "~/")
     self._travis_build_dir = os.environ.get("TRAVIS_BUILD_DIR", "../..")
     self._odoo_version = os.environ.get("VERSION")
     self._odoo_branch = os.environ.get("ODOO_BRANCH")
     self._langs = (parse_list(os.environ.get("LANG_ALLOWED")) if
                    os.environ.get("LANG_ALLOWED", False) else [])
     self._odoo_full = os.environ.get("ODOO_REPO", "odoo/odoo")
     self._server_path = get_server_path(self._odoo_full,
                                         (self._odoo_branch or
                                          self._odoo_version),
                                         self._travis_home)
     self._addons_path = get_addons_path(self._travis_home,
                                         self._travis_build_dir,
                                         self._server_path)
     self._database = os.environ.get('MQT_TEST_DB', 'openerp_test')
     self._connection_context = context_mapping.get(
         self._odoo_version, Odoo10Context)
     self._apply_patch_odoo()
     self._get_modules_installed()
예제 #8
0
 def __init__(self, git_url,
              root_path=None):
     self.git_url = git_url
     if root_path is None:
         root_path = gettempdir()
     else:
         root_path = os.path.expandvars(
             os.path.expanduser(root_path)
         )
     self.git_run_obj = GitRun(git_url, None)
     self.git_run_obj.path = os.path.join(
         root_path,
         self.git_run_obj.owner,
         self.git_run_obj.repo
     )
 def __init__(self):
     self._git = GitRun(os.path.join(os.getcwd(), '.git'), True)
     self.branch = os.environ.get("TRAVIS_BRANCH",
                                  self._git.get_branch_name())
     remote = self._git.run(["ls-remote", "--get-url", "origin"])
     name = remote.replace(':', '/')
     name = re.sub('.+@', '', name)
     name = re.sub('.git$', '', name)
     name = re.sub('^https://', '', name)
     name = re.sub('^http://', '', name)
     match = re.search(
         r'(?P<host>[^/]+)/(?P<owner>[^/]+)/(?P<repo>[^/]+)', name)
     if match:
         name = ("%(host)s:%(owner)s/%(repo)s (%(branch)s)" %
                 dict(match.groupdict(), branch=self.branch))
     self.repo_name = name
     self.wl_api = WeblateApi()
     self.gh_api = GitHubApi()
     self._travis_home = os.environ.get("HOME", "~/")
     self._travis_build_dir = os.environ.get("TRAVIS_BUILD_DIR", "../..")
     self._odoo_version = os.environ.get("VERSION")
     self._odoo_branch = os.environ.get("ODOO_BRANCH")
     self._langs = (parse_list(os.environ.get("LANG_ALLOWED")) if
                    os.environ.get("LANG_ALLOWED", False) else [])
     self._odoo_full = os.environ.get("ODOO_REPO", "odoo/odoo")
     self._server_path = get_server_path(self._odoo_full,
                                         (self._odoo_branch or
                                          self._odoo_version),
                                         self._travis_home)
     self._addons_path = get_addons_path(self._travis_home,
                                         self._travis_build_dir,
                                         self._server_path)
     self._database = os.environ.get('MQT_TEST_DB', 'openerp_test')
     self._connection_context = context_mapping.get(
         self._odoo_version, Odoo10Context)
     self._apply_patch_odoo()
     self._get_modules_installed()
예제 #10
0
 def __init__(self,
              git_project,
              revision,
              command_format='docker',
              docker_user=None,
              root_path=None,
              default_docker_image=None,
              remotes=None,
              exclude_after_success=None,
              run_extra_args=None,
              include_cleanup=None,
              build_extra_args=''):
     """
     Method Constructor
     @fname_dockerfile: str name of file dockerfile to save.
     @format_cmd: str name of format of command.
                  bash: Make a bash script.
                  docker: Make a dockerfile script.
     """
     if root_path is None:
         self.root_path = os.path.join(
             gettempdir(),
             os.path.splitext(os.path.basename(__file__))[0],
         )
     else:
         self.root_path = os.path.expandvars(os.path.expanduser(root_path))
     self.default_docker_image = default_docker_image
     self.git_project = git_project
     self.revision = revision
     git_path = self.get_repo_path(self.root_path)
     self.git_obj = GitRun(git_project, git_path)
     self.travis_data = self.load_travis_file(revision)
     self.sha = self.git_obj.get_sha(revision)
     self.exclude_after_success = exclude_after_success
     self.run_extra_args = run_extra_args
     self.include_cleanup = include_cleanup
     self.build_extra_args = build_extra_args
     if not self.travis_data:
         raise Exception("Make sure you have access to repository"
                         " " + git_project +
                         " and that the file .travis.yml"
                         " exists in branch or revision " + revision)
     self.travis2docker_section = [
         # ('build_image', 'build_image'),
         ('python', 'python'),
         ('env', 'env'),
         ('install', 'run'),
         ('script', 'script'),
         ('after_success', 'script'),
     ]
     self.travis2docker_section_dict = dict(self.travis2docker_section)
     self.scripts_root_path = self.get_script_path(self.root_path)
     env_regex_str = r"(?P<var>[\w]*)[ ]*[\=][ ]*[\"\']{0,1}" + \
         r"(?P<value>[\w\.\-\_/\$\{\}\:,\(\)\#\* ]*)[\"\']{0,1}"
     export_regex_str = r"^(?P<export>export|EXPORT)( )+" + env_regex_str
     self.env_regex = re.compile(env_regex_str, re.M)
     self.export_regex = re.compile(export_regex_str, re.M)
     self.extra_env_from_run = ""
     self.command_format = command_format
     self.docker_user = docker_user or 'root'
     self.remotes = remotes
예제 #11
0
class travis(object):
    def load_travis_file(self, branch):
        self.git_obj.update()
        yaml_loaded = None
        for fname in ['.travis.yml', '.shippable.yml']:
            yaml_str = self.git_obj.show_file(fname, branch)
            try:
                yaml_loaded = yaml.load(yaml_str)
                break
            except AttributeError:
                pass
        return yaml_loaded

    def get_folder_name(self, name):
        for invalid_char in '@:/#':
            name = name.replace(invalid_char, '_')
        return name

    def get_repo_path(self, root_path):
        name = self.get_folder_name(self.git_project.lower())
        repo_path = os.path.join(root_path, 'repo', name)
        return repo_path

    def get_script_path(self, root_path):
        name = self.get_folder_name(self.git_project.lower())
        script_path = os.path.join(root_path, 'script', name,
                                   self.get_folder_name(self.revision),
                                   self.sha)
        return script_path

    def __init__(self,
                 git_project,
                 revision,
                 command_format='docker',
                 docker_user=None,
                 root_path=None,
                 default_docker_image=None,
                 remotes=None,
                 exclude_after_success=None,
                 run_extra_args=None,
                 include_cleanup=None,
                 build_extra_args=''):
        """
        Method Constructor
        @fname_dockerfile: str name of file dockerfile to save.
        @format_cmd: str name of format of command.
                     bash: Make a bash script.
                     docker: Make a dockerfile script.
        """
        if root_path is None:
            self.root_path = os.path.join(
                gettempdir(),
                os.path.splitext(os.path.basename(__file__))[0],
            )
        else:
            self.root_path = os.path.expandvars(os.path.expanduser(root_path))
        self.default_docker_image = default_docker_image
        self.git_project = git_project
        self.revision = revision
        git_path = self.get_repo_path(self.root_path)
        self.git_obj = GitRun(git_project, git_path)
        self.travis_data = self.load_travis_file(revision)
        self.sha = self.git_obj.get_sha(revision)
        self.exclude_after_success = exclude_after_success
        self.run_extra_args = run_extra_args
        self.include_cleanup = include_cleanup
        self.build_extra_args = build_extra_args
        if not self.travis_data:
            raise Exception("Make sure you have access to repository"
                            " " + git_project +
                            " and that the file .travis.yml"
                            " exists in branch or revision " + revision)
        self.travis2docker_section = [
            # ('build_image', 'build_image'),
            ('python', 'python'),
            ('env', 'env'),
            ('install', 'run'),
            ('script', 'script'),
            ('after_success', 'script'),
        ]
        self.travis2docker_section_dict = dict(self.travis2docker_section)
        self.scripts_root_path = self.get_script_path(self.root_path)
        env_regex_str = r"(?P<var>[\w]*)[ ]*[\=][ ]*[\"\']{0,1}" + \
            r"(?P<value>[\w\.\-\_/\$\{\}\:,\(\)\#\* ]*)[\"\']{0,1}"
        export_regex_str = r"^(?P<export>export|EXPORT)( )+" + env_regex_str
        self.env_regex = re.compile(env_regex_str, re.M)
        self.export_regex = re.compile(export_regex_str, re.M)
        self.extra_env_from_run = ""
        self.command_format = command_format
        self.docker_user = docker_user or 'root'
        self.remotes = remotes

    # def get_travis_type(self, repo_git):
    #     self.repo_git_type = None
    #     if os.path.isdir(repo_git):
    #         self.repo_git_type = 'localpath'
    #     elif os.path.isfile(repo_git):
    #         self.repo_git_type = 'file'
    #     else:
    #         regex = r"(?P<host>(git@|https://)([\w\.@]+)(/|:))" \
    #                  r"(?P<owner>[~\w,\-,\_]+)/" \
    #                  r"(?P<repo>[\w,\-,\_]+)(.git){0,1}((/){0,1})"
    #         match_object = re.search(regex, repo_git)
    #         if match_object:
    #             self.repo_git_type = 'remote'

    # def get_travis_data(self, repo_git, branch=None):
    #     self.get_travis_type(repo_git)
    #     if self.repo_git_type == 'remote':
    #         if branch is None:
    #             raise "You need specify branch name with remote repository"
    #         "git show 8.0:.travis.yml"

    def get_travis_section(self, section, *args, **kwards):
        section_type = self.travis2docker_section_dict.get(section, False)
        if not section_type:
            return None
        section_data = self.travis_data.get(section, "")
        if isinstance(section_data, basestring):
            section_data = [section_data]
        job_method = getattr(self, 'get_travis2docker_' + section_type)
        return job_method(section_data, *args, **kwards)

    def get_travis2docker_run(self,
                              section_data,
                              custom_command_format=None,
                              add_extra_env=True):
        if custom_command_format is None:
            custom_command_format = self.command_format
        docker_run = ''
        for line in section_data:
            line = line.strip()
            export_regex_findall = self.export_regex.findall(line)
            for dummy, dummy, var, value in export_regex_findall:
                self.extra_env_from_run += "\n%s=%s " % (var, value)
            extra_env = add_extra_env and \
                self.extra_env_from_run.replace('\n', ' ') or ''
            if not export_regex_findall:
                if custom_command_format == 'bash':
                    docker_run += '\n' + extra_env + line
                elif custom_command_format == 'docker':
                    if line.startswith('(') and line.endswith(')'):
                        line = line[1:-1]
                    docker_run += '\nRUN ' + extra_env + line
        return docker_run

    def scape_bash_command(self, command):
        return command.replace('$', '\\$').replace('_', '\\_')

    def get_travis2docker_script(self, section_data):
        cmd_str = self.get_travis2docker_run(section_data,
                                             custom_command_format='bash',
                                             add_extra_env=False)
        if self.docker_user == 'root':
            sudo_prefix = ''
        else:
            sudo_prefix = 'sudo'
        if self.command_format == 'docker' and cmd_str:
            cmd_str = "#!/bin/bash" + "\n" + self.extra_env_from_run.replace(
                '\n', '\nexport ') + \
                '\n' + cmd_str.replace('\n', '\\\\n').strip()
            cmd_str = '\\\\n'.join(cmd_str.strip('\n').split('\n'))
            cmd_str = cmd_str.replace('$', r'\$').replace('"', r'\"')
            cmd_str = 'RUN echo """%s"""' % (cmd_str,) \
                + ' | ' + sudo_prefix + 'tee -a /entrypoint.sh \\' + \
                '\n    && ' + sudo_prefix + \
                ' chown %s:%s /entrypoint.sh \\' % (
                    self.docker_user, self.docker_user) + \
                '\n    && ' + sudo_prefix + ' chmod +x /entrypoint.sh'
            cmd_str += '\nENTRYPOINT /entrypoint.sh'
        return cmd_str

    def get_travis2docker_env(self, section_data):
        global_data_cmd = []
        matrix_data_cmd = []
        if 'global' in section_data:
            # The section 'global' is used as a item
            # Then the global_data_cmd even is a list of a item
            global_data = section_data.pop('global')
            global_data_cmd = [
                ''.join([
                    global_env
                    for global_env in self.get_travis2docker_env(global_data)
                ])
            ]
        if 'matrix' in section_data:
            matrix_data = section_data.pop('matrix')
            matrix_data_cmd = [
                matrix_env
                for matrix_env in self.get_travis2docker_env(matrix_data)
            ]
        for combination in itertools.product(global_data_cmd, matrix_data_cmd):
            command_str = '\n'.join(combination) + "\n"
            yield command_str

        for line in section_data:
            if line in ['global', 'matrix']:
                continue
            if not isinstance(line, str):
                continue
            docker_env = ""
            for var, value in self.env_regex.findall(line):
                if self.command_format == 'bash':
                    docker_env += "\nexport %s=%s" % (var, value)
                elif self.command_format == 'docker':
                    if var:
                        # To support case `export VAR="param1=value1"`
                        docker_env += '\nENV '
                    docker_env += "%s=%s" % (var, value)
            yield docker_env

    def get_travis2docker_python(self, section_data):
        for line in section_data:
            yield "\n# TODO: Use python version: " + line

    def get_default_cmd(self, dockerfile_path):
        home_user_path = self.docker_user == 'root' and "/root" \
            or os.path.join("/home", self.docker_user)
        project, branch = self.git_project, self.revision
        travis_build_dir = os.path.join(
            home_user_path,
            "build",
            self.git_obj.owner,
            self.git_obj.repo,
        )
        if self.command_format == 'bash':
            cmd = "\nsudo su - " + self.docker_user + \
                  "\nsudo chown -R %s:%s %s" % (
                      self.docker_user, self.docker_user,
                      home_user_path) + \
                  "\nexport TRAVIS_BUILD_DIR=%s" % (travis_build_dir) + \
                  "\ngit clone --single-branch %s -b %s " % (
                      project, branch) + \
                  "${TRAVIS_BUILD_DIR}" + \
                  "\n"
        elif self.command_format == 'docker':
            dkr_files_path = os.path.join(dockerfile_path, "files")
            if not os.path.exists(dkr_files_path):
                os.makedirs(dkr_files_path)
            ssh_dest = os.path.join(dkr_files_path, 'ssh')
            if os.path.exists(ssh_dest):
                shutil.rmtree(ssh_dest)
            shutil.copytree(os.path.expanduser("~/.ssh"), ssh_dest)
            cmd_refs = 'pull' in self.revision and \
                '+refs/%s/head:refs/%s' % (
                    self.revision, self.revision) or \
                '+refs/heads/%s:refs/heads/%s' % (
                    self.revision, self.revision)

            # TODO: Use sha
            remote_default = self.git_obj.owner.lower()
            cmd_git_clone = [
                "git init ${TRAVIS_BUILD_DIR}",
                "cd ${TRAVIS_BUILD_DIR}",
                "git remote add %s %s" % (remote_default, project),
                "git fetch --update-head-ok -p %s %s" %
                (remote_default, cmd_refs),
                "git checkout -qf " + self.revision,
            ]
            git_user_email = self.git_obj.get_config_data("user.email")
            if git_user_email:
                cmd_git_clone.append("git config --global user.email \"%s\"" %
                                     (git_user_email))
            else:
                cmd_git_clone.append("git config --unset --global user.email")
            git_user_name = self.git_obj.get_config_data("user.name")
            if git_user_name:
                cmd_git_clone.append("git config --global user.name \"%s\"" %
                                     (git_user_name))
            else:
                cmd_git_clone.append("git config --unset --global user.name")
            if self.remotes:
                for remote in self.remotes:
                    remote = remote.lower()
                    if remote != remote_default:
                        git_remote_url = self.git_obj.host + remote + '/' + \
                            self.git_obj.repo + '.git'
                        cmd_git_clone.append('git remote add %s %s' %
                                             (remote, git_remote_url))
            if self.docker_user == 'root':
                sudo_prefix = ''
            else:
                sudo_prefix = 'sudo'
            cmd = 'FROM ' + (self.travis_data.get('build_image', False) or
                             self.default_docker_image) + \
                  '\nUSER ' + self.docker_user + \
                  '\nENV HOME=' + home_user_path + \
                  '\nENV TRAVIS_REPO_SLUG=%s/%s' % (
                      self.git_obj.owner, self.git_obj.repo) + \
                  '\nADD ' + os.path.join("files", 'ssh') + ' ' + \
                  os.path.join(home_user_path, '.ssh') + \
                  "\nENV TRAVIS_BUILD_DIR=%s" % (travis_build_dir) + \
                  "\nWORKDIR ${TRAVIS_BUILD_DIR}" + \
                  "\nRUN %s chown -R %s:%s %s" % (
                      sudo_prefix, self.docker_user,
                      self.docker_user, home_user_path) + \
                  "\nRUN " + ' \\\n    && '.join(cmd_git_clone) + \
                  "\n"
        return cmd

    def get_travis2docker_iter(self):
        travis2docker_cmd_static_str = ""
        travis2docker_cmd_iter_list = []
        if self.exclude_after_success:
            self.travis2docker_section.remove(('after_success', 'script'))
        for travis_section, dummy in self.travis2docker_section:
            travis2docker_section = self.get_travis_section(travis_section)
            if isinstance(travis2docker_section, types.GeneratorType):
                travis2docker_cmd_iter_list.append(
                    [item_iter for item_iter in travis2docker_section])
            elif isinstance(travis2docker_section, basestring):
                travis2docker_cmd_static_str += travis2docker_section + "\n"
        for combination in itertools.product(*travis2docker_cmd_iter_list):
            command_str = '\n'.join(combination) + "\n" + \
                travis2docker_cmd_static_str
            yield command_str

    def get_travis2docker(self):
        count = 1
        fname_scripts = []
        for cmd in self.get_travis2docker_iter():
            fname = os.path.join(self.scripts_root_path, str(count))
            fname_build = None
            fname_run = None
            if not os.path.exists(fname):
                os.makedirs(fname)
            if self.command_format == 'bash':
                fname = os.path.join(fname, 'script.sh')
                cmd = self.get_default_cmd(os.path.dirname(fname)) + cmd
            elif self.command_format == 'docker':
                fname = os.path.join(fname, 'Dockerfile')
                cmd = self.get_default_cmd(os.path.dirname(fname)) + cmd
                fname_build = os.path.join(os.path.dirname(fname),
                                           '10-build.sh')
                fname_run = os.path.join(os.path.dirname(fname), '20-run.sh')
                image_name = self.git_obj.owner + '-' + \
                    self.git_obj.repo + \
                    ":" + self.get_folder_name(self.revision) + \
                    "_" + str(count)
                image_name = image_name.lower()
            else:
                raise Exception("No command format found %s" %
                                (self.command_format))
            with open(fname, "w") as fdockerfile:
                fdockerfile.write(cmd.encode('utf-8'))

            if fname_build:
                with open(fname_build, "w") as fbuild:
                    fbuild.write("#!/bin/bash\ndocker build $1 %s -t %s %s\n" %
                                 (self.build_extra_args, image_name,
                                  os.path.dirname(fname)))
                st = os.stat(fname_build)
                os.chmod(fname_build, st.st_mode | stat.S_IEXEC)
            if fname_run:
                with open(fname_run, "w") as fbuild:
                    fbuild.write("#!/bin/bash\ndocker run " + "$1 %s %s $2\n" %
                                 (
                                     self.run_extra_args,
                                     image_name,
                                 ))
                    if self.include_cleanup:
                        fbuild.write("docker rmi -f '%s'\n" % image_name)
                st = os.stat(fname_run)
                os.chmod(fname_run, st.st_mode | stat.S_IEXEC)
            if self.command_format == 'bash':
                st = os.stat(fname)
                os.chmod(fname, st.st_mode | stat.S_IEXEC)
            count += 1
            fname_scripts.append(os.path.dirname(fname))
        return fname_scripts
예제 #12
0
class travis(object):

    def load_travis_file(self, branch):
        self.git_obj.update()
        yaml_loaded = None
        for fname in ['.travis.yml', '.shippable.yml']:
            yaml_str = self.git_obj.show_file(fname, branch)
            try:
                yaml_loaded = yaml.load(yaml_str)
                break
            except AttributeError:
                pass
        return yaml_loaded

    def get_folder_name(self, name):
        for invalid_char in '@:/#':
            name = name.replace(invalid_char, '_')
        return name

    def get_repo_path(self, root_path):
        name = self.get_folder_name(self.git_project)
        repo_path = os.path.join(root_path, 'repo', name)
        return repo_path

    def get_script_path(self, root_path):
        name = self.get_folder_name(self.git_project)
        script_path = os.path.join(
            root_path, 'script',
            name, self.get_folder_name(self.revision), self.sha)
        return script_path

    def __init__(self, git_project, revision,
                 command_format='docker', docker_user=None,
                 root_path=None, default_docker_image=None):
        """
        Method Constructor
        @fname_dockerfile: str name of file dockerfile to save.
        @format_cmd: str name of format of command.
                     bash: Make a bash script.
                     docker: Make a dockerfile script.
        """
        if root_path is None:
            self.root_path = os.path.join(
                gettempdir(),
                os.path.splitext(os.path.basename(__file__))[0],
            )
        else:
            self.root_path = os.path.expandvars(
                os.path.expanduser(root_path)
            )
        self.default_docker_image = default_docker_image
        self.git_project = git_project
        self.revision = revision
        git_path = self.get_repo_path(self.root_path)
        self.git_obj = GitRun(git_project, git_path)
        self.travis_data = self.load_travis_file(revision)
        self.sha = self.git_obj.get_sha(revision)
        if not self.travis_data:
            raise Exception(
                "Make sure you have access to repository"
                " " + git_project + " and that the file .travis.yml"
                " exists in branch or revision " + revision
            )
        self.travis2docker_section = [
            # ('build_image', 'build_image'),
            ('python', 'python'),
            ('env', 'env'),
            ('install', 'run'),
            ('script', 'script'),
        ]
        self.travis2docker_section_dict = dict(self.travis2docker_section)
        self.scripts_root_path = self.get_script_path(self.root_path)
        env_regex_str = r"(?P<var>[\w]*)[ ]*[\=][ ]*[\"\']{0,1}" + \
            r"(?P<value>[\w\.\-\_/\$\{\}\:,]*)[\"\']{0,1}"
        export_regex_str = r"(?P<export>export|EXPORT)( )+" + env_regex_str
        self.env_regex = re.compile(env_regex_str, re.M)
        self.export_regex = re.compile(export_regex_str, re.M)
        self.extra_env_from_run = ""
        self.command_format = command_format
        self.docker_user = docker_user or 'root'

    # def get_travis_type(self, repo_git):
    #     self.repo_git_type = None
    #     if os.path.isdir(repo_git):
    #         self.repo_git_type = 'localpath'
    #     elif os.path.isfile(repo_git):
    #         self.repo_git_type = 'file'
    #     else:
    #         regex = r"(?P<host>(git@|https://)([\w\.@]+)(/|:))" \
    #                  r"(?P<owner>[~\w,\-,\_]+)/" \
    #                  r"(?P<repo>[\w,\-,\_]+)(.git){0,1}((/){0,1})"
    #         match_object = re.search(regex, repo_git)
    #         if match_object:
    #             self.repo_git_type = 'remote'

    # def get_travis_data(self, repo_git, branch=None):
    #     self.get_travis_type(repo_git)
    #     if self.repo_git_type == 'remote':
    #         if branch is None:
    #             raise "You need specify branch name with remote repository"
    #         "git show 8.0:.travis.yml"

    def get_travis_section(self, section, *args, **kwards):
        section_type = self.travis2docker_section_dict.get(section, False)
        if not section_type:
            return None
        section_data = self.travis_data.get(section, "")
        if isinstance(section_data, basestring):
            section_data = [section_data]
        job_method = getattr(self, 'get_travis2docker_' + section_type)
        return job_method(section_data, *args, **kwards)

    def get_travis2docker_run(self, section_data,
                              custom_command_format=None,
                              add_extra_env=True):
        if custom_command_format is None:
            custom_command_format = self.command_format
        docker_run = ''
        for line in section_data:
            export_regex_findall = self.export_regex.findall(line)
            for dummy, dummy, var, value in export_regex_findall:
                self.extra_env_from_run += "%s=%s " % (var, value)
            extra_env = add_extra_env and self.extra_env_from_run or ''
            if not export_regex_findall:
                if custom_command_format == 'bash':
                    docker_run += '\n' + extra_env + line
                elif custom_command_format == 'docker':
                    docker_run += '\nRUN ' + extra_env + line
        return docker_run

    def scape_bash_command(self, command):
        return command.replace('$', '\\$').replace('_', '\\_')

    def get_travis2docker_script(self, section_data):
        cmd_str = self.get_travis2docker_run(
            section_data, custom_command_format='bash',
            add_extra_env=False)
        if self.docker_user == 'root':
            sudo_prefix = ''
        else:
            sudo_prefix = 'sudo'
        if self.command_format == 'docker' and cmd_str:
            cmd_str = self.extra_env_from_run + cmd_str
            cmd_str = self.scape_bash_command(cmd_str)
            cmd_str = '\\\\n'.join(cmd_str.strip('\n').split('\n'))
            cmd_str = 'RUN echo """%s"""' % (cmd_str,) \
                + ' | ' + sudo_prefix + 'tee -a /entrypoint.sh \\' + \
                '\n    && ' + sudo_prefix + \
                ' chown %s:%s /entrypoint.sh \\' % (
                    self.docker_user, self.docker_user) + \
                '\n    && ' + sudo_prefix + ' chmod +x /entrypoint.sh'
            cmd_str += '\nENTRYPOINT /entrypoint.sh'
        return cmd_str

    def get_travis2docker_env(self, section_data):
        global_data_cmd = []
        matrix_data_cmd = []
        if 'global' in section_data:
            global_data = section_data.pop('global')
            global_data_cmd = [
                global_env
                for global_env
                in self.get_travis2docker_env(global_data)
            ]
        if 'matrix' in section_data:
            matrix_data = section_data.pop('matrix')
            matrix_data_cmd = [
                matrix_env
                for matrix_env
                in self.get_travis2docker_env(matrix_data)
            ]
        for combination in itertools.product(global_data_cmd, matrix_data_cmd):
            command_str = '\n'.join(combination) + "\n"
            yield command_str

        for line in section_data:
            if line in ['global', 'matrix']:
                continue
            if not isinstance(line, str):
                continue
            docker_env = ""
            for var, value in self.env_regex.findall(line):
                if self.command_format == 'bash':
                    docker_env += "\nexport %s=%s" % (var, value)
                elif self.command_format == 'docker':
                    docker_env += "\nENV %s=%s" % (var, value)
            yield docker_env

    def get_travis2docker_python(self, section_data):
        for line in section_data:
            yield "\n# TODO: Use python version: " + line

    def get_default_cmd(self, dockerfile_path):
        home_user_path = self.docker_user == 'root' and "/root" \
            or os.path.join("/home", self.docker_user)
        project, branch = self.git_project, self.revision
        travis_build_dir = os.path.join(
            home_user_path, "build",
            self.git_obj.owner, self.git_obj.repo,
        )
        if self.command_format == 'bash':
            cmd = "\nsudo su - " + self.docker_user + \
                  "\nsudo chown -R %s:%s %s" % (self.docker_user, self.docker_user, home_user_path) + \
                  "\nexport TRAVIS_BUILD_DIR=%s" % (travis_build_dir) + \
                  "\ngit clone --single-branch %s -b %s " % (project, branch) + \
                  "${TRAVIS_BUILD_DIR}" + \
                  "\n"
        elif self.command_format == 'docker':
            dkr_files_path = os.path.join(dockerfile_path, "files")
            if not os.path.exists(dkr_files_path):
                os.makedirs(dkr_files_path)
            if not os.path.exists(os.path.join(dkr_files_path, 'ssh')):
                shutil.copytree(
                    os.path.expanduser("~/.ssh"),
                    os.path.join(dkr_files_path, 'ssh')
                )
            cmd_refs = 'pull' in self.revision and \
                '+refs/%s/head:refs/%s' % (
                    self.revision, self.revision) or \
                '+refs/heads/%s:refs/heads/%s' % (
                    self.revision, self.revision)

            # TODO: Use sha
            cmd_git_clone = [
                "git init ${TRAVIS_BUILD_DIR}",
                "cd ${TRAVIS_BUILD_DIR}",
                "git remote add origin " + project,
                "git fetch --update-head-ok -p origin %s" % (cmd_refs),
                "git reset --hard " + self.revision,

            ]
            git_user_email = self.git_obj.get_config_data("user.email")
            if git_user_email:
                cmd_git_clone.append(
                    "git config --global user.email \"%s\"" % (git_user_email)
                )
            else:
                cmd_git_clone.append(
                    "git config --unset --global user.email"
                )
            git_user_name = self.git_obj.get_config_data("user.name")
            if git_user_name:
                cmd_git_clone.append(
                    "git config --global user.name \"%s\"" % (git_user_name)
                )
            else:
                cmd_git_clone.append(
                    "git config --unset --global user.name"
                )
            if self.docker_user == 'root':
                sudo_prefix = ''
            else:
                sudo_prefix = 'sudo'
            cmd = 'FROM ' + (self.travis_data.get('build_image', False) or
                             self.default_docker_image) + \
                  '\nUSER ' + self.docker_user + \
                  '\nENV HOME=' + home_user_path + \
                  '\nADD ' + os.path.join("files", 'ssh') + ' ' + \
                  os.path.join(home_user_path, '.ssh') + \
                  "\nENV TRAVIS_BUILD_DIR=%s" % (travis_build_dir) + \
                  "\nWORKDIR ${TRAVIS_BUILD_DIR}" + \
                  "\nRUN %s chown -R %s:%s %s" % (
                      sudo_prefix, self.docker_user,
                      self.docker_user, home_user_path) + \
                  "\nRUN " + ' \\\n    && '.join(cmd_git_clone) + \
                  "\n"
        return cmd

    def get_travis2docker_iter(self):
        travis2docker_cmd_static_str = ""
        travis2docker_cmd_iter_list = []
        for travis_section, dummy in self.travis2docker_section:
            travis2docker_section = self.get_travis_section(travis_section)
            if isinstance(travis2docker_section, types.GeneratorType):
                travis2docker_cmd_iter_list.append([
                    item_iter for item_iter in travis2docker_section
                ])
            elif isinstance(travis2docker_section, basestring):
                travis2docker_cmd_static_str += travis2docker_section + "\n"
        for combination in itertools.product(*travis2docker_cmd_iter_list):
            command_str = '\n'.join(combination) + "\n" + \
                travis2docker_cmd_static_str
            yield command_str

    def get_travis2docker(self):
        count = 1
        fname_scripts = []
        for cmd in self.get_travis2docker_iter():
            fname = os.path.join(
                self.scripts_root_path,
                str(count)
            )
            fname_build = None
            fname_run = None
            if not os.path.exists(fname):
                os.makedirs(fname)
            if self.command_format == 'bash':
                fname = os.path.join(fname, 'script.sh')
                cmd = self.get_default_cmd(os.path.dirname(fname)) + cmd
            elif self.command_format == 'docker':
                fname = os.path.join(fname, 'Dockerfile')
                cmd = self.get_default_cmd(os.path.dirname(fname)) + cmd
                fname_build = os.path.join(
                    os.path.dirname(fname), '10-build.sh'
                )
                fname_run = os.path.join(
                    os.path.dirname(fname), '20-run.sh'
                )
                image_name = self.git_obj.owner + '-' + \
                    self.git_obj.repo + \
                    ":" + self.get_folder_name(self.revision) + \
                    "_" + str(count)
                image_name = image_name.lower()
            else:
                raise Exception(
                    "No command format found %s" % (self.command_format)
                )
            with open(fname, "w") as fdockerfile:
                fdockerfile.write(cmd)

            if fname_build:
                with open(fname_build, "w") as fbuild:
                    fbuild.write(
                        "docker build $1 -t %s %s" % (
                            image_name,
                            os.path.dirname(fname)
                        )
                    )
                st = os.stat(fname_build)
                os.chmod(fname_build, st.st_mode | stat.S_IEXEC)
            if fname_run:
                with open(fname_run, "w") as fbuild:
                    fbuild.write(
                        "docker run $1 -itP %s $2" % (
                            image_name
                        )
                    )
                st = os.stat(fname_run)
                os.chmod(fname_run, st.st_mode | stat.S_IEXEC)
            if self.command_format == 'bash':
                st = os.stat(fname)
                os.chmod(fname, st.st_mode | stat.S_IEXEC)
            count += 1
            fname_scripts.append(os.path.dirname(fname))
        return fname_scripts
class TravisWeblateUpdate(object):

    GIT_COMMIT_INFO = {
        'author':
        'Weblate bot <weblate@bot>',
        'message':
        '[REF] i18n: Updating translation terms from weblate '
        '[ci skip]'
    }

    def __init__(self):
        self._git = GitRun(os.path.join(os.getcwd(), '.git'), True)
        self.branch = os.environ.get("TRAVIS_BRANCH",
                                     self._git.get_branch_name())
        remote = self._git.run(["ls-remote", "--get-url", "origin"])
        name = remote.replace(':', '/')
        name = re.sub('.+@', '', name)
        name = re.sub('.git$', '', name)
        name = re.sub('^https://', '', name)
        name = re.sub('^http://', '', name)
        match = re.search(r'(?P<host>[^/]+)/(?P<owner>[^/]+)/(?P<repo>[^/]+)',
                          name)
        if match:
            name = ("%(host)s:%(owner)s/%(repo)s (%(branch)s)" %
                    dict(match.groupdict(), branch=self.branch))
        self.repo_name = name
        self.wl_api = WeblateApi()
        self.gh_api = GitHubApi()
        self._travis_home = os.environ.get("HOME", "~/")
        self._travis_build_dir = os.environ.get("TRAVIS_BUILD_DIR", "../..")
        self._odoo_version = os.environ.get("VERSION")
        self._odoo_branch = os.environ.get("ODOO_BRANCH")
        self._langs = (parse_list(os.environ.get("LANG_ALLOWED"))
                       if os.environ.get("LANG_ALLOWED", False) else [])
        self._odoo_full = os.environ.get("ODOO_REPO", "odoo/odoo")
        self._server_path = get_server_path(
            self._odoo_full, (self._odoo_branch or self._odoo_version),
            self._travis_home)
        self._addons_path = get_addons_path(self._travis_home,
                                            self._travis_build_dir,
                                            self._server_path)
        self._database = os.environ.get('MQT_TEST_DB', 'openerp_test')
        self._connection_context = context_mapping.get(self._odoo_version,
                                                       Odoo10Context)
        self._apply_patch_odoo()
        self._get_modules_installed()

    def _check(self):
        self.wl_api._check()
        self.gh_api._check()

    def _apply_patch_odoo(self):
        """This patch is necessary because the weblate does not check which
        word and the translated are the same to use it in its percentage of
        translated"""
        for base in ('odoo', 'openerp'):
            p_file = os.path.join(self._server_path, base,
                                  os.path.join('tools', 'translate.py'))
            if os.path.isfile(p_file):
                sed = [
                    "sed", "-i", "-e",
                    r"s/translation'] = src/translation'] = ''/g", p_file
                ]
                print " ".join(sed)
                subprocess.call(sed)
            p_file = os.path.join(
                self._server_path, base,
                os.path.join('addons', 'base', 'ir', 'ir_translation.py'))
            if os.path.isfile(p_file):
                sed = [
                    "sed", "-i", "-e",
                    r"s/if\snot\strans_dict\['value']:/if False:/g", p_file
                ]
                print " ".join(sed)
                subprocess.call(sed)

    def _get_modules_installed(self):
        self._installed_modules = []
        modules_found = []
        for name in ('__openerp__.py', '__manifest__.py'):
            modules = glob.glob('%s/**/%s' % (self._travis_build_dir, name))
            if not modules:
                continue
            modules_found.extend(
                [os.path.dirname(module).split('/')[-1] for module in modules])
        with self._connection_context(self._server_path, self._addons_path,
                                      self._database) as odoo_context:
            odoo_context.cr.execute(
                "select name from ir_module_module"
                " where state = 'installed' and "
                "name in %s", (tuple(modules_found), ))
            modules = odoo_context.cr.dictfetchall()
            self._installed_modules = [module['name'] for module in modules]

    def _generate_odoo_po_files(self, component):
        generated = False
        with self._connection_context(self._server_path, self._addons_path,
                                      self._database) as odoo_context:
            module = component['name']
            if module not in self._installed_modules:
                return generated
            print("\n", yellow("Obtaining POT file for %s" % module))
            i18n_folder = os.path.join(self._travis_build_dir, module, 'i18n')
            if not os.path.isdir(i18n_folder):
                os.makedirs(i18n_folder)
            # Put git add for letting known git which translations to update
            po_files = glob.glob(os.path.join(i18n_folder, '*.po'))
            for lang in self._langs:
                if os.path.isfile(os.path.join(i18n_folder, lang + '.po')):
                    continue
                po_content = odoo_context.get_pot_contents(module, lang)
                if not po_content:
                    continue
                with open(os.path.join(i18n_folder, lang + '.po'), 'wb')\
                        as f_po:
                    f_po.write(po_content)
            for po_file_name in po_files:
                lang = os.path.basename(os.path.splitext(po_file_name)[0])
                if self._langs and lang not in self._langs:
                    # Limit just allowed languages if is defined
                    continue
                po_file_path = os.path.join(i18n_folder, po_file_name)
                with open(po_file_path, 'r') as f_po:
                    odoo_context.load_po(f_po, lang)
                new_content = odoo_context.get_pot_contents(module, lang)
                if not new_content:
                    continue
                with open(po_file_path, 'wb') as f_po:
                    f_po.write(new_content)
                diff = self._git.run(["diff", "HEAD", po_file_path])
                if diff.count('msgstr') == 1:
                    self._git.run(["checkout", po_file_path])
            if self._git.run(["add", "-v"] + po_files):
                generated = True
        return generated

    def _check_conflict(self, component):
        status = self._git.run(["status"])
        conflicts = [
            item for item in status.split('\n')
            if (item.startswith('\tboth modified')
                and component['filemask'].replace('/*.po', '') in item)
        ]
        if conflicts:
            self._register_pull_request(component, status)
            return True
        return False

    def _register_pull_request(self, component, status):
        branch_name = 'conflict-%s-weblate' % self.branch
        self._git.run(["add", component['filemask']])
        self._git.run([
            "commit", "--no-verify", "--author='Weblate bot <weblate@bot>'",
            "-m", "[REF] i18n: Conflict on the daily cron", "-m", status
        ])
        self._git.run(["branch", "-m", branch_name])
        self._git.run(["push", "-f", "origin", branch_name])
        pull = self.gh_api.create_pull_request({
            'title':
            '[REF] i18n: Conflict on the daily cron',
            'head':
            '%s:%s' %
            (self.repo_name.split('/')[0].split(':')[1], branch_name),
            'base':
            self.branch,
            'body':
            status
        })
        self._git.run(
            ["checkout", "-qb", self.branch,
             "origin/%s" % self.branch])
        self._git.run(["branch", "-D", branch_name])
        print yellow("The pull request register is: %s" % pull['html_url'])

    def _commit_weblate(self, first_commit=False):
        if ('nothing to commit, working tree clean'
                in self._git.run(["status"])):
            return first_commit
        if first_commit:
            self._git.run(["commit", "--no-verify", "--amend", "--no-edit"])
        else:
            self._git.run([
                "commit", "--no-verify",
                "--author='%s'" % self.GIT_COMMIT_INFO['author'], "-m",
                self.GIT_COMMIT_INFO['message']
            ])
            first_commit = True
        return first_commit

    def _push_git_repository(self):
        po_files = self._git.run(
            ["show", "--format=format:'%H'", "--name-only"]).split('\n')
        if not len(po_files) > 1:
            return False
        commit = self.gh_api.create_commit(self.GIT_COMMIT_INFO['message'],
                                           self.branch, po_files[1:])
        if commit:
            for component in self.wl_api.components:
                self.wl_api.component_repository(component, 'reset')
                self.wl_api.component_repository(component, 'pull')
        return commit

    def update(self):
        self._check()
        self.wl_api.load_project(self.repo_name, self.branch)
        if not self.wl_api.components:
            print yellow("No component found for %s" % self.repo_name)
            return 1
        with self.wl_api.component_lock():
            self._git.run(["fetch", "origin"])
            first_commit = False
            for component in self.wl_api.components:
                print yellow("Component %s" % component['slug'])
                name = '%s-wl' % component['slug']
                remote = (self.wl_api.host.replace('api', 'git') + '/' +
                          self.wl_api.project['slug'] + '/' +
                          component['slug'])
                self._git.run([
                    "checkout", "-qb", self.branch,
                    "origin/%s" % self.branch
                ])
                self.wl_api.component_repository(component, 'pull')
                self._git.run(["remote", "add", name, remote])
                self._git.run(["fetch", name])
                if self._generate_odoo_po_files(component):
                    first_commit = self._commit_weblate(first_commit)
                self._git.run([
                    "merge", "--squash", "-s", "recursive", "-X", "ours",
                    "%s/%s" % (name, self.branch)
                ])
                self._git.run(["remote", "remove", name])
                if self._check_conflict(component):
                    break
                if (component['filemask'].replace('/*.po', '')
                        in self._git.run(["status"])):
                    self._git.run(["add", component['filemask']])
                    first_commit = self._commit_weblate(first_commit)
                if self._check_conflict(component):
                    break
                first_commit = self._commit_weblate(first_commit)
            if not self._push_git_repository():
                return 1
        return 0
class TravisWeblateUpdate(object):

    GIT_COMMIT_INFO = {
        'author': 'Weblate bot <weblate@bot>',
        'message': '[REF] i18n: Updating translation terms from weblate '
                   '[ci skip]'
    }

    def __init__(self):
        self._git = GitRun(os.path.join(os.getcwd(), '.git'), True)
        self.branch = os.environ.get("TRAVIS_BRANCH",
                                     self._git.get_branch_name())
        remote = self._git.run(["ls-remote", "--get-url", "origin"])
        name = remote.replace(':', '/')
        name = re.sub('.+@', '', name)
        name = re.sub('.git$', '', name)
        name = re.sub('^https://', '', name)
        name = re.sub('^http://', '', name)
        match = re.search(
            r'(?P<host>[^/]+)/(?P<owner>[^/]+)/(?P<repo>[^/]+)', name)
        if match:
            name = ("%(host)s:%(owner)s/%(repo)s (%(branch)s)" %
                    dict(match.groupdict(), branch=self.branch))
        self.repo_name = name
        self.wl_api = WeblateApi()
        self.gh_api = GitHubApi()
        self._travis_home = os.environ.get("HOME", "~/")
        self._travis_build_dir = os.environ.get("TRAVIS_BUILD_DIR", "../..")
        self._odoo_version = os.environ.get("VERSION")
        self._odoo_branch = os.environ.get("ODOO_BRANCH")
        self._langs = (parse_list(os.environ.get("LANG_ALLOWED")) if
                       os.environ.get("LANG_ALLOWED", False) else [])
        self._odoo_full = os.environ.get("ODOO_REPO", "odoo/odoo")
        self._server_path = get_server_path(self._odoo_full,
                                            (self._odoo_branch or
                                             self._odoo_version),
                                            self._travis_home)
        self._addons_path = get_addons_path(self._travis_home,
                                            self._travis_build_dir,
                                            self._server_path)
        self._database = os.environ.get('MQT_TEST_DB', 'openerp_test')
        self._connection_context = context_mapping.get(
            self._odoo_version, Odoo10Context)
        self._apply_patch_odoo()
        self._get_modules_installed()

    def _check(self):
        self.wl_api._check()
        self.gh_api._check()

    def _apply_patch_odoo(self):
        """This patch is necessary because the weblate does not check which
        word and the translated are the same to use it in its percentage of
        translated"""
        for base in ('odoo', 'openerp'):
            p_file = os.path.join(self._server_path, base,
                                  os.path.join('tools', 'translate.py'))
            if os.path.isfile(p_file):
                sed = ["sed", "-i", "-e",
                       r"s/translation'] = src/translation'] = ''/g", p_file]
                print " ".join(sed)
                subprocess.call(sed)
            p_file = os.path.join(self._server_path, base,
                                  os.path.join('addons', 'base', 'ir',
                                               'ir_translation.py'))
            if os.path.isfile(p_file):
                sed = ["sed", "-i", "-e",
                       r"s/if\snot\strans_dict\['value']:/if False:/g", p_file]
                print " ".join(sed)
                subprocess.call(sed)

    def _get_modules_installed(self):
        self._installed_modules = []
        modules_found = []
        for name in ('__openerp__.py', '__manifest__.py'):
            modules = glob.glob('%s/**/%s' % (self._travis_build_dir, name))
            if not modules:
                continue
            modules_found.extend([
                os.path.dirname(module).split('/')[-1] for module in
                modules])
        with self._connection_context(self._server_path, self._addons_path,
                                      self._database) as odoo_context:
            odoo_context.cr.execute("select name from ir_module_module"
                                    " where state = 'installed' and "
                                    "name in %s", (tuple(modules_found),))
            modules = odoo_context.cr.dictfetchall()
            self._installed_modules = [module['name'] for module in modules]

    def _generate_odoo_po_files(self, module, only_installed=True):
        generated = False
        with self._connection_context(self._server_path, self._addons_path,
                                      self._database) as odoo_context:
            if only_installed and module not in self._installed_modules:
                return generated
            print("\n", yellow("Obtaining POT file for %s" % module))
            i18n_folder = os.path.join(self._travis_build_dir, module, 'i18n')
            if not os.path.isdir(i18n_folder):
                os.makedirs(i18n_folder)
            # Put git add for letting known git which translations to update
            po_files = glob.glob(os.path.join(i18n_folder, '*.po'))
            for lang in self._langs:
                if os.path.isfile(os.path.join(i18n_folder, lang + '.po')):
                    continue
                po_content = odoo_context.get_pot_contents(module, lang)
                if not po_content:
                    continue
                with open(os.path.join(i18n_folder, lang + '.po'), 'wb')\
                        as f_po:
                    f_po.write(po_content)
                    if self._git.run(["add", "-v", f_po.name]):
                        generated = True
            for po_file_name in po_files:
                lang = os.path.basename(os.path.splitext(po_file_name)[0])
                if self._langs and lang not in self._langs:
                    # Limit just allowed languages if is defined
                    continue
                po_file_path = os.path.join(i18n_folder, po_file_name)
                with open(po_file_path, 'r') as f_po:
                    odoo_context.load_po(f_po, lang)
                new_content = odoo_context.get_pot_contents(module, lang)
                if not new_content:
                    continue
                with open(po_file_path, 'wb') as f_po:
                    f_po.write(new_content)
                diff = self._git.run(["diff", "HEAD", po_file_path])
                if diff.count('msgstr') == 1:
                    self._git.run(["checkout", po_file_path])
            if self._git.run(["add", "-v"] + po_files):
                generated = True
        return generated

    def _check_conflict(self, component):
        status = self._git.run(["status"])
        conflicts = [item for item in status.split('\n')
                     if (item.startswith('\tboth modified') and
                         component['filemask'].replace('/*.po', '') in item)]
        if conflicts:
            self._register_pull_request(component, status)
            return True
        return False

    def _register_pull_request(self, component, status):
        branch_name = 'conflict-%s-weblate' % self.branch
        self._git.run(["add", component['filemask']])
        self._git.run(["commit", "--no-verify",
                       "--author='Weblate bot <weblate@bot>'",
                       "-m", "[REF] i18n: Conflict on the daily cron",
                       "-m", status])
        self._git.run(["branch", "-m", branch_name])
        self._git.run(["push", "-f", "origin", branch_name])
        pull = self.gh_api.create_pull_request({
            'title': '[REF] i18n: Conflict on the daily cron',
            'head': '%s:%s' % (self.repo_name.split('/')[0].split(':')[1],
                               branch_name),
            'base': self.branch,
            'body': status
        })
        self._git.run(["checkout", "-qb", self.branch,
                       "origin/%s" % self.branch])
        self._git.run(["branch", "-D", branch_name])
        print yellow("The pull request register is: %s" % pull['html_url'])

    def _commit_weblate(self, first_commit=False):
        if ('nothing to commit, working tree clean'
                in self._git.run(["status"])):
            return first_commit
        if first_commit:
            self._git.run(["commit", "--no-verify", "--amend",
                           "--no-edit"])
        else:
            self._git.run(["commit", "--no-verify",
                           "--author='%s'" % self.GIT_COMMIT_INFO['author'],
                           "-m", self.GIT_COMMIT_INFO['message']])
            first_commit = True
        return first_commit

    def _push_git_repository(self):
        po_files = self._git.run(["show", "--format=format:'%H'",
                                  "--name-only"]).split('\n')
        if not len(po_files) > 1:
            return False
        commit = self.gh_api.create_commit(self.GIT_COMMIT_INFO['message'],
                                           self.branch,
                                           po_files[1:])
        if commit:
            for component in self.wl_api.components:
                self.wl_api.component_repository(component, 'reset')
                self.wl_api.component_repository(component, 'pull')
        return commit

    def update(self):
        self._check()
        self.wl_api.load_project(self.repo_name, self.branch)
        with self.wl_api.component_lock():
            self._git.run(["fetch", "origin"])
            first_commit = False
            component = [item for item in self.wl_api.components
                         if item['git_export']]
            if len(component) > 1:
                print yellow("To many repository for this project %s" %
                             self.wl_api.project['name'])
                return 1
            remote = (self.wl_api.ssh + '/' + self.wl_api.project['slug'] +
                      '/' + component[0]['slug'])
            name = '%s-wl' % self.wl_api.project['slug']
            self._git.run(["remote", "add", name, remote])
            for component in self.wl_api.components:
                print yellow("Component %s" % component['slug'])
                self._git.run(["checkout", "-qb", self.branch,
                               "origin/%s" % self.branch])
                self.wl_api.component_repository(component, 'pull')
                self._git.run(["fetch", name])
                if self._generate_odoo_po_files(component['name']):
                    first_commit = self._commit_weblate(first_commit)
                self._git.run(["merge", "--squash", "-s", "recursive", "-X",
                               "ours", "%s/%s" % (name, self.branch)])
                if self._check_conflict(component):
                    break
                if (component['filemask'].replace('/*.po', '') in
                        self._git.run(["status"])):
                    self._git.run(["add", component['filemask']])
                    first_commit = self._commit_weblate(first_commit)
                if self._check_conflict(component):
                    break
                first_commit = self._commit_weblate(first_commit)
            self._git.run(["remote", "remove", name])
            modules_no_processed = [module for module in
                                    self._installed_modules if module not in
                                    [comp['name'] for comp in
                                     self.wl_api.components]]
            for component in modules_no_processed:
                if self._generate_odoo_po_files(component,
                                                only_installed=False):
                    first_commit = self._commit_weblate(first_commit)
            if not self._push_git_repository():
                return 1
        return 0
def translate_gitlab_env():
    """ This sets Travis CI's variables using GitLab CI's variables. This
        makes possible to run scripts which depend on Travis CI's variables
        when running under Gitlab CI.

         For documentation on these environment variables, please check:
        - Predefined environment variables on Travis CI:
          https://docs.travis-ci.com/user/environment-variables#Default-Environment-Variables
        - Predefined environment variables on GitLab CI:
          https://docs.gitlab.com/ee/ci/variables/#predefined-variables-environment-variables
    """
    if not os.environ.get("GITLAB_CI"):
        return False
    build_dir = os.environ.get("CI_PROJECT_DIR", ".")
    git_run_obj = GitRun(repo_path=os.path.join(build_dir, ".git"))
    commit_message = git_run_obj.run(["log", "-1", "--pretty=%b"])
    head_branch = os.environ.get("CI_COMMIT_REF_NAME")
    # This is a guess
    target_branch = os.environ.get("VERSION")

    # Set environment variables
    os.environ.update({
        # Predefined values
        "TRAVIS": "true",
        "CONTINUOUS_INTEGRATION": "true",
        "HAS_JOSH_K_SEAL_OF_APPROVAL": "true",
        "RAILS_ENV": "test",
        "RACK_ENV": "test",
        "MERB_ENV": "test",
        "TRAVIS_ALLOW_FAILURE": "false",
        "TRAVIS_JOB_NUMBER": "1",
        "TRAVIS_TEST_RESULT": "0",
        "TRAVIS_COMMIT_RANGE": "unknown",
        # Dinamic values
        "TRAVIS_BRANCH": target_branch,
        "TRAVIS_COMMIT_MESSAGE": commit_message,
        "TRAVIS_OS_NAME": sys.platform,
        "TRAVIS_SUDO": "true" if getuser() == "root" else "false",
    })

# Set variables whose values are already directly set in other variables
    equivalent_vars = [
        ("TRAVIS_BUILD_DIR", "CI_PROJECT_DIR"),
        ("TRAVIS_BUILD_ID", "CI_BUILD_ID"),
        ("TRAVIS_BUILD_NUMBER", "CI_JOB_ID"),
        ("TRAVIS_COMMIT", "CI_COMMIT_SHA"),
        ("TRAVIS_EVENT_TYPE", "CI_PIPELINE_SOURCE"),
        ("TRAVIS_JOB_ID", "CI_JOB_ID"),
        ("TRAVIS_REPO_SLUG", "CI_PROJECT_PATH"),
        ("TRAVIS_BUILD_STAGE_NAME", "CI_BUILD_STAGE"),
    ]
    os.environ.update({
        travis_var: os.environ.get(gitlab_var)
        for travis_var, gitlab_var in equivalent_vars
        if gitlab_var in os.environ
    })

    # If within an MR
    is_mr = target_branch != head_branch
    if is_mr:
        os.environ.update({
            # there's no way to know the MR number. For more info, see:
            # https://gitlab.com/gitlab-org/gitlab-ce/issues/15280
            "TRAVIS_PULL_REQUEST": "unknown",
            "TRAVIS_PULL_REQUEST_BRANCH": head_branch,
            "TRAVIS_PULL_REQUEST_SHA": os.environ.get("CI_COMMIT_SHA"),
            "TRAVIS_PULL_REQUEST_SLUG": os.environ.get("CI_PROJECT_PATH"),
        })
    else:
        os.environ.update({
            "TRAVIS_PULL_REQUEST": "false",
            "TRAVIS_PULL_REQUEST_BRANCH": "",
            "TRAVIS_PULL_REQUEST_SHA": "",
            "TRAVIS_PULL_REQUEST_SLUG": "",
        })
    return True
def translate_gitlab_env():
    """ This sets Travis CI's variables using GitLab CI's variables. This
        makes possible to run scripts which depend on Travis CI's variables
        when running under Gitlab CI.

         For documentation on these environment variables, please check:
        - Predefined environment variables on Travis CI:
          https://docs.travis-ci.com/user/environment-variables#Default-Environment-Variables
        - Predefined environment variables on GitLab CI:
          https://docs.gitlab.com/ee/ci/variables/#predefined-variables-environment-variables
    """
    if not os.environ.get("GITLAB_CI"):
        return False
    build_dir = os.environ.get("CI_PROJECT_DIR", ".")
    git_run_obj = GitRun(repo_path=os.path.join(build_dir, ".git"))
    commit_message = git_run_obj.run(["log", "-1", "--pretty=%b"])
    head_branch = os.environ.get("CI_COMMIT_REF_NAME")
    # This is a guess
    target_branch = os.environ.get("VERSION")

    # Set environment variables
    os.environ.update({
        # Predefined values
        "TRAVIS":
        "true",
        "CONTINUOUS_INTEGRATION":
        "true",
        "HAS_JOSH_K_SEAL_OF_APPROVAL":
        "true",
        "RAILS_ENV":
        "test",
        "RACK_ENV":
        "test",
        "MERB_ENV":
        "test",
        "TRAVIS_ALLOW_FAILURE":
        "false",
        "TRAVIS_JOB_NUMBER":
        "1",
        "TRAVIS_TEST_RESULT":
        "0",
        "TRAVIS_COMMIT_RANGE":
        "unknown",
        # Dinamic values
        "TRAVIS_BRANCH":
        target_branch,
        "TRAVIS_COMMIT_MESSAGE":
        commit_message,
        "TRAVIS_OS_NAME":
        sys.platform,
        "TRAVIS_SUDO":
        "true" if getuser() == "root" else "false",
    })

    # Set variables whose values are already directly set in other variables
    equivalent_vars = [
        ("TRAVIS_BUILD_DIR", "CI_PROJECT_DIR"),
        ("TRAVIS_BUILD_ID", "CI_BUILD_ID"),
        ("TRAVIS_BUILD_NUMBER", "CI_JOB_ID"),
        ("TRAVIS_COMMIT", "CI_COMMIT_SHA"),
        ("TRAVIS_EVENT_TYPE", "CI_PIPELINE_SOURCE"),
        ("TRAVIS_JOB_ID", "CI_JOB_ID"),
        ("TRAVIS_REPO_SLUG", "CI_PROJECT_PATH"),
        ("TRAVIS_BUILD_STAGE_NAME", "CI_BUILD_STAGE"),
    ]
    os.environ.update({
        travis_var: os.environ.get(gitlab_var)
        for travis_var, gitlab_var in equivalent_vars
        if gitlab_var in os.environ
    })

    # If within an MR
    is_mr = target_branch != head_branch
    if is_mr:
        os.environ.update({
            # there's no way to know the MR number. For more info, see:
            # https://gitlab.com/gitlab-org/gitlab-ce/issues/15280
            "TRAVIS_PULL_REQUEST":
            "unknown",
            "TRAVIS_PULL_REQUEST_BRANCH":
            head_branch,
            "TRAVIS_PULL_REQUEST_SHA":
            os.environ.get("CI_COMMIT_SHA"),
            "TRAVIS_PULL_REQUEST_SLUG":
            os.environ.get("CI_PROJECT_PATH"),
        })
    else:
        os.environ.update({
            "TRAVIS_PULL_REQUEST": "false",
            "TRAVIS_PULL_REQUEST_BRANCH": "",
            "TRAVIS_PULL_REQUEST_SHA": "",
            "TRAVIS_PULL_REQUEST_SLUG": "",
        })
    return True