def get_modules_changed(path, ref='HEAD'):
'''Get modules changed from git diff-index {ref}
:param path: String path of git repo
:param ref: branch or remote/branch or sha to compare
:return: List of paths of modules changed
'''
git_run_obj = GitRun(os.path.join(path, '.git'))
if ref != 'HEAD':
fetch_ref = ref
if ':' not in fetch_ref:
# to force create branch
fetch_ref += ':' + fetch_ref
git_run_obj.run(['fetch'] + fetch_ref.split('/', 1))
items_changed = git_run_obj.get_items_changed(ref)
folders_changed = set([
item_changed.split('/')[0]
for item_changed in items_changed
if '/' in item_changed]
)
modules = set(get_modules(path))
modules_changed = list(modules & folders_changed)
modules_changed_path = [
os.path.join(path, module_changed)
for module_changed in modules_changed]
return modules_changed_path
def get_modules_changed(path, ref='HEAD'):
'''Get modules changed from git diff-index {ref}
:param path: String path of git repo
:param ref: branch or remote/branch or sha to compare
:return: List of paths of modules changed
'''
git_run_obj = GitRun(os.path.join(path, '.git'))
if ref != 'HEAD':
fetch_ref = ref
if ':' not in fetch_ref:
# to force create branch
fetch_ref += ':' + fetch_ref
git_run_obj.run(['fetch'] + fetch_ref.split('/', 1))
items_changed = git_run_obj.get_items_changed(ref)
folders_changed = set([
item_changed.split('/')[0]
for item_changed in items_changed
if '/' in item_changed]
)
modules = set(get_modules(path))
modules_changed = list(modules & folders_changed)
modules_changed_path = [
os.path.join(path, module_changed)
for module_changed in modules_changed]
return modules_changed_path
def version_validate(version, dir):
if not version and dir:
repo_path = os.path.join(dir, '.git')
branch_name = GitRun(repo_path).get_branch_name()
version = (branch_name.replace('_', '-').split('-')[:1]
if branch_name else False)
version = version[0] if version else None
if not version:
print(
travis_helpers.yellow('Undefined environment variable'
' `VERSION`.\nSet `VERSION` for '
'compatibility with guidelines by version.'))
return version
def version_validate(version, dir):
if not version and dir:
repo_path = os.path.join(dir, '.git')
branch_name = GitRun(repo_path).get_branch_name()
version = (branch_name.replace('_', '-').split('-')[:1]
if branch_name else False)
version = version[0] if version else None
if not version:
print(travis_helpers.yellow(
'Undefined environment variable'
' `VERSION`.\nSet `VERSION` for '
'compatibility with guidelines by version.'))
return version
def __init__(self, git_project, revision,
command_format='docker', docker_user=None,
root_path=None, default_docker_image=None,
remotes=None, exclude_after_success=None,
run_extra_args=None, include_cleanup=None,
build_extra_args=''):
"""
Method Constructor
@fname_dockerfile: str name of file dockerfile to save.
@format_cmd: str name of format of command.
bash: Make a bash script.
docker: Make a dockerfile script.
"""
if root_path is None:
self.root_path = os.path.join(
gettempdir(),
os.path.splitext(os.path.basename(__file__))[0],
)
else:
self.root_path = os.path.expandvars(
os.path.expanduser(root_path)
)
self.default_docker_image = default_docker_image
self.git_project = git_project
self.revision = revision
git_path = self.get_repo_path(self.root_path)
self.git_obj = GitRun(git_project, git_path)
self.travis_data = self.load_travis_file(revision)
self.sha = self.git_obj.get_sha(revision)
self.exclude_after_success = exclude_after_success
self.run_extra_args = run_extra_args
self.include_cleanup = include_cleanup
self.build_extra_args = build_extra_args
if not self.travis_data:
raise Exception(
"Make sure you have access to repository"
" " + git_project + " and that the file .travis.yml"
" exists in branch or revision " + revision
)
self.travis2docker_section = [
# ('build_image', 'build_image'),
('python', 'python'),
('env', 'env'),
('install', 'run'),
('script', 'script'),
('after_success', 'script'),
]
self.travis2docker_section_dict = dict(self.travis2docker_section)
self.scripts_root_path = self.get_script_path(self.root_path)
self.env_regex_str2 = r"\s(?=\w+=)"
env_regex_str = r"(?P<var>[\w]*)[ ]*[\=][ ]*[\"\']{0,1}" + \
r"(?P<value>[\w\.\-\_/\$\{\}\:,\(\)\#\* ]*)[\"\']{0,1}"
export_regex_str = r"^(?P<export>export|EXPORT)( )+" + env_regex_str
self.env_regex = re.compile(env_regex_str, re.M)
self.export_regex = re.compile(export_regex_str, re.M)
self.extra_env_from_run = ""
self.command_format = command_format
self.docker_user = docker_user or 'root'
self.remotes = remotes or []
class GitRmAllLog(object):
def __init__(self, git_url,
root_path=None):
self.git_url = git_url
if root_path is None:
root_path = gettempdir()
else:
root_path = os.path.expandvars(
os.path.expanduser(root_path)
)
self.git_run_obj = GitRun(git_url, None)
self.git_run_obj.path = os.path.join(
root_path,
self.git_run_obj.owner,
self.git_run_obj.repo
)
def rm_all_log(self, items):
self.git_run_obj.update()
refs = self.git_run_obj.get_ref_data(['refs/heads']).keys()
for ref in refs:
self.git_run_obj.checkout_bare(ref)
self.git_run_obj.run([
"filter-branch", "-f", "--tree-filter",
"rm -rf %s" % items, "HEAD"
])
def __init__(self):
self._git = GitRun(os.path.join(os.getcwd(), '.git'), True)
self.branch = os.environ.get("TRAVIS_BRANCH",
self._git.get_branch_name())
remote = self._git.run(["ls-remote", "--get-url", "origin"])
name = remote.replace(':', '/')
name = re.sub('.+@', '', name)
name = re.sub('.git$', '', name)
name = re.sub('^https://', '', name)
name = re.sub('^http://', '', name)
match = re.search(
r'(?P<host>[^/]+)/(?P<owner>[^/]+)/(?P<repo>[^/]+)', name)
if match:
name = ("%(host)s:%(owner)s/%(repo)s (%(branch)s)" %
dict(match.groupdict(), branch=self.branch))
self.repo_name = name
self.wl_api = WeblateApi()
self.gh_api = GitHubApi()
self._travis_home = os.environ.get("HOME", "~/")
self._travis_build_dir = os.environ.get("TRAVIS_BUILD_DIR", "../..")
self._odoo_version = os.environ.get("VERSION")
self._odoo_branch = os.environ.get("ODOO_BRANCH")
self._langs = (parse_list(os.environ.get("LANG_ALLOWED")) if
os.environ.get("LANG_ALLOWED", False) else [])
self._odoo_full = os.environ.get("ODOO_REPO", "odoo/odoo")
self._server_path = get_server_path(self._odoo_full,
(self._odoo_branch or
self._odoo_version),
self._travis_home)
self._addons_path = get_addons_path(self._travis_home,
self._travis_build_dir,
self._server_path)
self._database = os.environ.get('MQT_TEST_DB', 'openerp_test')
self._connection_context = context_mapping.get(
self._odoo_version, Odoo10Context)
self._apply_patch_odoo()
self._get_modules_installed()
def __init__(self, git_url,
root_path=None):
self.git_url = git_url
if root_path is None:
root_path = gettempdir()
else:
root_path = os.path.expandvars(
os.path.expanduser(root_path)
)
self.git_run_obj = GitRun(git_url, None)
self.git_run_obj.path = os.path.join(
root_path,
self.git_run_obj.owner,
self.git_run_obj.repo
)
def __init__(self):
self._git = GitRun(os.path.join(os.getcwd(), '.git'), True)
self.branch = os.environ.get("TRAVIS_BRANCH",
self._git.get_branch_name())
remote = self._git.run(["ls-remote", "--get-url", "origin"])
name = remote.replace(':', '/')
name = re.sub('.+@', '', name)
name = re.sub('.git$', '', name)
name = re.sub('^https://', '', name)
name = re.sub('^http://', '', name)
match = re.search(
r'(?P<host>[^/]+)/(?P<owner>[^/]+)/(?P<repo>[^/]+)', name)
if match:
name = ("%(host)s:%(owner)s/%(repo)s (%(branch)s)" %
dict(match.groupdict(), branch=self.branch))
self.repo_name = name
self.wl_api = WeblateApi()
self.gh_api = GitHubApi()
self._travis_home = os.environ.get("HOME", "~/")
self._travis_build_dir = os.environ.get("TRAVIS_BUILD_DIR", "../..")
self._odoo_version = os.environ.get("VERSION")
self._odoo_branch = os.environ.get("ODOO_BRANCH")
self._langs = (parse_list(os.environ.get("LANG_ALLOWED")) if
os.environ.get("LANG_ALLOWED", False) else [])
self._odoo_full = os.environ.get("ODOO_REPO", "odoo/odoo")
self._server_path = get_server_path(self._odoo_full,
(self._odoo_branch or
self._odoo_version),
self._travis_home)
self._addons_path = get_addons_path(self._travis_home,
self._travis_build_dir,
self._server_path)
self._database = os.environ.get('MQT_TEST_DB', 'openerp_test')
self._connection_context = context_mapping.get(
self._odoo_version, Odoo10Context)
self._apply_patch_odoo()
self._get_modules_installed()
def __init__(self,
git_project,
revision,
command_format='docker',
docker_user=None,
root_path=None,
default_docker_image=None,
remotes=None,
exclude_after_success=None,
run_extra_args=None,
include_cleanup=None,
build_extra_args=''):
"""
Method Constructor
@fname_dockerfile: str name of file dockerfile to save.
@format_cmd: str name of format of command.
bash: Make a bash script.
docker: Make a dockerfile script.
"""
if root_path is None:
self.root_path = os.path.join(
gettempdir(),
os.path.splitext(os.path.basename(__file__))[0],
)
else:
self.root_path = os.path.expandvars(os.path.expanduser(root_path))
self.default_docker_image = default_docker_image
self.git_project = git_project
self.revision = revision
git_path = self.get_repo_path(self.root_path)
self.git_obj = GitRun(git_project, git_path)
self.travis_data = self.load_travis_file(revision)
self.sha = self.git_obj.get_sha(revision)
self.exclude_after_success = exclude_after_success
self.run_extra_args = run_extra_args
self.include_cleanup = include_cleanup
self.build_extra_args = build_extra_args
if not self.travis_data:
raise Exception("Make sure you have access to repository"
" " + git_project +
" and that the file .travis.yml"
" exists in branch or revision " + revision)
self.travis2docker_section = [
# ('build_image', 'build_image'),
('python', 'python'),
('env', 'env'),
('install', 'run'),
('script', 'script'),
('after_success', 'script'),
]
self.travis2docker_section_dict = dict(self.travis2docker_section)
self.scripts_root_path = self.get_script_path(self.root_path)
env_regex_str = r"(?P<var>[\w]*)[ ]*[\=][ ]*[\"\']{0,1}" + \
r"(?P<value>[\w\.\-\_/\$\{\}\:,\(\)\#\* ]*)[\"\']{0,1}"
export_regex_str = r"^(?P<export>export|EXPORT)( )+" + env_regex_str
self.env_regex = re.compile(env_regex_str, re.M)
self.export_regex = re.compile(export_regex_str, re.M)
self.extra_env_from_run = ""
self.command_format = command_format
self.docker_user = docker_user or 'root'
self.remotes = remotes
class travis(object):
def load_travis_file(self, branch):
self.git_obj.update()
yaml_loaded = None
for fname in ['.travis.yml', '.shippable.yml']:
yaml_str = self.git_obj.show_file(fname, branch)
try:
yaml_loaded = yaml.load(yaml_str)
break
except AttributeError:
pass
return yaml_loaded
def get_folder_name(self, name):
for invalid_char in '@:/#':
name = name.replace(invalid_char, '_')
return name
def get_repo_path(self, root_path):
name = self.get_folder_name(self.git_project.lower())
repo_path = os.path.join(root_path, 'repo', name)
return repo_path
def get_script_path(self, root_path):
name = self.get_folder_name(self.git_project.lower())
script_path = os.path.join(root_path, 'script', name,
self.get_folder_name(self.revision),
self.sha)
return script_path
def __init__(self,
git_project,
revision,
command_format='docker',
docker_user=None,
root_path=None,
default_docker_image=None,
remotes=None,
exclude_after_success=None,
run_extra_args=None,
include_cleanup=None,
build_extra_args=''):
"""
Method Constructor
@fname_dockerfile: str name of file dockerfile to save.
@format_cmd: str name of format of command.
bash: Make a bash script.
docker: Make a dockerfile script.
"""
if root_path is None:
self.root_path = os.path.join(
gettempdir(),
os.path.splitext(os.path.basename(__file__))[0],
)
else:
self.root_path = os.path.expandvars(os.path.expanduser(root_path))
self.default_docker_image = default_docker_image
self.git_project = git_project
self.revision = revision
git_path = self.get_repo_path(self.root_path)
self.git_obj = GitRun(git_project, git_path)
self.travis_data = self.load_travis_file(revision)
self.sha = self.git_obj.get_sha(revision)
self.exclude_after_success = exclude_after_success
self.run_extra_args = run_extra_args
self.include_cleanup = include_cleanup
self.build_extra_args = build_extra_args
if not self.travis_data:
raise Exception("Make sure you have access to repository"
" " + git_project +
" and that the file .travis.yml"
" exists in branch or revision " + revision)
self.travis2docker_section = [
# ('build_image', 'build_image'),
('python', 'python'),
('env', 'env'),
('install', 'run'),
('script', 'script'),
('after_success', 'script'),
]
self.travis2docker_section_dict = dict(self.travis2docker_section)
self.scripts_root_path = self.get_script_path(self.root_path)
env_regex_str = r"(?P<var>[\w]*)[ ]*[\=][ ]*[\"\']{0,1}" + \
r"(?P<value>[\w\.\-\_/\$\{\}\:,\(\)\#\* ]*)[\"\']{0,1}"
export_regex_str = r"^(?P<export>export|EXPORT)( )+" + env_regex_str
self.env_regex = re.compile(env_regex_str, re.M)
self.export_regex = re.compile(export_regex_str, re.M)
self.extra_env_from_run = ""
self.command_format = command_format
self.docker_user = docker_user or 'root'
self.remotes = remotes
# def get_travis_type(self, repo_git):
# self.repo_git_type = None
# if os.path.isdir(repo_git):
# self.repo_git_type = 'localpath'
# elif os.path.isfile(repo_git):
# self.repo_git_type = 'file'
# else:
# regex = r"(?P<host>(git@|https://)([\w\.@]+)(/|:))" \
# r"(?P<owner>[~\w,\-,\_]+)/" \
# r"(?P<repo>[\w,\-,\_]+)(.git){0,1}((/){0,1})"
# match_object = re.search(regex, repo_git)
# if match_object:
# self.repo_git_type = 'remote'
# def get_travis_data(self, repo_git, branch=None):
# self.get_travis_type(repo_git)
# if self.repo_git_type == 'remote':
# if branch is None:
# raise "You need specify branch name with remote repository"
# "git show 8.0:.travis.yml"
def get_travis_section(self, section, *args, **kwards):
section_type = self.travis2docker_section_dict.get(section, False)
if not section_type:
return None
section_data = self.travis_data.get(section, "")
if isinstance(section_data, basestring):
section_data = [section_data]
job_method = getattr(self, 'get_travis2docker_' + section_type)
return job_method(section_data, *args, **kwards)
def get_travis2docker_run(self,
section_data,
custom_command_format=None,
add_extra_env=True):
if custom_command_format is None:
custom_command_format = self.command_format
docker_run = ''
for line in section_data:
line = line.strip()
export_regex_findall = self.export_regex.findall(line)
for dummy, dummy, var, value in export_regex_findall:
self.extra_env_from_run += "\n%s=%s " % (var, value)
extra_env = add_extra_env and \
self.extra_env_from_run.replace('\n', ' ') or ''
if not export_regex_findall:
if custom_command_format == 'bash':
docker_run += '\n' + extra_env + line
elif custom_command_format == 'docker':
if line.startswith('(') and line.endswith(')'):
line = line[1:-1]
docker_run += '\nRUN ' + extra_env + line
return docker_run
def scape_bash_command(self, command):
return command.replace('$', '\\$').replace('_', '\\_')
def get_travis2docker_script(self, section_data):
cmd_str = self.get_travis2docker_run(section_data,
custom_command_format='bash',
add_extra_env=False)
if self.docker_user == 'root':
sudo_prefix = ''
else:
sudo_prefix = 'sudo'
if self.command_format == 'docker' and cmd_str:
cmd_str = "#!/bin/bash" + "\n" + self.extra_env_from_run.replace(
'\n', '\nexport ') + \
'\n' + cmd_str.replace('\n', '\\\\n').strip()
cmd_str = '\\\\n'.join(cmd_str.strip('\n').split('\n'))
cmd_str = cmd_str.replace('$', r'\$').replace('"', r'\"')
cmd_str = 'RUN echo """%s"""' % (cmd_str,) \
+ ' | ' + sudo_prefix + 'tee -a /entrypoint.sh \\' + \
'\n && ' + sudo_prefix + \
' chown %s:%s /entrypoint.sh \\' % (
self.docker_user, self.docker_user) + \
'\n && ' + sudo_prefix + ' chmod +x /entrypoint.sh'
cmd_str += '\nENTRYPOINT /entrypoint.sh'
return cmd_str
def get_travis2docker_env(self, section_data):
global_data_cmd = []
matrix_data_cmd = []
if 'global' in section_data:
# The section 'global' is used as a item
# Then the global_data_cmd even is a list of a item
global_data = section_data.pop('global')
global_data_cmd = [
''.join([
global_env
for global_env in self.get_travis2docker_env(global_data)
])
]
if 'matrix' in section_data:
matrix_data = section_data.pop('matrix')
matrix_data_cmd = [
matrix_env
for matrix_env in self.get_travis2docker_env(matrix_data)
]
for combination in itertools.product(global_data_cmd, matrix_data_cmd):
command_str = '\n'.join(combination) + "\n"
yield command_str
for line in section_data:
if line in ['global', 'matrix']:
continue
if not isinstance(line, str):
continue
docker_env = ""
for var, value in self.env_regex.findall(line):
if self.command_format == 'bash':
docker_env += "\nexport %s=%s" % (var, value)
elif self.command_format == 'docker':
if var:
# To support case `export VAR="param1=value1"`
docker_env += '\nENV '
docker_env += "%s=%s" % (var, value)
yield docker_env
def get_travis2docker_python(self, section_data):
for line in section_data:
yield "\n# TODO: Use python version: " + line
def get_default_cmd(self, dockerfile_path):
home_user_path = self.docker_user == 'root' and "/root" \
or os.path.join("/home", self.docker_user)
project, branch = self.git_project, self.revision
travis_build_dir = os.path.join(
home_user_path,
"build",
self.git_obj.owner,
self.git_obj.repo,
)
if self.command_format == 'bash':
cmd = "\nsudo su - " + self.docker_user + \
"\nsudo chown -R %s:%s %s" % (
self.docker_user, self.docker_user,
home_user_path) + \
"\nexport TRAVIS_BUILD_DIR=%s" % (travis_build_dir) + \
"\ngit clone --single-branch %s -b %s " % (
project, branch) + \
"${TRAVIS_BUILD_DIR}" + \
"\n"
elif self.command_format == 'docker':
dkr_files_path = os.path.join(dockerfile_path, "files")
if not os.path.exists(dkr_files_path):
os.makedirs(dkr_files_path)
ssh_dest = os.path.join(dkr_files_path, 'ssh')
if os.path.exists(ssh_dest):
shutil.rmtree(ssh_dest)
shutil.copytree(os.path.expanduser("~/.ssh"), ssh_dest)
cmd_refs = 'pull' in self.revision and \
'+refs/%s/head:refs/%s' % (
self.revision, self.revision) or \
'+refs/heads/%s:refs/heads/%s' % (
self.revision, self.revision)
# TODO: Use sha
remote_default = self.git_obj.owner.lower()
cmd_git_clone = [
"git init ${TRAVIS_BUILD_DIR}",
"cd ${TRAVIS_BUILD_DIR}",
"git remote add %s %s" % (remote_default, project),
"git fetch --update-head-ok -p %s %s" %
(remote_default, cmd_refs),
"git checkout -qf " + self.revision,
]
git_user_email = self.git_obj.get_config_data("user.email")
if git_user_email:
cmd_git_clone.append("git config --global user.email \"%s\"" %
(git_user_email))
else:
cmd_git_clone.append("git config --unset --global user.email")
git_user_name = self.git_obj.get_config_data("user.name")
if git_user_name:
cmd_git_clone.append("git config --global user.name \"%s\"" %
(git_user_name))
else:
cmd_git_clone.append("git config --unset --global user.name")
if self.remotes:
for remote in self.remotes:
remote = remote.lower()
if remote != remote_default:
git_remote_url = self.git_obj.host + remote + '/' + \
self.git_obj.repo + '.git'
cmd_git_clone.append('git remote add %s %s' %
(remote, git_remote_url))
if self.docker_user == 'root':
sudo_prefix = ''
else:
sudo_prefix = 'sudo'
cmd = 'FROM ' + (self.travis_data.get('build_image', False) or
self.default_docker_image) + \
'\nUSER ' + self.docker_user + \
'\nENV HOME=' + home_user_path + \
'\nENV TRAVIS_REPO_SLUG=%s/%s' % (
self.git_obj.owner, self.git_obj.repo) + \
'\nADD ' + os.path.join("files", 'ssh') + ' ' + \
os.path.join(home_user_path, '.ssh') + \
"\nENV TRAVIS_BUILD_DIR=%s" % (travis_build_dir) + \
"\nWORKDIR ${TRAVIS_BUILD_DIR}" + \
"\nRUN %s chown -R %s:%s %s" % (
sudo_prefix, self.docker_user,
self.docker_user, home_user_path) + \
"\nRUN " + ' \\\n && '.join(cmd_git_clone) + \
"\n"
return cmd
def get_travis2docker_iter(self):
travis2docker_cmd_static_str = ""
travis2docker_cmd_iter_list = []
if self.exclude_after_success:
self.travis2docker_section.remove(('after_success', 'script'))
for travis_section, dummy in self.travis2docker_section:
travis2docker_section = self.get_travis_section(travis_section)
if isinstance(travis2docker_section, types.GeneratorType):
travis2docker_cmd_iter_list.append(
[item_iter for item_iter in travis2docker_section])
elif isinstance(travis2docker_section, basestring):
travis2docker_cmd_static_str += travis2docker_section + "\n"
for combination in itertools.product(*travis2docker_cmd_iter_list):
command_str = '\n'.join(combination) + "\n" + \
travis2docker_cmd_static_str
yield command_str
def get_travis2docker(self):
count = 1
fname_scripts = []
for cmd in self.get_travis2docker_iter():
fname = os.path.join(self.scripts_root_path, str(count))
fname_build = None
fname_run = None
if not os.path.exists(fname):
os.makedirs(fname)
if self.command_format == 'bash':
fname = os.path.join(fname, 'script.sh')
cmd = self.get_default_cmd(os.path.dirname(fname)) + cmd
elif self.command_format == 'docker':
fname = os.path.join(fname, 'Dockerfile')
cmd = self.get_default_cmd(os.path.dirname(fname)) + cmd
fname_build = os.path.join(os.path.dirname(fname),
'10-build.sh')
fname_run = os.path.join(os.path.dirname(fname), '20-run.sh')
image_name = self.git_obj.owner + '-' + \
self.git_obj.repo + \
":" + self.get_folder_name(self.revision) + \
"_" + str(count)
image_name = image_name.lower()
else:
raise Exception("No command format found %s" %
(self.command_format))
with open(fname, "w") as fdockerfile:
fdockerfile.write(cmd.encode('utf-8'))
if fname_build:
with open(fname_build, "w") as fbuild:
fbuild.write("#!/bin/bash\ndocker build $1 %s -t %s %s\n" %
(self.build_extra_args, image_name,
os.path.dirname(fname)))
st = os.stat(fname_build)
os.chmod(fname_build, st.st_mode | stat.S_IEXEC)
if fname_run:
with open(fname_run, "w") as fbuild:
fbuild.write("#!/bin/bash\ndocker run " + "$1 %s %s $2\n" %
(
self.run_extra_args,
image_name,
))
if self.include_cleanup:
fbuild.write("docker rmi -f '%s'\n" % image_name)
st = os.stat(fname_run)
os.chmod(fname_run, st.st_mode | stat.S_IEXEC)
if self.command_format == 'bash':
st = os.stat(fname)
os.chmod(fname, st.st_mode | stat.S_IEXEC)
count += 1
fname_scripts.append(os.path.dirname(fname))
return fname_scripts
class travis(object):
def load_travis_file(self, branch):
self.git_obj.update()
yaml_loaded = None
for fname in ['.travis.yml', '.shippable.yml']:
yaml_str = self.git_obj.show_file(fname, branch)
try:
yaml_loaded = yaml.load(yaml_str)
break
except AttributeError:
pass
return yaml_loaded
def get_folder_name(self, name):
for invalid_char in '@:/#':
name = name.replace(invalid_char, '_')
return name
def get_repo_path(self, root_path):
name = self.get_folder_name(self.git_project)
repo_path = os.path.join(root_path, 'repo', name)
return repo_path
def get_script_path(self, root_path):
name = self.get_folder_name(self.git_project)
script_path = os.path.join(
root_path, 'script',
name, self.get_folder_name(self.revision), self.sha)
return script_path
def __init__(self, git_project, revision,
command_format='docker', docker_user=None,
root_path=None, default_docker_image=None):
"""
Method Constructor
@fname_dockerfile: str name of file dockerfile to save.
@format_cmd: str name of format of command.
bash: Make a bash script.
docker: Make a dockerfile script.
"""
if root_path is None:
self.root_path = os.path.join(
gettempdir(),
os.path.splitext(os.path.basename(__file__))[0],
)
else:
self.root_path = os.path.expandvars(
os.path.expanduser(root_path)
)
self.default_docker_image = default_docker_image
self.git_project = git_project
self.revision = revision
git_path = self.get_repo_path(self.root_path)
self.git_obj = GitRun(git_project, git_path)
self.travis_data = self.load_travis_file(revision)
self.sha = self.git_obj.get_sha(revision)
if not self.travis_data:
raise Exception(
"Make sure you have access to repository"
" " + git_project + " and that the file .travis.yml"
" exists in branch or revision " + revision
)
self.travis2docker_section = [
# ('build_image', 'build_image'),
('python', 'python'),
('env', 'env'),
('install', 'run'),
('script', 'script'),
]
self.travis2docker_section_dict = dict(self.travis2docker_section)
self.scripts_root_path = self.get_script_path(self.root_path)
env_regex_str = r"(?P<var>[\w]*)[ ]*[\=][ ]*[\"\']{0,1}" + \
r"(?P<value>[\w\.\-\_/\$\{\}\:,]*)[\"\']{0,1}"
export_regex_str = r"(?P<export>export|EXPORT)( )+" + env_regex_str
self.env_regex = re.compile(env_regex_str, re.M)
self.export_regex = re.compile(export_regex_str, re.M)
self.extra_env_from_run = ""
self.command_format = command_format
self.docker_user = docker_user or 'root'
# def get_travis_type(self, repo_git):
# self.repo_git_type = None
# if os.path.isdir(repo_git):
# self.repo_git_type = 'localpath'
# elif os.path.isfile(repo_git):
# self.repo_git_type = 'file'
# else:
# regex = r"(?P<host>(git@|https://)([\w\.@]+)(/|:))" \
# r"(?P<owner>[~\w,\-,\_]+)/" \
# r"(?P<repo>[\w,\-,\_]+)(.git){0,1}((/){0,1})"
# match_object = re.search(regex, repo_git)
# if match_object:
# self.repo_git_type = 'remote'
# def get_travis_data(self, repo_git, branch=None):
# self.get_travis_type(repo_git)
# if self.repo_git_type == 'remote':
# if branch is None:
# raise "You need specify branch name with remote repository"
# "git show 8.0:.travis.yml"
def get_travis_section(self, section, *args, **kwards):
section_type = self.travis2docker_section_dict.get(section, False)
if not section_type:
return None
section_data = self.travis_data.get(section, "")
if isinstance(section_data, basestring):
section_data = [section_data]
job_method = getattr(self, 'get_travis2docker_' + section_type)
return job_method(section_data, *args, **kwards)
def get_travis2docker_run(self, section_data,
custom_command_format=None,
add_extra_env=True):
if custom_command_format is None:
custom_command_format = self.command_format
docker_run = ''
for line in section_data:
export_regex_findall = self.export_regex.findall(line)
for dummy, dummy, var, value in export_regex_findall:
self.extra_env_from_run += "%s=%s " % (var, value)
extra_env = add_extra_env and self.extra_env_from_run or ''
if not export_regex_findall:
if custom_command_format == 'bash':
docker_run += '\n' + extra_env + line
elif custom_command_format == 'docker':
docker_run += '\nRUN ' + extra_env + line
return docker_run
def scape_bash_command(self, command):
return command.replace('$', '\\$').replace('_', '\\_')
def get_travis2docker_script(self, section_data):
cmd_str = self.get_travis2docker_run(
section_data, custom_command_format='bash',
add_extra_env=False)
if self.docker_user == 'root':
sudo_prefix = ''
else:
sudo_prefix = 'sudo'
if self.command_format == 'docker' and cmd_str:
cmd_str = self.extra_env_from_run + cmd_str
cmd_str = self.scape_bash_command(cmd_str)
cmd_str = '\\\\n'.join(cmd_str.strip('\n').split('\n'))
cmd_str = 'RUN echo """%s"""' % (cmd_str,) \
+ ' | ' + sudo_prefix + 'tee -a /entrypoint.sh \\' + \
'\n && ' + sudo_prefix + \
' chown %s:%s /entrypoint.sh \\' % (
self.docker_user, self.docker_user) + \
'\n && ' + sudo_prefix + ' chmod +x /entrypoint.sh'
cmd_str += '\nENTRYPOINT /entrypoint.sh'
return cmd_str
def get_travis2docker_env(self, section_data):
global_data_cmd = []
matrix_data_cmd = []
if 'global' in section_data:
global_data = section_data.pop('global')
global_data_cmd = [
global_env
for global_env
in self.get_travis2docker_env(global_data)
]
if 'matrix' in section_data:
matrix_data = section_data.pop('matrix')
matrix_data_cmd = [
matrix_env
for matrix_env
in self.get_travis2docker_env(matrix_data)
]
for combination in itertools.product(global_data_cmd, matrix_data_cmd):
command_str = '\n'.join(combination) + "\n"
yield command_str
for line in section_data:
if line in ['global', 'matrix']:
continue
if not isinstance(line, str):
continue
docker_env = ""
for var, value in self.env_regex.findall(line):
if self.command_format == 'bash':
docker_env += "\nexport %s=%s" % (var, value)
elif self.command_format == 'docker':
docker_env += "\nENV %s=%s" % (var, value)
yield docker_env
def get_travis2docker_python(self, section_data):
for line in section_data:
yield "\n# TODO: Use python version: " + line
def get_default_cmd(self, dockerfile_path):
home_user_path = self.docker_user == 'root' and "/root" \
or os.path.join("/home", self.docker_user)
project, branch = self.git_project, self.revision
travis_build_dir = os.path.join(
home_user_path, "build",
self.git_obj.owner, self.git_obj.repo,
)
if self.command_format == 'bash':
cmd = "\nsudo su - " + self.docker_user + \
"\nsudo chown -R %s:%s %s" % (self.docker_user, self.docker_user, home_user_path) + \
"\nexport TRAVIS_BUILD_DIR=%s" % (travis_build_dir) + \
"\ngit clone --single-branch %s -b %s " % (project, branch) + \
"${TRAVIS_BUILD_DIR}" + \
"\n"
elif self.command_format == 'docker':
dkr_files_path = os.path.join(dockerfile_path, "files")
if not os.path.exists(dkr_files_path):
os.makedirs(dkr_files_path)
if not os.path.exists(os.path.join(dkr_files_path, 'ssh')):
shutil.copytree(
os.path.expanduser("~/.ssh"),
os.path.join(dkr_files_path, 'ssh')
)
cmd_refs = 'pull' in self.revision and \
'+refs/%s/head:refs/%s' % (
self.revision, self.revision) or \
'+refs/heads/%s:refs/heads/%s' % (
self.revision, self.revision)
# TODO: Use sha
cmd_git_clone = [
"git init ${TRAVIS_BUILD_DIR}",
"cd ${TRAVIS_BUILD_DIR}",
"git remote add origin " + project,
"git fetch --update-head-ok -p origin %s" % (cmd_refs),
"git reset --hard " + self.revision,
]
git_user_email = self.git_obj.get_config_data("user.email")
if git_user_email:
cmd_git_clone.append(
"git config --global user.email \"%s\"" % (git_user_email)
)
else:
cmd_git_clone.append(
"git config --unset --global user.email"
)
git_user_name = self.git_obj.get_config_data("user.name")
if git_user_name:
cmd_git_clone.append(
"git config --global user.name \"%s\"" % (git_user_name)
)
else:
cmd_git_clone.append(
"git config --unset --global user.name"
)
if self.docker_user == 'root':
sudo_prefix = ''
else:
sudo_prefix = 'sudo'
cmd = 'FROM ' + (self.travis_data.get('build_image', False) or
self.default_docker_image) + \
'\nUSER ' + self.docker_user + \
'\nENV HOME=' + home_user_path + \
'\nADD ' + os.path.join("files", 'ssh') + ' ' + \
os.path.join(home_user_path, '.ssh') + \
"\nENV TRAVIS_BUILD_DIR=%s" % (travis_build_dir) + \
"\nWORKDIR ${TRAVIS_BUILD_DIR}" + \
"\nRUN %s chown -R %s:%s %s" % (
sudo_prefix, self.docker_user,
self.docker_user, home_user_path) + \
"\nRUN " + ' \\\n && '.join(cmd_git_clone) + \
"\n"
return cmd
def get_travis2docker_iter(self):
travis2docker_cmd_static_str = ""
travis2docker_cmd_iter_list = []
for travis_section, dummy in self.travis2docker_section:
travis2docker_section = self.get_travis_section(travis_section)
if isinstance(travis2docker_section, types.GeneratorType):
travis2docker_cmd_iter_list.append([
item_iter for item_iter in travis2docker_section
])
elif isinstance(travis2docker_section, basestring):
travis2docker_cmd_static_str += travis2docker_section + "\n"
for combination in itertools.product(*travis2docker_cmd_iter_list):
command_str = '\n'.join(combination) + "\n" + \
travis2docker_cmd_static_str
yield command_str
def get_travis2docker(self):
count = 1
fname_scripts = []
for cmd in self.get_travis2docker_iter():
fname = os.path.join(
self.scripts_root_path,
str(count)
)
fname_build = None
fname_run = None
if not os.path.exists(fname):
os.makedirs(fname)
if self.command_format == 'bash':
fname = os.path.join(fname, 'script.sh')
cmd = self.get_default_cmd(os.path.dirname(fname)) + cmd
elif self.command_format == 'docker':
fname = os.path.join(fname, 'Dockerfile')
cmd = self.get_default_cmd(os.path.dirname(fname)) + cmd
fname_build = os.path.join(
os.path.dirname(fname), '10-build.sh'
)
fname_run = os.path.join(
os.path.dirname(fname), '20-run.sh'
)
image_name = self.git_obj.owner + '-' + \
self.git_obj.repo + \
":" + self.get_folder_name(self.revision) + \
"_" + str(count)
image_name = image_name.lower()
else:
raise Exception(
"No command format found %s" % (self.command_format)
)
with open(fname, "w") as fdockerfile:
fdockerfile.write(cmd)
if fname_build:
with open(fname_build, "w") as fbuild:
fbuild.write(
"docker build $1 -t %s %s" % (
image_name,
os.path.dirname(fname)
)
)
st = os.stat(fname_build)
os.chmod(fname_build, st.st_mode | stat.S_IEXEC)
if fname_run:
with open(fname_run, "w") as fbuild:
fbuild.write(
"docker run $1 -itP %s $2" % (
image_name
)
)
st = os.stat(fname_run)
os.chmod(fname_run, st.st_mode | stat.S_IEXEC)
if self.command_format == 'bash':
st = os.stat(fname)
os.chmod(fname, st.st_mode | stat.S_IEXEC)
count += 1
fname_scripts.append(os.path.dirname(fname))
return fname_scripts
class TravisWeblateUpdate(object):
GIT_COMMIT_INFO = {
'author':
'Weblate bot <weblate@bot>',
'message':
'[REF] i18n: Updating translation terms from weblate '
'[ci skip]'
}
def __init__(self):
self._git = GitRun(os.path.join(os.getcwd(), '.git'), True)
self.branch = os.environ.get("TRAVIS_BRANCH",
self._git.get_branch_name())
remote = self._git.run(["ls-remote", "--get-url", "origin"])
name = remote.replace(':', '/')
name = re.sub('.+@', '', name)
name = re.sub('.git$', '', name)
name = re.sub('^https://', '', name)
name = re.sub('^http://', '', name)
match = re.search(r'(?P<host>[^/]+)/(?P<owner>[^/]+)/(?P<repo>[^/]+)',
name)
if match:
name = ("%(host)s:%(owner)s/%(repo)s (%(branch)s)" %
dict(match.groupdict(), branch=self.branch))
self.repo_name = name
self.wl_api = WeblateApi()
self.gh_api = GitHubApi()
self._travis_home = os.environ.get("HOME", "~/")
self._travis_build_dir = os.environ.get("TRAVIS_BUILD_DIR", "../..")
self._odoo_version = os.environ.get("VERSION")
self._odoo_branch = os.environ.get("ODOO_BRANCH")
self._langs = (parse_list(os.environ.get("LANG_ALLOWED"))
if os.environ.get("LANG_ALLOWED", False) else [])
self._odoo_full = os.environ.get("ODOO_REPO", "odoo/odoo")
self._server_path = get_server_path(
self._odoo_full, (self._odoo_branch or self._odoo_version),
self._travis_home)
self._addons_path = get_addons_path(self._travis_home,
self._travis_build_dir,
self._server_path)
self._database = os.environ.get('MQT_TEST_DB', 'openerp_test')
self._connection_context = context_mapping.get(self._odoo_version,
Odoo10Context)
self._apply_patch_odoo()
self._get_modules_installed()
def _check(self):
self.wl_api._check()
self.gh_api._check()
def _apply_patch_odoo(self):
"""This patch is necessary because the weblate does not check which
word and the translated are the same to use it in its percentage of
translated"""
for base in ('odoo', 'openerp'):
p_file = os.path.join(self._server_path, base,
os.path.join('tools', 'translate.py'))
if os.path.isfile(p_file):
sed = [
"sed", "-i", "-e",
r"s/translation'] = src/translation'] = ''/g", p_file
]
print " ".join(sed)
subprocess.call(sed)
p_file = os.path.join(
self._server_path, base,
os.path.join('addons', 'base', 'ir', 'ir_translation.py'))
if os.path.isfile(p_file):
sed = [
"sed", "-i", "-e",
r"s/if\snot\strans_dict\['value']:/if False:/g", p_file
]
print " ".join(sed)
subprocess.call(sed)
def _get_modules_installed(self):
self._installed_modules = []
modules_found = []
for name in ('__openerp__.py', '__manifest__.py'):
modules = glob.glob('%s/**/%s' % (self._travis_build_dir, name))
if not modules:
continue
modules_found.extend(
[os.path.dirname(module).split('/')[-1] for module in modules])
with self._connection_context(self._server_path, self._addons_path,
self._database) as odoo_context:
odoo_context.cr.execute(
"select name from ir_module_module"
" where state = 'installed' and "
"name in %s", (tuple(modules_found), ))
modules = odoo_context.cr.dictfetchall()
self._installed_modules = [module['name'] for module in modules]
def _generate_odoo_po_files(self, component):
generated = False
with self._connection_context(self._server_path, self._addons_path,
self._database) as odoo_context:
module = component['name']
if module not in self._installed_modules:
return generated
print("\n", yellow("Obtaining POT file for %s" % module))
i18n_folder = os.path.join(self._travis_build_dir, module, 'i18n')
if not os.path.isdir(i18n_folder):
os.makedirs(i18n_folder)
# Put git add for letting known git which translations to update
po_files = glob.glob(os.path.join(i18n_folder, '*.po'))
for lang in self._langs:
if os.path.isfile(os.path.join(i18n_folder, lang + '.po')):
continue
po_content = odoo_context.get_pot_contents(module, lang)
if not po_content:
continue
with open(os.path.join(i18n_folder, lang + '.po'), 'wb')\
as f_po:
f_po.write(po_content)
for po_file_name in po_files:
lang = os.path.basename(os.path.splitext(po_file_name)[0])
if self._langs and lang not in self._langs:
# Limit just allowed languages if is defined
continue
po_file_path = os.path.join(i18n_folder, po_file_name)
with open(po_file_path, 'r') as f_po:
odoo_context.load_po(f_po, lang)
new_content = odoo_context.get_pot_contents(module, lang)
if not new_content:
continue
with open(po_file_path, 'wb') as f_po:
f_po.write(new_content)
diff = self._git.run(["diff", "HEAD", po_file_path])
if diff.count('msgstr') == 1:
self._git.run(["checkout", po_file_path])
if self._git.run(["add", "-v"] + po_files):
generated = True
return generated
def _check_conflict(self, component):
status = self._git.run(["status"])
conflicts = [
item for item in status.split('\n')
if (item.startswith('\tboth modified')
and component['filemask'].replace('/*.po', '') in item)
]
if conflicts:
self._register_pull_request(component, status)
return True
return False
def _register_pull_request(self, component, status):
branch_name = 'conflict-%s-weblate' % self.branch
self._git.run(["add", component['filemask']])
self._git.run([
"commit", "--no-verify", "--author='Weblate bot <weblate@bot>'",
"-m", "[REF] i18n: Conflict on the daily cron", "-m", status
])
self._git.run(["branch", "-m", branch_name])
self._git.run(["push", "-f", "origin", branch_name])
pull = self.gh_api.create_pull_request({
'title':
'[REF] i18n: Conflict on the daily cron',
'head':
'%s:%s' %
(self.repo_name.split('/')[0].split(':')[1], branch_name),
'base':
self.branch,
'body':
status
})
self._git.run(
["checkout", "-qb", self.branch,
"origin/%s" % self.branch])
self._git.run(["branch", "-D", branch_name])
print yellow("The pull request register is: %s" % pull['html_url'])
def _commit_weblate(self, first_commit=False):
if ('nothing to commit, working tree clean'
in self._git.run(["status"])):
return first_commit
if first_commit:
self._git.run(["commit", "--no-verify", "--amend", "--no-edit"])
else:
self._git.run([
"commit", "--no-verify",
"--author='%s'" % self.GIT_COMMIT_INFO['author'], "-m",
self.GIT_COMMIT_INFO['message']
])
first_commit = True
return first_commit
def _push_git_repository(self):
po_files = self._git.run(
["show", "--format=format:'%H'", "--name-only"]).split('\n')
if not len(po_files) > 1:
return False
commit = self.gh_api.create_commit(self.GIT_COMMIT_INFO['message'],
self.branch, po_files[1:])
if commit:
for component in self.wl_api.components:
self.wl_api.component_repository(component, 'reset')
self.wl_api.component_repository(component, 'pull')
return commit
def update(self):
self._check()
self.wl_api.load_project(self.repo_name, self.branch)
if not self.wl_api.components:
print yellow("No component found for %s" % self.repo_name)
return 1
with self.wl_api.component_lock():
self._git.run(["fetch", "origin"])
first_commit = False
for component in self.wl_api.components:
print yellow("Component %s" % component['slug'])
name = '%s-wl' % component['slug']
remote = (self.wl_api.host.replace('api', 'git') + '/' +
self.wl_api.project['slug'] + '/' +
component['slug'])
self._git.run([
"checkout", "-qb", self.branch,
"origin/%s" % self.branch
])
self.wl_api.component_repository(component, 'pull')
self._git.run(["remote", "add", name, remote])
self._git.run(["fetch", name])
if self._generate_odoo_po_files(component):
first_commit = self._commit_weblate(first_commit)
self._git.run([
"merge", "--squash", "-s", "recursive", "-X", "ours",
"%s/%s" % (name, self.branch)
])
self._git.run(["remote", "remove", name])
if self._check_conflict(component):
break
if (component['filemask'].replace('/*.po', '')
in self._git.run(["status"])):
self._git.run(["add", component['filemask']])
first_commit = self._commit_weblate(first_commit)
if self._check_conflict(component):
break
first_commit = self._commit_weblate(first_commit)
if not self._push_git_repository():
return 1
return 0
class TravisWeblateUpdate(object):
GIT_COMMIT_INFO = {
'author': 'Weblate bot <weblate@bot>',
'message': '[REF] i18n: Updating translation terms from weblate '
'[ci skip]'
}
def __init__(self):
self._git = GitRun(os.path.join(os.getcwd(), '.git'), True)
self.branch = os.environ.get("TRAVIS_BRANCH",
self._git.get_branch_name())
remote = self._git.run(["ls-remote", "--get-url", "origin"])
name = remote.replace(':', '/')
name = re.sub('.+@', '', name)
name = re.sub('.git$', '', name)
name = re.sub('^https://', '', name)
name = re.sub('^http://', '', name)
match = re.search(
r'(?P<host>[^/]+)/(?P<owner>[^/]+)/(?P<repo>[^/]+)', name)
if match:
name = ("%(host)s:%(owner)s/%(repo)s (%(branch)s)" %
dict(match.groupdict(), branch=self.branch))
self.repo_name = name
self.wl_api = WeblateApi()
self.gh_api = GitHubApi()
self._travis_home = os.environ.get("HOME", "~/")
self._travis_build_dir = os.environ.get("TRAVIS_BUILD_DIR", "../..")
self._odoo_version = os.environ.get("VERSION")
self._odoo_branch = os.environ.get("ODOO_BRANCH")
self._langs = (parse_list(os.environ.get("LANG_ALLOWED")) if
os.environ.get("LANG_ALLOWED", False) else [])
self._odoo_full = os.environ.get("ODOO_REPO", "odoo/odoo")
self._server_path = get_server_path(self._odoo_full,
(self._odoo_branch or
self._odoo_version),
self._travis_home)
self._addons_path = get_addons_path(self._travis_home,
self._travis_build_dir,
self._server_path)
self._database = os.environ.get('MQT_TEST_DB', 'openerp_test')
self._connection_context = context_mapping.get(
self._odoo_version, Odoo10Context)
self._apply_patch_odoo()
self._get_modules_installed()
def _check(self):
self.wl_api._check()
self.gh_api._check()
def _apply_patch_odoo(self):
"""This patch is necessary because the weblate does not check which
word and the translated are the same to use it in its percentage of
translated"""
for base in ('odoo', 'openerp'):
p_file = os.path.join(self._server_path, base,
os.path.join('tools', 'translate.py'))
if os.path.isfile(p_file):
sed = ["sed", "-i", "-e",
r"s/translation'] = src/translation'] = ''/g", p_file]
print " ".join(sed)
subprocess.call(sed)
p_file = os.path.join(self._server_path, base,
os.path.join('addons', 'base', 'ir',
'ir_translation.py'))
if os.path.isfile(p_file):
sed = ["sed", "-i", "-e",
r"s/if\snot\strans_dict\['value']:/if False:/g", p_file]
print " ".join(sed)
subprocess.call(sed)
def _get_modules_installed(self):
self._installed_modules = []
modules_found = []
for name in ('__openerp__.py', '__manifest__.py'):
modules = glob.glob('%s/**/%s' % (self._travis_build_dir, name))
if not modules:
continue
modules_found.extend([
os.path.dirname(module).split('/')[-1] for module in
modules])
with self._connection_context(self._server_path, self._addons_path,
self._database) as odoo_context:
odoo_context.cr.execute("select name from ir_module_module"
" where state = 'installed' and "
"name in %s", (tuple(modules_found),))
modules = odoo_context.cr.dictfetchall()
self._installed_modules = [module['name'] for module in modules]
def _generate_odoo_po_files(self, module, only_installed=True):
generated = False
with self._connection_context(self._server_path, self._addons_path,
self._database) as odoo_context:
if only_installed and module not in self._installed_modules:
return generated
print("\n", yellow("Obtaining POT file for %s" % module))
i18n_folder = os.path.join(self._travis_build_dir, module, 'i18n')
if not os.path.isdir(i18n_folder):
os.makedirs(i18n_folder)
# Put git add for letting known git which translations to update
po_files = glob.glob(os.path.join(i18n_folder, '*.po'))
for lang in self._langs:
if os.path.isfile(os.path.join(i18n_folder, lang + '.po')):
continue
po_content = odoo_context.get_pot_contents(module, lang)
if not po_content:
continue
with open(os.path.join(i18n_folder, lang + '.po'), 'wb')\
as f_po:
f_po.write(po_content)
if self._git.run(["add", "-v", f_po.name]):
generated = True
for po_file_name in po_files:
lang = os.path.basename(os.path.splitext(po_file_name)[0])
if self._langs and lang not in self._langs:
# Limit just allowed languages if is defined
continue
po_file_path = os.path.join(i18n_folder, po_file_name)
with open(po_file_path, 'r') as f_po:
odoo_context.load_po(f_po, lang)
new_content = odoo_context.get_pot_contents(module, lang)
if not new_content:
continue
with open(po_file_path, 'wb') as f_po:
f_po.write(new_content)
diff = self._git.run(["diff", "HEAD", po_file_path])
if diff.count('msgstr') == 1:
self._git.run(["checkout", po_file_path])
if self._git.run(["add", "-v"] + po_files):
generated = True
return generated
def _check_conflict(self, component):
status = self._git.run(["status"])
conflicts = [item for item in status.split('\n')
if (item.startswith('\tboth modified') and
component['filemask'].replace('/*.po', '') in item)]
if conflicts:
self._register_pull_request(component, status)
return True
return False
def _register_pull_request(self, component, status):
branch_name = 'conflict-%s-weblate' % self.branch
self._git.run(["add", component['filemask']])
self._git.run(["commit", "--no-verify",
"--author='Weblate bot <weblate@bot>'",
"-m", "[REF] i18n: Conflict on the daily cron",
"-m", status])
self._git.run(["branch", "-m", branch_name])
self._git.run(["push", "-f", "origin", branch_name])
pull = self.gh_api.create_pull_request({
'title': '[REF] i18n: Conflict on the daily cron',
'head': '%s:%s' % (self.repo_name.split('/')[0].split(':')[1],
branch_name),
'base': self.branch,
'body': status
})
self._git.run(["checkout", "-qb", self.branch,
"origin/%s" % self.branch])
self._git.run(["branch", "-D", branch_name])
print yellow("The pull request register is: %s" % pull['html_url'])
def _commit_weblate(self, first_commit=False):
if ('nothing to commit, working tree clean'
in self._git.run(["status"])):
return first_commit
if first_commit:
self._git.run(["commit", "--no-verify", "--amend",
"--no-edit"])
else:
self._git.run(["commit", "--no-verify",
"--author='%s'" % self.GIT_COMMIT_INFO['author'],
"-m", self.GIT_COMMIT_INFO['message']])
first_commit = True
return first_commit
def _push_git_repository(self):
po_files = self._git.run(["show", "--format=format:'%H'",
"--name-only"]).split('\n')
if not len(po_files) > 1:
return False
commit = self.gh_api.create_commit(self.GIT_COMMIT_INFO['message'],
self.branch,
po_files[1:])
if commit:
for component in self.wl_api.components:
self.wl_api.component_repository(component, 'reset')
self.wl_api.component_repository(component, 'pull')
return commit
def update(self):
self._check()
self.wl_api.load_project(self.repo_name, self.branch)
with self.wl_api.component_lock():
self._git.run(["fetch", "origin"])
first_commit = False
component = [item for item in self.wl_api.components
if item['git_export']]
if len(component) > 1:
print yellow("To many repository for this project %s" %
self.wl_api.project['name'])
return 1
remote = (self.wl_api.ssh + '/' + self.wl_api.project['slug'] +
'/' + component[0]['slug'])
name = '%s-wl' % self.wl_api.project['slug']
self._git.run(["remote", "add", name, remote])
for component in self.wl_api.components:
print yellow("Component %s" % component['slug'])
self._git.run(["checkout", "-qb", self.branch,
"origin/%s" % self.branch])
self.wl_api.component_repository(component, 'pull')
self._git.run(["fetch", name])
if self._generate_odoo_po_files(component['name']):
first_commit = self._commit_weblate(first_commit)
self._git.run(["merge", "--squash", "-s", "recursive", "-X",
"ours", "%s/%s" % (name, self.branch)])
if self._check_conflict(component):
break
if (component['filemask'].replace('/*.po', '') in
self._git.run(["status"])):
self._git.run(["add", component['filemask']])
first_commit = self._commit_weblate(first_commit)
if self._check_conflict(component):
break
first_commit = self._commit_weblate(first_commit)
self._git.run(["remote", "remove", name])
modules_no_processed = [module for module in
self._installed_modules if module not in
[comp['name'] for comp in
self.wl_api.components]]
for component in modules_no_processed:
if self._generate_odoo_po_files(component,
only_installed=False):
first_commit = self._commit_weblate(first_commit)
if not self._push_git_repository():
return 1
return 0
def translate_gitlab_env():
""" This sets Travis CI's variables using GitLab CI's variables. This
makes possible to run scripts which depend on Travis CI's variables
when running under Gitlab CI.
For documentation on these environment variables, please check:
- Predefined environment variables on Travis CI:
https://docs.travis-ci.com/user/environment-variables#Default-Environment-Variables
- Predefined environment variables on GitLab CI:
https://docs.gitlab.com/ee/ci/variables/#predefined-variables-environment-variables
"""
if not os.environ.get("GITLAB_CI"):
return False
build_dir = os.environ.get("CI_PROJECT_DIR", ".")
git_run_obj = GitRun(repo_path=os.path.join(build_dir, ".git"))
commit_message = git_run_obj.run(["log", "-1", "--pretty=%b"])
head_branch = os.environ.get("CI_COMMIT_REF_NAME")
# This is a guess
target_branch = os.environ.get("VERSION")
# Set environment variables
os.environ.update({
# Predefined values
"TRAVIS": "true",
"CONTINUOUS_INTEGRATION": "true",
"HAS_JOSH_K_SEAL_OF_APPROVAL": "true",
"RAILS_ENV": "test",
"RACK_ENV": "test",
"MERB_ENV": "test",
"TRAVIS_ALLOW_FAILURE": "false",
"TRAVIS_JOB_NUMBER": "1",
"TRAVIS_TEST_RESULT": "0",
"TRAVIS_COMMIT_RANGE": "unknown",
# Dinamic values
"TRAVIS_BRANCH": target_branch,
"TRAVIS_COMMIT_MESSAGE": commit_message,
"TRAVIS_OS_NAME": sys.platform,
"TRAVIS_SUDO": "true" if getuser() == "root" else "false",
})
# Set variables whose values are already directly set in other variables
equivalent_vars = [
("TRAVIS_BUILD_DIR", "CI_PROJECT_DIR"),
("TRAVIS_BUILD_ID", "CI_BUILD_ID"),
("TRAVIS_BUILD_NUMBER", "CI_JOB_ID"),
("TRAVIS_COMMIT", "CI_COMMIT_SHA"),
("TRAVIS_EVENT_TYPE", "CI_PIPELINE_SOURCE"),
("TRAVIS_JOB_ID", "CI_JOB_ID"),
("TRAVIS_REPO_SLUG", "CI_PROJECT_PATH"),
("TRAVIS_BUILD_STAGE_NAME", "CI_BUILD_STAGE"),
]
os.environ.update({
travis_var: os.environ.get(gitlab_var)
for travis_var, gitlab_var in equivalent_vars
if gitlab_var in os.environ
})
# If within an MR
is_mr = target_branch != head_branch
if is_mr:
os.environ.update({
# there's no way to know the MR number. For more info, see:
# https://gitlab.com/gitlab-org/gitlab-ce/issues/15280
"TRAVIS_PULL_REQUEST": "unknown",
"TRAVIS_PULL_REQUEST_BRANCH": head_branch,
"TRAVIS_PULL_REQUEST_SHA": os.environ.get("CI_COMMIT_SHA"),
"TRAVIS_PULL_REQUEST_SLUG": os.environ.get("CI_PROJECT_PATH"),
})
else:
os.environ.update({
"TRAVIS_PULL_REQUEST": "false",
"TRAVIS_PULL_REQUEST_BRANCH": "",
"TRAVIS_PULL_REQUEST_SHA": "",
"TRAVIS_PULL_REQUEST_SLUG": "",
})
return True
def translate_gitlab_env():
""" This sets Travis CI's variables using GitLab CI's variables. This
makes possible to run scripts which depend on Travis CI's variables
when running under Gitlab CI.
For documentation on these environment variables, please check:
- Predefined environment variables on Travis CI:
https://docs.travis-ci.com/user/environment-variables#Default-Environment-Variables
- Predefined environment variables on GitLab CI:
https://docs.gitlab.com/ee/ci/variables/#predefined-variables-environment-variables
"""
if not os.environ.get("GITLAB_CI"):
return False
build_dir = os.environ.get("CI_PROJECT_DIR", ".")
git_run_obj = GitRun(repo_path=os.path.join(build_dir, ".git"))
commit_message = git_run_obj.run(["log", "-1", "--pretty=%b"])
head_branch = os.environ.get("CI_COMMIT_REF_NAME")
# This is a guess
target_branch = os.environ.get("VERSION")
# Set environment variables
os.environ.update({
# Predefined values
"TRAVIS":
"true",
"CONTINUOUS_INTEGRATION":
"true",
"HAS_JOSH_K_SEAL_OF_APPROVAL":
"true",
"RAILS_ENV":
"test",
"RACK_ENV":
"test",
"MERB_ENV":
"test",
"TRAVIS_ALLOW_FAILURE":
"false",
"TRAVIS_JOB_NUMBER":
"1",
"TRAVIS_TEST_RESULT":
"0",
"TRAVIS_COMMIT_RANGE":
"unknown",
# Dinamic values
"TRAVIS_BRANCH":
target_branch,
"TRAVIS_COMMIT_MESSAGE":
commit_message,
"TRAVIS_OS_NAME":
sys.platform,
"TRAVIS_SUDO":
"true" if getuser() == "root" else "false",
})
# Set variables whose values are already directly set in other variables
equivalent_vars = [
("TRAVIS_BUILD_DIR", "CI_PROJECT_DIR"),
("TRAVIS_BUILD_ID", "CI_BUILD_ID"),
("TRAVIS_BUILD_NUMBER", "CI_JOB_ID"),
("TRAVIS_COMMIT", "CI_COMMIT_SHA"),
("TRAVIS_EVENT_TYPE", "CI_PIPELINE_SOURCE"),
("TRAVIS_JOB_ID", "CI_JOB_ID"),
("TRAVIS_REPO_SLUG", "CI_PROJECT_PATH"),
("TRAVIS_BUILD_STAGE_NAME", "CI_BUILD_STAGE"),
]
os.environ.update({
travis_var: os.environ.get(gitlab_var)
for travis_var, gitlab_var in equivalent_vars
if gitlab_var in os.environ
})
# If within an MR
is_mr = target_branch != head_branch
if is_mr:
os.environ.update({
# there's no way to know the MR number. For more info, see:
# https://gitlab.com/gitlab-org/gitlab-ce/issues/15280
"TRAVIS_PULL_REQUEST":
"unknown",
"TRAVIS_PULL_REQUEST_BRANCH":
head_branch,
"TRAVIS_PULL_REQUEST_SHA":
os.environ.get("CI_COMMIT_SHA"),
"TRAVIS_PULL_REQUEST_SLUG":
os.environ.get("CI_PROJECT_PATH"),
})
else:
os.environ.update({
"TRAVIS_PULL_REQUEST": "false",
"TRAVIS_PULL_REQUEST_BRANCH": "",
"TRAVIS_PULL_REQUEST_SHA": "",
"TRAVIS_PULL_REQUEST_SLUG": "",
})
return True