def __init__(self, target=None, email=None):
# Read config
config = read_config()
self.gh_token = config.get('GitHub', 'token')
tx_username = (
config.get('Transifex', 'username') or
os.environ.get('TRANSIFEX_USER') or
TX_USERNAME_DEFAULT)
tx_password = (
config.get('Transifex', 'password') or
os.environ.get('TRANSIFEX_PASSWORD'))
self.tx_num_retries = (
config.get('Transifex', 'num_retries') or
os.environ.get('TRANSIFEX_RETRIES'))
self.tx_org = (
config.get('Transifex', 'organization') or
os.environ.get('TRANSIFEX_ORGANIZATION') or
TX_ORG_DEFAULT)
self.gh_org = target or self.tx_org
# Connect to GitHub
self.github = github_login.login()
gh_user = wrap_gh_call(self.github.user)
if not gh_user.email and not email:
raise Exception(
'Email required to commit to github. Please provide one on '
'the command line or make the one of your github profile '
'public.'
)
self.gh_credentials = {
'name': gh_user.name or str(gh_user),
'email': gh_user.email or email,
}
# Connect to Transifex
self.tx_api = API(TX_URL, auth=(tx_username, tx_password))
def __init__(self, target=None, email=None):
# Read config
config = read_config()
self.gh_token = config.get('GitHub', 'token')
tx_username = (
config.get('Transifex', 'username') or
os.environ.get('TRANSIFEX_USER') or
TX_USERNAME_DEFAULT)
tx_password = (
config.get('Transifex', 'password') or
os.environ.get('TRANSIFEX_PASSWORD'))
self.tx_num_retries = (
config.get('Transifex', 'num_retries') or
os.environ.get('TRANSIFEX_RETRIES'))
self.tx_org = (
config.get('Transifex', 'organization') or
os.environ.get('TRANSIFEX_ORGANIZATION') or
TX_ORG_DEFAULT)
self.gh_org = target or self.tx_org
# Connect to GitHub
self.github = github_login.login()
gh_user = self.github.user()
if not gh_user.email and not email:
raise Exception(
'Email required to commit to github. Please provide one on '
'the command line or make the one of your github profile '
'public.')
self.gh_credentials = {'name': gh_user.name or str(gh_user),
'email': gh_user.email or email}
# Connect to Transifex
self.tx_api = API(TX_URL, auth=(tx_username, tx_password))
def setup_api():
session = requests.Session()
if API_HOST.startswith('https'):
# Only use the HostHeaderSSLAdapter for HTTPS connections
adapter_class = host_header_ssl.HostHeaderSSLAdapter
else:
adapter_class = requests.adapters.HTTPAdapter
session.mount(
API_HOST,
adapter_class(max_retries=3),
)
session.headers.update({'Host': PRODUCTION_DOMAIN})
api_config = {
'base_url': '%s/api/v2/' % API_HOST,
'serializer': serialize.Serializer(
default='json-drf',
serializers=[
serialize.JsonSerializer(),
DrfJsonSerializer(),
],
),
'session': session,
}
if USER and PASS:
log.debug(
'Using slumber v2 with user %s, pointed at %s',
USER,
API_HOST,
)
session.auth = (USER, PASS)
else:
log.warning('SLUMBER_USERNAME/PASSWORD settings are not set')
return API(**api_config)
def setup_api():
session = Session()
session.headers.update({'Host': PRODUCTION_DOMAIN})
api_config = {
'base_url': '%s/api/v1/' % API_HOST,
'session': session,
}
if USER and PASS:
log.debug("Using slumber with user %s, pointed at %s", USER, API_HOST)
session.auth = (USER, PASS)
else:
log.warning("SLUMBER_USERNAME/PASSWORD settings are not set")
return API(**api_config)
def setup_api():
session = requests.Session()
if settings.SLUMBER_API_HOST.startswith('https'):
# Only use the HostHeaderSSLAdapter for HTTPS connections
adapter_class = host_header_ssl.HostHeaderSSLAdapter
else:
adapter_class = requests.adapters.HTTPAdapter
# Define a retry mechanism trying to attempt to not fail in the first
# error. Builders hit this issue frequently because the webs are high loaded
retry = Retry(
total=3,
read=3,
connect=3,
status=3,
backoff_factor=0.5, # 0.5, 1, 2 seconds
method_whitelist=('GET', 'PUT', 'PATCH', 'POST'),
status_forcelist=(408, 413, 429, 500, 502, 503, 504),
)
session.mount(
settings.SLUMBER_API_HOST,
adapter_class(max_retries=retry),
)
session.headers.update({'Host': settings.PRODUCTION_DOMAIN})
api_config = {
'base_url':
'%s/api/v2/' % settings.SLUMBER_API_HOST,
'serializer':
serialize.Serializer(
default='json-drf',
serializers=[
serialize.JsonSerializer(),
DrfJsonSerializer(),
],
),
'session':
session,
}
if settings.SLUMBER_USERNAME and settings.SLUMBER_PASSWORD:
log.debug(
'Using slumber v2 with user %s, pointed at %s',
settings.SLUMBER_USERNAME,
settings.SLUMBER_API_HOST,
)
session.auth = (settings.SLUMBER_USERNAME, settings.SLUMBER_PASSWORD)
else:
log.warning('SLUMBER_USERNAME/PASSWORD settings are not set')
return API(**api_config)
def setup_api():
api_config = {
'base_url': '%s/api/v2/' % API_HOST,
'serializer': serialize.Serializer(
default='json-drf',
serializers=[
serialize.JsonSerializer(),
DrfJsonSerializer(),
]
)
}
if USER and PASS:
log.debug("Using slumber v2 with user %s, pointed at %s", USER, API_HOST)
api_config['auth'] = (USER, PASS)
else:
log.warning("SLUMBER_USERNAME/PASSWORD settings are not set")
return API(**api_config)
def setup_api():
session = Session()
session.headers.update({'Host': PRODUCTION_DOMAIN})
api_config = {
'base_url': '%s/api/v2/' % API_HOST,
'serializer': serialize.Serializer(
default='json-drf',
serializers=[
serialize.JsonSerializer(),
DrfJsonSerializer(),
]
),
'session': session,
}
if USER and PASS:
log.debug("Using slumber v2 with user %s, pointed at %s", USER, API_HOST)
session.auth = (USER, PASS)
else:
log.warning("SLUMBER_USERNAME/PASSWORD settings are not set")
return API(**api_config)
class TransifexPuller(object):
def __init__(self, target=None, email=None):
# Read config
config = read_config()
self.gh_token = config.get('GitHub', 'token')
tx_username = (
config.get('Transifex', 'username') or
os.environ.get('TRANSIFEX_USER') or
TX_USERNAME_DEFAULT)
tx_password = (
config.get('Transifex', 'password') or
os.environ.get('TRANSIFEX_PASSWORD'))
self.tx_num_retries = (
config.get('Transifex', 'num_retries') or
os.environ.get('TRANSIFEX_RETRIES'))
self.tx_org = (
config.get('Transifex', 'organization') or
os.environ.get('TRANSIFEX_ORGANIZATION') or
TX_ORG_DEFAULT)
self.gh_org = target or self.tx_org
# Connect to GitHub
self.github = github_login.login()
gh_user = wrap_gh_call(self.github.user)
if not gh_user.email and not email:
raise Exception(
'Email required to commit to github. Please provide one on '
'the command line or make the one of your github profile '
'public.'
)
self.gh_credentials = {
'name': gh_user.name or str(gh_user),
'email': gh_user.email or email,
}
# Connect to Transifex
self.tx_api = API(TX_URL, auth=(tx_username, tx_password))
@classmethod
def _load_po_dict(cls, po_file):
po_dict = {}
for po_entry in po_file:
if po_entry.msgstr:
key = u'\n'.join(x[0] for x in po_entry.occurrences)
key += u'\nmsgid "%s"' % po_entry.msgid
po_dict[key] = po_entry.msgstr
return po_dict
@classmethod
def _get_oca_project_info(cls, tx_project):
"""Retrieve project and branch on github from transifex project
information
"""
# use the project name since it's always formatted using the convention
# my-project (version)
# The usage of the name is required since it's hard to find a rule
# that covers the following cases when using the tx_slug
# OCA-l10n-xxx-8-0
# OCA-l10n-xxx-master
# OCA XXX_xxx-xxx
tx_name = tx_project['name']
regex = r'(?P<repo>[^\s]+) \((?P<branch>[^\s]+)\)'
match_object = re.search(regex, tx_name)
oca_project = match_object.group('repo')
oca_branch = match_object.group('branch').replace('-', '.')
return oca_project, oca_branch
def process_projects(self, projects=None):
"""For each project, get translations from transifex and push to
the corresponding project in gihub """
tx_projects = []
if projects:
# Check that provided projects are correct
for project_slug in projects:
try:
tx_project = wrap_tx_call(
self.tx_api.project(project_slug).get
)
tx_projects.append(tx_project)
except exceptions.HttpNotFoundError:
print "ERROR: Transifex project slug '%s' is invalid" % (
project_slug
)
else:
start = 1
temp_projects = []
print "Getting Transifex projects..."
while temp_projects or start == 1:
temp_projects = wrap_tx_call(
self.tx_api.projects().get, kwargs={'start': start},
)
start += len(temp_projects)
tx_projects += temp_projects
for tx_project in tx_projects:
if self.tx_org + '-' in tx_project['slug']:
self._process_project(tx_project)
def _process_project(self, tx_project):
print "Processing project '%s'..." % tx_project['name']
oca_project, oca_branch = self._get_oca_project_info(tx_project)
# get a reference to the github repo and branch where to push the
# the translations
gh_repo = wrap_gh_call(
self.github.repository, [self.gh_org, oca_project],
)
gh_branch = wrap_gh_call(gh_repo.branch, [oca_branch])
tree_data = []
# Check resources on Transifex
tx_project_api = self.tx_api.project(tx_project['slug'])
resources = wrap_tx_call(tx_project_api.resources().get)
for resource in resources:
print "Checking resource %s..." % resource['name']
tx_resource_api = tx_project_api.resource(resource['slug'])
stats = wrap_tx_call(tx_resource_api.stats().get)
for lang in stats.keys():
# Discard english (native language in Odoo) or empty langs
if lang == 'en' or not stats[lang]['translated_words']:
continue
cont = 0
tx_lang = False
while cont < self.tx_num_retries and not tx_lang:
# for some weird reason, sometimes Transifex fails to
# some requests, so this retry mechanism handles this
# problem
try:
tx_lang = wrap_tx_call(
tx_resource_api.translation(lang).get,
)
except (exceptions.HttpClientError,
exceptions.HttpServerError):
tx_lang = False
cont += 1
if tx_lang:
gh_i18n_path = os.path.join('/', resource['slug'], "i18n")
gh_file_path = os.path.join(gh_i18n_path, lang + '.po')
tx_po_file = polib.pofile(tx_lang['content'])
tx_po_dict = self._load_po_dict(tx_po_file)
gh_file = wrap_gh_call(
gh_repo.contents, [gh_file_path, gh_branch.name],
)
if gh_file:
try:
gh_po_file = polib.pofile(
gh_file.decoded.decode('utf-8'))
except IOError:
print "...ERROR reading %s" % gh_file_path
continue
gh_po_dict = self._load_po_dict(gh_po_file)
unmatched_items = (set(gh_po_dict.items()) ^
set(tx_po_dict.items()))
if not unmatched_items:
print "...no change in %s" % gh_file_path
continue
print '..replacing %s' % gh_file_path
new_file_blob = wrap_gh_call(
gh_repo.create_blob,
args=[tx_lang['content']],
kwargs={'encoding': 'utf-8'},
)
tree_data.append({
'path': gh_file_path[1:],
'mode': '100644',
'type': 'blob',
'sha': new_file_blob,
})
else:
print "ERROR: fetching lang '%s'" % lang
if tree_data:
tree_sha = gh_branch.commit.commit.tree.sha
tree = wrap_gh_call(
gh_repo.create_tree, [tree_data, tree_sha],
)
message = '[i18n] New Translations! We need your help ... \n\n to translate more: https://www.it-projects.info/page/translate'
if tree:
commit = wrap_gh_call(
gh_repo.create_commit,
kwargs={
'message': message,
'tree': tree.sha,
'parents': [gh_branch.commit.sha],
'author': self.gh_credentials,
'committer': self.gh_credentials,
},
)
print "Pushing to GitHub"
wrap_gh_call(
gh_repo.ref('heads/{}'.format(gh_branch.name)).update,
args=[commit.sha],
)
class TransifexPuller(object):
def __init__(self, target=None, email=None):
# Read config
config = read_config()
self.gh_token = config.get('GitHub', 'token')
tx_username = (
config.get('Transifex', 'username') or
os.environ.get('TRANSIFEX_USER') or
TX_USERNAME_DEFAULT)
tx_password = (
config.get('Transifex', 'password') or
os.environ.get('TRANSIFEX_PASSWORD'))
self.tx_num_retries = (
config.get('Transifex', 'num_retries') or
os.environ.get('TRANSIFEX_RETRIES'))
self.tx_org = (
config.get('Transifex', 'organization') or
os.environ.get('TRANSIFEX_ORGANIZATION') or
TX_ORG_DEFAULT)
self.gh_org = target or self.tx_org
# Connect to GitHub
self.github = github_login.login()
gh_user = self.github.user()
if not gh_user.email and not email:
raise Exception(
'Email required to commit to github. Please provide one on '
'the command line or make the one of your github profile '
'public.')
self.gh_credentials = {'name': gh_user.name or str(gh_user),
'email': gh_user.email or email}
# Connect to Transifex
self.tx_api = API(TX_URL, auth=(tx_username, tx_password))
@classmethod
def _load_po_dict(cls, po_file):
po_dict = {}
for po_entry in po_file:
if po_entry.msgstr:
key = u'\n'.join(x[0] for x in po_entry.occurrences)
key += u'\nmsgid "%s"' % po_entry.msgid
po_dict[key] = po_entry.msgstr
return po_dict
@classmethod
def _get_oca_project_info(cls, tx_project):
"""Retrieve project and branch on github from transifex project
information
"""
# use the project name since it's always formatted using the convention
# my-project (version)
# The usage of the name is required since it's hard to find a rule
# that covers the following cases when using the tx_slug
# OCA-l10n-xxx-8-0
# OCA-l10n-xxx-master
# OCA XXX_xxx-xxx
tx_name = tx_project['name']
regex = r'(?P<repo>[^\s]+) \((?P<branch>[^\s]+)\)'
match_object = re.search(regex, tx_name)
oca_project = match_object.group('repo')
oca_branch = match_object.group('branch').replace('-', '.')
return oca_project, oca_branch
def process_projects(self, projects=None):
"""For each project, get translations from transifex and push to
the corresponding project in gihub """
tx_projects = []
if projects:
# Check that provided projects are correct
for project_slug in projects:
try:
tx_project = self.tx_api.project(project_slug).get()
tx_projects.append(tx_project)
except (KeyboardInterrupt, SystemExit):
raise
except:
print "ERROR: Transifex project slug %s is invalid" % (
project_slug)
return
else:
start = 1
temp_projects = []
print "Getting Transifex projects..."
while temp_projects or start == 1:
temp_projects = self.tx_api.projects().get(start=start)
start += len(temp_projects)
tx_projects += temp_projects
for tx_project in tx_projects:
if self.tx_org + '-' in tx_project['slug']:
self._process_project(tx_project)
def _process_project(self, tx_project):
print "Processing project '%s'..." % tx_project['name']
oca_project, oca_branch = self._get_oca_project_info(tx_project)
# get a reference to the github repo and branch where to push the
# the translations
gh_repo = self.github.repository(self.gh_org, oca_project)
gh_branch = gh_repo.branch(oca_branch)
tree_data = []
# Check resources on Transifex
tx_project_api = self.tx_api.project(tx_project['slug'])
resources = tx_project_api.resources().get()
for resource in resources:
print "Checking resource %s..." % resource['name']
if resource['slug'] != 'date_range':
continue
tx_resource_api = tx_project_api.resource(resource['slug'])
stats = tx_resource_api.stats().get()
for lang in stats.keys():
# Discard english (native language in Odoo) or empty langs
if lang == 'en' or not stats[lang]['translated_words']:
continue
cont = 0
tx_lang = False
while cont < self.tx_num_retries and not tx_lang:
# for some weird reason, sometimes Transifex fails to
# some requests, so this retry mechanism handles this
# problem
try:
tx_lang = tx_resource_api.translation(lang).get()
except (KeyboardInterrupt, SystemExit):
raise
except exceptions.HttpClientError:
tx_lang = False
cont += 1
if tx_lang:
gh_i18n_path = os.path.join('/', resource['slug'], "i18n")
gh_file_path = os.path.join(gh_i18n_path, lang + '.po')
try:
tx_po_file = polib.pofile(tx_lang['content'])
tx_po_dict = self._load_po_dict(tx_po_file)
gh_file = gh_repo.contents(
gh_file_path, gh_branch.name)
if gh_file:
gh_po_file = polib.pofile(
gh_file.decoded.decode('utf-8'))
gh_po_dict = self._load_po_dict(gh_po_file)
unmatched_items = (set(gh_po_dict.items()) ^
set(tx_po_dict.items()))
if not unmatched_items:
print "...no change in %s" % gh_file_path
continue
print '..replacing %s' % gh_file_path
new_file_blob = gh_repo.create_blob(
tx_lang['content'], encoding='utf-8')
tree_data.append({
'path': gh_file_path[1:],
'mode': '100644',
'type': 'blob',
'sha': new_file_blob})
except (KeyboardInterrupt, SystemExit):
raise
except:
print "ERROR: processing lang '%s'" % lang
else:
print "ERROR: fetching lang '%s'" % lang
# Wait a minute before the next file to avoid reaching Transifex
# API limitations
# TODO: Request the API to get the date for the next request
# http://docs.rackspace.com/loadbalancers/api/v1.0/clb-devguide/\
# content/Determining_Limits_Programmatically-d1e1039.html
print "Sleeping 70 seconds..."
time.sleep(70)
if tree_data:
tree_sha = gh_branch.commit.commit.tree.sha
tree = gh_repo.create_tree(tree_data, tree_sha)
message = 'OCA Transbot updated translations from Transifex'
print "message", message
commit = gh_repo.create_commit(
message=message, tree=tree.sha, parents=[gh_branch.commit.sha],
author=self.gh_credentials, committer=self.gh_credentials)
print "git pushing"
gh_repo.ref('heads/{}'.format(gh_branch.name)).update(commit.sha)
# Wait 5 minutes before the next project to avoid reaching Transifex
# API limitations
# TODO: Request the API to get the date for the next request
# http://docs.rackspace.com/loadbalancers/api/v1.0/clb-devguide/\
# content/Determining_Limits_Programmatically-d1e1039.html
print "Sleeping 5 minutes..."
time.sleep(300)
def main(argv=None):
"""
Export translation files and push them to Transifex
The transifex password should be encrypted in .travis.yml
If not, export exits early.
"""
if argv is None:
argv = sys.argv
transifex_user = os.environ.get("TRANSIFEX_USER")
transifex_password = os.environ.get("TRANSIFEX_PASSWORD")
if not transifex_user:
print(
yellow_light("WARNING! Transifex user not defined- "
"exiting early."))
return 1
if not transifex_password:
print(
yellow_light("WARNING! Transifex password not recognized- "
"exiting early."))
return 1
travis_home = os.environ.get("HOME", "~/")
travis_build_dir = os.environ.get("TRAVIS_BUILD_DIR", "../..")
travis_repo_slug = os.environ.get("TRAVIS_REPO_SLUG")
travis_repo_owner = travis_repo_slug.split("/")[0]
travis_repo_shortname = travis_repo_slug.split("/")[1]
odoo_unittest = False
odoo_exclude = os.environ.get("EXCLUDE")
odoo_include = os.environ.get("INCLUDE")
install_options = os.environ.get("INSTALL_OPTIONS", "").split()
odoo_version = os.environ.get("VERSION")
if not odoo_version:
# For backward compatibility, take version from parameter
# if it's not globally set
odoo_version = sys.argv[1]
print(
yellow_light("WARNING: no env variable set for VERSION. "
"Using '%s'" % odoo_version))
default_project_slug = "%s-%s" % (travis_repo_slug.replace(
'/', '-'), odoo_version.replace('.', '-'))
transifex_project_slug = os.environ.get("TRANSIFEX_PROJECT_SLUG",
default_project_slug)
transifex_project_name = "%s (%s)" % (travis_repo_shortname, odoo_version)
transifex_organization = os.environ.get("TRANSIFEX_ORGANIZATION",
travis_repo_owner)
transifex_fill_up_resources = os.environ.get("TRANSIFEX_FILL_UP_RESOURCES",
"True")
transifex_team = os.environ.get("TRANSIFEX_TEAM", "23907")
repository_url = "https://github.com/%s" % travis_repo_slug
odoo_full = os.environ.get("ODOO_REPO", "odoo/odoo")
server_path = get_server_path(odoo_full, odoo_version, travis_home)
addons_path = get_addons_path(travis_home, travis_build_dir, server_path)
addons_list = get_addons_to_check(travis_build_dir, odoo_include,
odoo_exclude)
addons = ','.join(addons_list)
print("\nWorking in %s" % travis_build_dir)
print("Using repo %s and addons path %s" % (odoo_full, addons_path))
if not addons:
print(yellow_light("WARNING! Nothing to translate- exiting early."))
return 0
# Create Transifex project if it doesn't exist
print()
print(yellow("Creating Transifex project if it doesn't exist"))
auth = (transifex_user, transifex_password)
api_url = "https://www.transifex.com/api/2/"
api = API(api_url, auth=auth)
project_data = {
"slug": transifex_project_slug,
"name": transifex_project_name,
"source_language_code": "en",
"description": transifex_project_name,
"repository_url": repository_url,
"organization": transifex_organization,
"license": "permissive_open_source",
"fill_up_resources": transifex_fill_up_resources,
"team": transifex_team,
}
try:
api.project(transifex_project_slug).get()
print('This Transifex project already exists.')
except exceptions.HttpClientError:
try:
api.projects.post(project_data)
print('Transifex project has been successfully created.')
except exceptions.HttpClientError:
print('Transifex organization: %s' % transifex_organization)
print('Transifex username: %s' % transifex_user)
print('Transifex project slug: %s' % transifex_project_slug)
print(
red('Error: Authentication failed. Please verify that '
'Transifex organization, user and password are '
'correct. You can change these variables in your '
'.travis.yml file.'))
raise
print("\nModules to translate: %s" % addons)
# Install the modules on the database
database = "openerp_i18n"
setup_server(database, odoo_unittest, addons, server_path, addons_path,
install_options, addons_list)
# Initialize Transifex project
print()
print(yellow('Initializing Transifex project'))
init_args = [
'--host=https://www.transifex.com',
'--user=%s' % transifex_user,
'--pass=%s' % transifex_password
]
commands.cmd_init(init_args, path_to_tx=None)
path_to_tx = utils.find_dot_tx()
connection_context = context_mapping[odoo_version]
with connection_context(server_path, addons_path, database) \
as odoo_context:
for module in addons_list:
print()
print(yellow("Downloading PO file for %s" % module))
source_filename = os.path.join(travis_build_dir, module, 'i18n',
module + ".pot")
# Create i18n/ directory if doesn't exist
if not os.path.exists(os.path.dirname(source_filename)):
os.makedirs(os.path.dirname(source_filename))
with open(source_filename, 'w') as f:
f.write(odoo_context.get_pot_contents(module))
print()
print(yellow("Linking PO file and Transifex resource"))
set_args = [
'-t', 'PO', '--auto-local', '-r',
'%s.%s' % (transifex_project_slug, module),
'%s/i18n/<lang>.po' % module, '--source-lang', 'en',
'--source-file', source_filename, '--execute'
]
commands.cmd_set(set_args, path_to_tx)
print()
print(yellow('Pushing translation files to Transifex'))
push_args = ['-s', '-t', '--skip']
commands.cmd_push(push_args, path_to_tx)
return 0
def main(argv=None):
"""
Export translation files and push them to Transifex
The transifex password should be encrypted in .travis.yml
If not, export exits early.
"""
if argv is None:
argv = sys.argv
transifex_user = os.environ.get("TRANSIFEX_USER")
transifex_password = os.environ.get("TRANSIFEX_PASSWORD")
if not transifex_user:
print(yellow_light("WARNING! Transifex user not defined- "
"exiting early."))
return 1
if not transifex_password:
print(yellow_light("WARNING! Transifex password not recognized- "
"exiting early."))
return 1
travis_home = os.environ.get("HOME", "~/")
travis_dependencies_dir = os.path.join(travis_home, 'dependencies')
travis_build_dir = os.environ.get("TRAVIS_BUILD_DIR", "../..")
travis_repo_slug = os.environ.get("TRAVIS_REPO_SLUG")
travis_repo_owner = travis_repo_slug.split("/")[0]
travis_repo_shortname = travis_repo_slug.split("/")[1]
odoo_unittest = False
odoo_exclude = os.environ.get("EXCLUDE")
odoo_include = os.environ.get("INCLUDE")
install_options = os.environ.get("INSTALL_OPTIONS", "").split()
odoo_version = os.environ.get("VERSION")
if not odoo_version:
# For backward compatibility, take version from parameter
# if it's not globally set
odoo_version = argv[1]
print(yellow_light("WARNING: no env variable set for VERSION. "
"Using '%s'" % odoo_version))
default_project_slug = "%s-%s" % (travis_repo_slug.replace('/', '-'),
odoo_version.replace('.', '-'))
transifex_project_slug = os.environ.get("TRANSIFEX_PROJECT_SLUG",
default_project_slug)
transifex_project_name = "%s (%s)" % (travis_repo_shortname, odoo_version)
transifex_organization = os.environ.get("TRANSIFEX_ORGANIZATION",
travis_repo_owner)
transifex_fill_up_resources = os.environ.get(
"TRANSIFEX_FILL_UP_RESOURCES", "True"
)
transifex_team = os.environ.get(
"TRANSIFEX_TEAM", "23907"
)
repository_url = "https://github.com/%s" % travis_repo_slug
odoo_full = os.environ.get("ODOO_REPO", "odoo/odoo")
server_path = get_server_path(odoo_full, odoo_version, travis_home)
addons_path = get_addons_path(travis_dependencies_dir,
travis_build_dir,
server_path)
addons_list = get_addons_to_check(travis_build_dir, odoo_include,
odoo_exclude)
addons = ','.join(addons_list)
create_server_conf({'addons_path': addons_path}, odoo_version)
print("\nWorking in %s" % travis_build_dir)
print("Using repo %s and addons path %s" % (odoo_full, addons_path))
if not addons:
print(yellow_light("WARNING! Nothing to translate- exiting early."))
return 0
# Create Transifex project if it doesn't exist
print()
print(yellow("Creating Transifex project if it doesn't exist"))
auth = (transifex_user, transifex_password)
api_url = "https://www.transifex.com/api/2/"
api = API(api_url, auth=auth)
project_data = {"slug": transifex_project_slug,
"name": transifex_project_name,
"source_language_code": "en",
"description": transifex_project_name,
"repository_url": repository_url,
"organization": transifex_organization,
"license": "permissive_open_source",
"fill_up_resources": transifex_fill_up_resources,
"team": transifex_team,
}
try:
api.project(transifex_project_slug).get()
print('This Transifex project already exists.')
except exceptions.HttpClientError:
try:
api.projects.post(project_data)
print('Transifex project has been successfully created.')
except exceptions.HttpClientError:
print('Transifex organization: %s' % transifex_organization)
print('Transifex username: %s' % transifex_user)
print('Transifex project slug: %s' % transifex_project_slug)
print(red('Error: Authentication failed. Please verify that '
'Transifex organization, user and password are '
'correct. You can change these variables in your '
'.travis.yml file.'))
raise
print("\nModules to translate: %s" % addons)
# Install the modules on the database
database = "openerp_i18n"
setup_server(database, odoo_unittest, addons, server_path, addons_path,
install_options, addons_list)
# Initialize Transifex project
print()
print(yellow('Initializing Transifex project'))
init_args = ['--host=https://www.transifex.com',
'--user=%s' % transifex_user,
'--pass=%s' % transifex_password]
commands.cmd_init(init_args, path_to_tx=None)
path_to_tx = utils.find_dot_tx()
connection_context = context_mapping[odoo_version]
with connection_context(server_path, addons_path, database) \
as odoo_context:
for module in addons_list:
print()
print(yellow("Downloading POT file for %s" % module))
source_filename = os.path.join(travis_build_dir, module, 'i18n',
module + ".pot")
# Create i18n/ directory if doesn't exist
if not os.path.exists(os.path.dirname(source_filename)):
os.makedirs(os.path.dirname(source_filename))
with open(source_filename, 'w') as f:
f.write(odoo_context.get_pot_contents(module))
print()
print(yellow("Linking POT file and Transifex resource"))
set_args = ['-t', 'PO',
'--auto-local',
'-r', '%s.%s' % (transifex_project_slug, module),
'%s/i18n/<lang>.po' % module,
'--source-lang', 'en',
'--source-file', source_filename,
'--execute']
commands.cmd_set(set_args, path_to_tx)
print()
print(yellow('Pushing translation files to Transifex'))
push_args = ['-s', '-t', '--skip']
commands.cmd_push(push_args, path_to_tx)
return 0
def main(argv=None):
"""
Export translation files and push them to Transifex
The transifex password should be encrypted in .travis.yml
If not, export exits early.
"""
if argv is None:
argv = sys.argv
transifex_user = os.environ.get("TRANSIFEX_USER", "*****@*****.**")
transifex_password = os.environ.get("TRANSIFEX_PASSWORD")
if not transifex_user:
print(
yellow_light("WARNING! Transifex user not defined- "
"exiting early."))
return 1
if not transifex_password:
print(
yellow_light("WARNING! Transifex password not recognized- "
"exiting early."))
return 1
travis_home = os.environ.get("HOME", "~/")
travis_build_dir = os.environ.get("TRAVIS_BUILD_DIR", "../..")
travis_repo_slug = os.environ.get("TRAVIS_REPO_SLUG")
travis_repo_owner = travis_repo_slug.split("/")[0]
travis_repo_shortname = travis_repo_slug.split("/")[1]
odoo_unittest = False
odoo_exclude = os.environ.get("EXCLUDE")
odoo_include = os.environ.get("INCLUDE")
install_options = os.environ.get("INSTALL_OPTIONS", "").split()
odoo_version = os.environ.get("VERSION")
langs = parse_list(os.environ.get("LANG_ALLOWED", ""))
if not odoo_version:
# For backward compatibility, take version from parameter
# if it's not globally set
odoo_version = argv[1]
print(
yellow_light("WARNING: no env variable set for VERSION. "
"Using '%s'" % odoo_version))
default_project_slug = "%s-%s" % (travis_repo_slug.replace(
'/', '-'), odoo_version.replace('.', '-'))
transifex_project_slug = os.environ.get("TRANSIFEX_PROJECT_SLUG",
default_project_slug)
transifex_project_name = "%s (%s)" % (travis_repo_shortname, odoo_version)
transifex_organization = os.environ.get("TRANSIFEX_ORGANIZATION",
travis_repo_owner)
transifex_fill_up_resources = os.environ.get("TRANSIFEX_FILL_UP_RESOURCES",
"True")
transifex_team = os.environ.get("TRANSIFEX_TEAM", "45447")
repository_url = "https://github.com/%s" % travis_repo_slug
odoo_full = os.environ.get("ODOO_REPO", "odoo/odoo")
server_path = get_server_path(odoo_full, odoo_version, travis_home)
addons_path = get_addons_path(travis_home, travis_build_dir, server_path)
addons_list = get_addons_to_check(travis_build_dir, odoo_include,
odoo_exclude)
addons_path_list = parse_list(addons_path)
all_depends = get_depends(addons_path_list, addons_list)
main_modules = set(os.listdir(travis_build_dir))
main_depends = main_modules & all_depends
addons_list = list(main_depends)
addons = ','.join(addons_list)
create_server_conf({'addons_path': addons_path}, odoo_version)
print("\nWorking in %s" % travis_build_dir)
print("Using repo %s and addons path %s" % (odoo_full, addons_path))
if not addons:
print(yellow_light("WARNING! Nothing to translate- exiting early."))
return 0
# Create Transifex project if it doesn't exist
print()
print(yellow("Creating Transifex project if it doesn't exist"))
auth = (transifex_user, transifex_password)
api_url = "https://www.transifex.com/api/2/"
api = API(api_url, auth=auth)
project_data = {
"slug": transifex_project_slug,
"name": transifex_project_name,
"source_language_code": "en",
"description": transifex_project_name,
"repository_url": repository_url,
"organization": transifex_organization,
"license": "permissive_open_source",
"fill_up_resources": transifex_fill_up_resources,
"team": transifex_team,
}
try:
api.project(transifex_project_slug).get()
print('This Transifex project already exists.')
except exceptions.HttpClientError:
try:
api.projects.post(project_data)
print('Transifex project has been successfully created.')
except exceptions.HttpClientError:
print('Transifex organization: %s' % transifex_organization)
print('Transifex username: %s' % transifex_user)
print('Transifex project slug: %s' % transifex_project_slug)
print(
red('Error: Authentication failed. Please verify that '
'Transifex organization, user and password are '
'correct. You can change these variables in your '
'.travis.yml file.'))
raise
print("\nModules to translate: %s" % addons)
# Install the modules on the database
database = "openerp_test"
script_name = get_server_script(odoo_version)
setup_server(database, odoo_unittest, addons, server_path, script_name,
addons_path, install_options, addons_list)
# Initialize Transifex project
print()
print(yellow('Initializing Transifex project'))
init_args = [
'--host=https://www.transifex.com',
'--user=%s' % transifex_user,
'--pass=%s' % transifex_password
]
commands.cmd_init(init_args, path_to_tx=None)
path_to_tx = utils.find_dot_tx()
# Use by default version 10 connection context
connection_context = context_mapping.get(odoo_version, Odoo10Context)
with connection_context(server_path, addons_path, database) \
as odoo_context:
for module in addons_list:
print()
print(yellow("Obtaining POT file for %s" % module))
i18n_folder = os.path.join(travis_build_dir, module, 'i18n')
source_filename = os.path.join(i18n_folder, module + ".pot")
# Create i18n/ directory if doesn't exist
if not os.path.exists(os.path.dirname(source_filename)):
os.makedirs(os.path.dirname(source_filename))
with open(source_filename, 'w') as f:
f.write(odoo_context.get_pot_contents(module))
# Put the correct timestamp for letting known tx client which
# translations to update
for po_file_name in os.listdir(i18n_folder):
if not po_file_name.endswith('.po'):
continue
if langs and os.path.splitext(po_file_name)[0] not in langs:
# Limit just allowed languages if is defined
os.remove(os.path.join(i18n_folder, po_file_name))
continue
po_file_name = os.path.join(i18n_folder, po_file_name)
command = [
'git', 'log', '--pretty=format:%cd', '-n1', '--date=raw',
po_file_name
]
timestamp = float(subprocess.check_output(command).split()[0])
# This converts to UTC the timestamp
timestamp = time.mktime(time.gmtime(timestamp))
os.utime(po_file_name, (timestamp, timestamp))
print()
print(yellow("Linking POT file and Transifex resource"))
set_args = [
'-t', 'PO', '--auto-local', '-r',
'%s.%s' % (transifex_project_slug, module),
'%s/i18n/<lang>.po' % module, '--source-lang', 'en',
'--source-file', source_filename, '--execute'
]
commands.cmd_set(set_args, path_to_tx)
print()
print(yellow('Pushing translation files to Transifex'))
push_args = ['-s', '-t', '--skip']
commands.cmd_push(push_args, path_to_tx)
return 0
# Usage example for Python
# To use this example install slumber library (pip install slumber==0.6.0)
import sys
from requests import ConnectionError
from slumber import API
from slumber.exceptions import HttpClientError
SERVICE_URL = 'http://auth.svoe.tv/login/'
SERVICE_API_URL = 'http://auth.svoe.tv/api/v1/'
APP_ID = 2
APP_PASS = '******'
if __name__ == '__main__':
api = API(SERVICE_API_URL, auth=(APP_ID, APP_PASS))
# Try get active session
try:
session = api.sessions.get()['objects'][0]
user_id = session['user_id']
print "Check for user with session id - %s" % session['session_key']
except (IndexError, KeyError):
print "No active session (please, login first at %s)" % SERVICE_URL
sys.exit()
except ConnectionError:
print "Service %s unavailable" % SERVICE_API_URL
sys.exit()
# 1. Get all permissions for authenticated user
print "1. User permissions: \n", api.users(user_id).permissions.get()
class TransifexPuller(object):
def __init__(self, target=None, email=None):
# Read config
config = read_config()
self.gh_token = config.get('GitHub', 'token')
tx_username = (
config.get('Transifex', 'username') or
os.environ.get('TRANSIFEX_USER') or
TX_USERNAME_DEFAULT)
tx_password = (
config.get('Transifex', 'password') or
os.environ.get('TRANSIFEX_PASSWORD'))
self.tx_num_retries = (
config.get('Transifex', 'num_retries') or
os.environ.get('TRANSIFEX_RETRIES'))
self.tx_org = (
config.get('Transifex', 'organization') or
os.environ.get('TRANSIFEX_ORGANIZATION') or
TX_ORG_DEFAULT)
self.gh_org = target or self.tx_org
# Connect to GitHub
self.github = github_login.login()
gh_user = wrap_gh_call(self.github.user)
if not gh_user.email and not email:
raise Exception(
'Email required to commit to github. Please provide one on '
'the command line or make the one of your github profile '
'public.'
)
self.gh_credentials = {
'name': gh_user.name or str(gh_user),
'email': gh_user.email or email,
}
# Connect to Transifex
self.tx_api = API(TX_URL, auth=(tx_username, tx_password))
@classmethod
def _load_po_dict(cls, po_file):
po_dict = {}
for po_entry in po_file:
if po_entry.msgstr:
key = u'\n'.join(x[0] for x in po_entry.occurrences)
key += u'\nmsgid "%s"' % po_entry.msgid
po_dict[key] = po_entry.msgstr
return po_dict
@classmethod
def _get_oca_project_info(cls, tx_project):
"""Retrieve project and branch on github from transifex project
information
"""
# use the project name since it's always formatted using the convention
# my-project (version)
# The usage of the name is required since it's hard to find a rule
# that covers the following cases when using the tx_slug
# OCA-l10n-xxx-8-0
# OCA-l10n-xxx-master
# OCA XXX_xxx-xxx
tx_name = tx_project['name']
regex = r'(?P<repo>[^\s]+) \((?P<branch>[^\s]+)\)'
match_object = re.search(regex, tx_name)
oca_project = match_object.group('repo')
oca_branch = match_object.group('branch').replace('-', '.')
return oca_project, oca_branch
def process_projects(self, projects=None):
"""For each project, get translations from transifex and push to
the corresponding project in gihub """
tx_projects = []
if projects:
# Check that provided projects are correct
for project_slug in projects:
try:
tx_project = wrap_tx_call(
self.tx_api.project(project_slug).get
)
tx_projects.append(tx_project)
except exceptions.HttpNotFoundError:
print "ERROR: Transifex project slug '%s' is invalid" % (
project_slug
)
else:
start = 1
temp_projects = []
print "Getting Transifex projects..."
while temp_projects or start == 1:
temp_projects = wrap_tx_call(
self.tx_api.projects().get, kwargs={'start': start},
)
start += len(temp_projects)
tx_projects += temp_projects
for tx_project in tx_projects:
if self.tx_org + '-' in tx_project['slug']:
self._process_project(tx_project)
def _process_project(self, tx_project):
print "Processing project '%s'..." % tx_project['name']
oca_project, oca_branch = self._get_oca_project_info(tx_project)
# get a reference to the github repo and branch where to push the
# the translations
gh_repo = wrap_gh_call(
self.github.repository, [self.gh_org, oca_project],
)
gh_branch = wrap_gh_call(gh_repo.branch, [oca_branch])
tree_data = []
# Check resources on Transifex
tx_project_api = self.tx_api.project(tx_project['slug'])
resources = wrap_tx_call(tx_project_api.resources().get)
for resource in resources:
print "Checking resource %s..." % resource['name']
tx_resource_api = tx_project_api.resource(resource['slug'])
stats = wrap_tx_call(tx_resource_api.stats().get)
for lang in stats.keys():
# Discard english (native language in Odoo) or empty langs
if lang == 'en' or not stats[lang]['translated_words']:
continue
cont = 0
tx_lang = False
while cont < self.tx_num_retries and not tx_lang:
# for some weird reason, sometimes Transifex fails to
# some requests, so this retry mechanism handles this
# problem
try:
tx_lang = wrap_tx_call(
tx_resource_api.translation(lang).get,
)
except (exceptions.HttpClientError,
exceptions.HttpServerError):
tx_lang = False
cont += 1
if tx_lang:
gh_i18n_path = os.path.join('/', resource['slug'], "i18n")
gh_file_path = os.path.join(gh_i18n_path, lang + '.po')
tx_po_file = polib.pofile(tx_lang['content'])
tx_po_dict = self._load_po_dict(tx_po_file)
gh_file = wrap_gh_call(
gh_repo.contents, [gh_file_path, gh_branch.name],
)
if gh_file:
try:
gh_po_file = polib.pofile(
gh_file.decoded.decode('utf-8'))
except IOError:
print "...ERROR reading %s" % gh_file_path
continue
gh_po_dict = self._load_po_dict(gh_po_file)
unmatched_items = (set(gh_po_dict.items()) ^
set(tx_po_dict.items()))
if not unmatched_items:
print "...no change in %s" % gh_file_path
continue
print '..replacing %s' % gh_file_path
new_file_blob = wrap_gh_call(
gh_repo.create_blob,
args=[tx_lang['content']],
kwargs={'encoding': 'utf-8'},
)
tree_data.append({
'path': gh_file_path[1:],
'mode': '100644',
'type': 'blob',
'sha': new_file_blob,
})
else:
print "ERROR: fetching lang '%s'" % lang
if tree_data:
tree_sha = gh_branch.commit.commit.tree.sha
tree = wrap_gh_call(
gh_repo.create_tree, [tree_data, tree_sha],
)
message = 'OCA Transbot updated translations from Transifex'
if tree:
commit = wrap_gh_call(
gh_repo.create_commit,
kwargs={
'message': message,
'tree': tree.sha,
'parents': [gh_branch.commit.sha],
'author': self.gh_credentials,
'committer': self.gh_credentials,
},
)
print "Pushing to GitHub"
wrap_gh_call(
gh_repo.ref('heads/{}'.format(gh_branch.name)).update,
args=[commit.sha],
)
