class TransifexPuller(object):
def __init__(self, target=None, email=None):
# Read config
config = read_config()
self.gh_token = config.get('GitHub', 'token')
tx_username = (
config.get('Transifex', 'username') or
os.environ.get('TRANSIFEX_USER') or
TX_USERNAME_DEFAULT)
tx_password = (
config.get('Transifex', 'password') or
os.environ.get('TRANSIFEX_PASSWORD'))
self.tx_num_retries = (
config.get('Transifex', 'num_retries') or
os.environ.get('TRANSIFEX_RETRIES'))
self.tx_org = (
config.get('Transifex', 'organization') or
os.environ.get('TRANSIFEX_ORGANIZATION') or
TX_ORG_DEFAULT)
self.gh_org = target or self.tx_org
# Connect to GitHub
self.github = github_login.login()
gh_user = self.github.user()
if not gh_user.email and not email:
raise Exception(
'Email required to commit to github. Please provide one on '
'the command line or make the one of your github profile '
'public.')
self.gh_credentials = {'name': gh_user.name or str(gh_user),
'email': gh_user.email or email}
# Connect to Transifex
self.tx_api = API(TX_URL, auth=(tx_username, tx_password))
@classmethod
def _load_po_dict(cls, po_file):
po_dict = {}
for po_entry in po_file:
if po_entry.msgstr:
key = u'\n'.join(x[0] for x in po_entry.occurrences)
key += u'\nmsgid "%s"' % po_entry.msgid
po_dict[key] = po_entry.msgstr
return po_dict
@classmethod
def _get_oca_project_info(cls, tx_project):
"""Retrieve project and branch on github from transifex project
information
"""
# use the project name since it's always formatted using the convention
# my-project (version)
# The usage of the name is required since it's hard to find a rule
# that covers the following cases when using the tx_slug
# OCA-l10n-xxx-8-0
# OCA-l10n-xxx-master
# OCA XXX_xxx-xxx
tx_name = tx_project['name']
regex = r'(?P<repo>[^\s]+) \((?P<branch>[^\s]+)\)'
match_object = re.search(regex, tx_name)
oca_project = match_object.group('repo')
oca_branch = match_object.group('branch').replace('-', '.')
return oca_project, oca_branch
def process_projects(self, projects=None):
"""For each project, get translations from transifex and push to
the corresponding project in gihub """
tx_projects = []
if projects:
# Check that provided projects are correct
for project_slug in projects:
try:
tx_project = self.tx_api.project(project_slug).get()
tx_projects.append(tx_project)
except (KeyboardInterrupt, SystemExit):
raise
except:
print "ERROR: Transifex project slug %s is invalid" % (
project_slug)
return
else:
start = 1
temp_projects = []
print "Getting Transifex projects..."
while temp_projects or start == 1:
temp_projects = self.tx_api.projects().get(start=start)
start += len(temp_projects)
tx_projects += temp_projects
for tx_project in tx_projects:
if self.tx_org + '-' in tx_project['slug']:
self._process_project(tx_project)
def _process_project(self, tx_project):
print "Processing project '%s'..." % tx_project['name']
oca_project, oca_branch = self._get_oca_project_info(tx_project)
# get a reference to the github repo and branch where to push the
# the translations
gh_repo = self.github.repository(self.gh_org, oca_project)
gh_branch = gh_repo.branch(oca_branch)
tree_data = []
# Check resources on Transifex
tx_project_api = self.tx_api.project(tx_project['slug'])
resources = tx_project_api.resources().get()
for resource in resources:
print "Checking resource %s..." % resource['name']
if resource['slug'] != 'date_range':
continue
tx_resource_api = tx_project_api.resource(resource['slug'])
stats = tx_resource_api.stats().get()
for lang in stats.keys():
# Discard english (native language in Odoo) or empty langs
if lang == 'en' or not stats[lang]['translated_words']:
continue
cont = 0
tx_lang = False
while cont < self.tx_num_retries and not tx_lang:
# for some weird reason, sometimes Transifex fails to
# some requests, so this retry mechanism handles this
# problem
try:
tx_lang = tx_resource_api.translation(lang).get()
except (KeyboardInterrupt, SystemExit):
raise
except exceptions.HttpClientError:
tx_lang = False
cont += 1
if tx_lang:
gh_i18n_path = os.path.join('/', resource['slug'], "i18n")
gh_file_path = os.path.join(gh_i18n_path, lang + '.po')
try:
tx_po_file = polib.pofile(tx_lang['content'])
tx_po_dict = self._load_po_dict(tx_po_file)
gh_file = gh_repo.contents(
gh_file_path, gh_branch.name)
if gh_file:
gh_po_file = polib.pofile(
gh_file.decoded.decode('utf-8'))
gh_po_dict = self._load_po_dict(gh_po_file)
unmatched_items = (set(gh_po_dict.items()) ^
set(tx_po_dict.items()))
if not unmatched_items:
print "...no change in %s" % gh_file_path
continue
print '..replacing %s' % gh_file_path
new_file_blob = gh_repo.create_blob(
tx_lang['content'], encoding='utf-8')
tree_data.append({
'path': gh_file_path[1:],
'mode': '100644',
'type': 'blob',
'sha': new_file_blob})
except (KeyboardInterrupt, SystemExit):
raise
except:
print "ERROR: processing lang '%s'" % lang
else:
print "ERROR: fetching lang '%s'" % lang
# Wait a minute before the next file to avoid reaching Transifex
# API limitations
# TODO: Request the API to get the date for the next request
# http://docs.rackspace.com/loadbalancers/api/v1.0/clb-devguide/\
# content/Determining_Limits_Programmatically-d1e1039.html
print "Sleeping 70 seconds..."
time.sleep(70)
if tree_data:
tree_sha = gh_branch.commit.commit.tree.sha
tree = gh_repo.create_tree(tree_data, tree_sha)
message = 'OCA Transbot updated translations from Transifex'
print "message", message
commit = gh_repo.create_commit(
message=message, tree=tree.sha, parents=[gh_branch.commit.sha],
author=self.gh_credentials, committer=self.gh_credentials)
print "git pushing"
gh_repo.ref('heads/{}'.format(gh_branch.name)).update(commit.sha)
# Wait 5 minutes before the next project to avoid reaching Transifex
# API limitations
# TODO: Request the API to get the date for the next request
# http://docs.rackspace.com/loadbalancers/api/v1.0/clb-devguide/\
# content/Determining_Limits_Programmatically-d1e1039.html
print "Sleeping 5 minutes..."
time.sleep(300)
class TransifexPuller(object):
def __init__(self, target=None, email=None):
# Read config
config = read_config()
self.gh_token = config.get('GitHub', 'token')
tx_username = (
config.get('Transifex', 'username') or
os.environ.get('TRANSIFEX_USER') or
TX_USERNAME_DEFAULT)
tx_password = (
config.get('Transifex', 'password') or
os.environ.get('TRANSIFEX_PASSWORD'))
self.tx_num_retries = (
config.get('Transifex', 'num_retries') or
os.environ.get('TRANSIFEX_RETRIES'))
self.tx_org = (
config.get('Transifex', 'organization') or
os.environ.get('TRANSIFEX_ORGANIZATION') or
TX_ORG_DEFAULT)
self.gh_org = target or self.tx_org
# Connect to GitHub
self.github = github_login.login()
gh_user = wrap_gh_call(self.github.user)
if not gh_user.email and not email:
raise Exception(
'Email required to commit to github. Please provide one on '
'the command line or make the one of your github profile '
'public.'
)
self.gh_credentials = {
'name': gh_user.name or str(gh_user),
'email': gh_user.email or email,
}
# Connect to Transifex
self.tx_api = API(TX_URL, auth=(tx_username, tx_password))
@classmethod
def _load_po_dict(cls, po_file):
po_dict = {}
for po_entry in po_file:
if po_entry.msgstr:
key = u'\n'.join(x[0] for x in po_entry.occurrences)
key += u'\nmsgid "%s"' % po_entry.msgid
po_dict[key] = po_entry.msgstr
return po_dict
@classmethod
def _get_oca_project_info(cls, tx_project):
"""Retrieve project and branch on github from transifex project
information
"""
# use the project name since it's always formatted using the convention
# my-project (version)
# The usage of the name is required since it's hard to find a rule
# that covers the following cases when using the tx_slug
# OCA-l10n-xxx-8-0
# OCA-l10n-xxx-master
# OCA XXX_xxx-xxx
tx_name = tx_project['name']
regex = r'(?P<repo>[^\s]+) \((?P<branch>[^\s]+)\)'
match_object = re.search(regex, tx_name)
oca_project = match_object.group('repo')
oca_branch = match_object.group('branch').replace('-', '.')
return oca_project, oca_branch
def process_projects(self, projects=None):
"""For each project, get translations from transifex and push to
the corresponding project in gihub """
tx_projects = []
if projects:
# Check that provided projects are correct
for project_slug in projects:
try:
tx_project = wrap_tx_call(
self.tx_api.project(project_slug).get
)
tx_projects.append(tx_project)
except exceptions.HttpNotFoundError:
print "ERROR: Transifex project slug '%s' is invalid" % (
project_slug
)
else:
start = 1
temp_projects = []
print "Getting Transifex projects..."
while temp_projects or start == 1:
temp_projects = wrap_tx_call(
self.tx_api.projects().get, kwargs={'start': start},
)
start += len(temp_projects)
tx_projects += temp_projects
for tx_project in tx_projects:
if self.tx_org + '-' in tx_project['slug']:
self._process_project(tx_project)
def _process_project(self, tx_project):
print "Processing project '%s'..." % tx_project['name']
oca_project, oca_branch = self._get_oca_project_info(tx_project)
# get a reference to the github repo and branch where to push the
# the translations
gh_repo = wrap_gh_call(
self.github.repository, [self.gh_org, oca_project],
)
gh_branch = wrap_gh_call(gh_repo.branch, [oca_branch])
tree_data = []
# Check resources on Transifex
tx_project_api = self.tx_api.project(tx_project['slug'])
resources = wrap_tx_call(tx_project_api.resources().get)
for resource in resources:
print "Checking resource %s..." % resource['name']
tx_resource_api = tx_project_api.resource(resource['slug'])
stats = wrap_tx_call(tx_resource_api.stats().get)
for lang in stats.keys():
# Discard english (native language in Odoo) or empty langs
if lang == 'en' or not stats[lang]['translated_words']:
continue
cont = 0
tx_lang = False
while cont < self.tx_num_retries and not tx_lang:
# for some weird reason, sometimes Transifex fails to
# some requests, so this retry mechanism handles this
# problem
try:
tx_lang = wrap_tx_call(
tx_resource_api.translation(lang).get,
)
except (exceptions.HttpClientError,
exceptions.HttpServerError):
tx_lang = False
cont += 1
if tx_lang:
gh_i18n_path = os.path.join('/', resource['slug'], "i18n")
gh_file_path = os.path.join(gh_i18n_path, lang + '.po')
tx_po_file = polib.pofile(tx_lang['content'])
tx_po_dict = self._load_po_dict(tx_po_file)
gh_file = wrap_gh_call(
gh_repo.contents, [gh_file_path, gh_branch.name],
)
if gh_file:
try:
gh_po_file = polib.pofile(
gh_file.decoded.decode('utf-8'))
except IOError:
print "...ERROR reading %s" % gh_file_path
continue
gh_po_dict = self._load_po_dict(gh_po_file)
unmatched_items = (set(gh_po_dict.items()) ^
set(tx_po_dict.items()))
if not unmatched_items:
print "...no change in %s" % gh_file_path
continue
print '..replacing %s' % gh_file_path
new_file_blob = wrap_gh_call(
gh_repo.create_blob,
args=[tx_lang['content']],
kwargs={'encoding': 'utf-8'},
)
tree_data.append({
'path': gh_file_path[1:],
'mode': '100644',
'type': 'blob',
'sha': new_file_blob,
})
else:
print "ERROR: fetching lang '%s'" % lang
if tree_data:
tree_sha = gh_branch.commit.commit.tree.sha
tree = wrap_gh_call(
gh_repo.create_tree, [tree_data, tree_sha],
)
message = '[i18n] New Translations! We need your help ... \n\n to translate more: https://www.it-projects.info/page/translate'
if tree:
commit = wrap_gh_call(
gh_repo.create_commit,
kwargs={
'message': message,
'tree': tree.sha,
'parents': [gh_branch.commit.sha],
'author': self.gh_credentials,
'committer': self.gh_credentials,
},
)
print "Pushing to GitHub"
wrap_gh_call(
gh_repo.ref('heads/{}'.format(gh_branch.name)).update,
args=[commit.sha],
)
class TransifexPuller(object):
def __init__(self, target=None, email=None):
# Read config
config = read_config()
self.gh_token = config.get('GitHub', 'token')
tx_username = (
config.get('Transifex', 'username') or
os.environ.get('TRANSIFEX_USER') or
TX_USERNAME_DEFAULT)
tx_password = (
config.get('Transifex', 'password') or
os.environ.get('TRANSIFEX_PASSWORD'))
self.tx_num_retries = (
config.get('Transifex', 'num_retries') or
os.environ.get('TRANSIFEX_RETRIES'))
self.tx_org = (
config.get('Transifex', 'organization') or
os.environ.get('TRANSIFEX_ORGANIZATION') or
TX_ORG_DEFAULT)
self.gh_org = target or self.tx_org
# Connect to GitHub
self.github = github_login.login()
gh_user = wrap_gh_call(self.github.user)
if not gh_user.email and not email:
raise Exception(
'Email required to commit to github. Please provide one on '
'the command line or make the one of your github profile '
'public.'
)
self.gh_credentials = {
'name': gh_user.name or str(gh_user),
'email': gh_user.email or email,
}
# Connect to Transifex
self.tx_api = API(TX_URL, auth=(tx_username, tx_password))
@classmethod
def _load_po_dict(cls, po_file):
po_dict = {}
for po_entry in po_file:
if po_entry.msgstr:
key = u'\n'.join(x[0] for x in po_entry.occurrences)
key += u'\nmsgid "%s"' % po_entry.msgid
po_dict[key] = po_entry.msgstr
return po_dict
@classmethod
def _get_oca_project_info(cls, tx_project):
"""Retrieve project and branch on github from transifex project
information
"""
# use the project name since it's always formatted using the convention
# my-project (version)
# The usage of the name is required since it's hard to find a rule
# that covers the following cases when using the tx_slug
# OCA-l10n-xxx-8-0
# OCA-l10n-xxx-master
# OCA XXX_xxx-xxx
tx_name = tx_project['name']
regex = r'(?P<repo>[^\s]+) \((?P<branch>[^\s]+)\)'
match_object = re.search(regex, tx_name)
oca_project = match_object.group('repo')
oca_branch = match_object.group('branch').replace('-', '.')
return oca_project, oca_branch
def process_projects(self, projects=None):
"""For each project, get translations from transifex and push to
the corresponding project in gihub """
tx_projects = []
if projects:
# Check that provided projects are correct
for project_slug in projects:
try:
tx_project = wrap_tx_call(
self.tx_api.project(project_slug).get
)
tx_projects.append(tx_project)
except exceptions.HttpNotFoundError:
print "ERROR: Transifex project slug '%s' is invalid" % (
project_slug
)
else:
start = 1
temp_projects = []
print "Getting Transifex projects..."
while temp_projects or start == 1:
temp_projects = wrap_tx_call(
self.tx_api.projects().get, kwargs={'start': start},
)
start += len(temp_projects)
tx_projects += temp_projects
for tx_project in tx_projects:
if self.tx_org + '-' in tx_project['slug']:
self._process_project(tx_project)
def _process_project(self, tx_project):
print "Processing project '%s'..." % tx_project['name']
oca_project, oca_branch = self._get_oca_project_info(tx_project)
# get a reference to the github repo and branch where to push the
# the translations
gh_repo = wrap_gh_call(
self.github.repository, [self.gh_org, oca_project],
)
gh_branch = wrap_gh_call(gh_repo.branch, [oca_branch])
tree_data = []
# Check resources on Transifex
tx_project_api = self.tx_api.project(tx_project['slug'])
resources = wrap_tx_call(tx_project_api.resources().get)
for resource in resources:
print "Checking resource %s..." % resource['name']
tx_resource_api = tx_project_api.resource(resource['slug'])
stats = wrap_tx_call(tx_resource_api.stats().get)
for lang in stats.keys():
# Discard english (native language in Odoo) or empty langs
if lang == 'en' or not stats[lang]['translated_words']:
continue
cont = 0
tx_lang = False
while cont < self.tx_num_retries and not tx_lang:
# for some weird reason, sometimes Transifex fails to
# some requests, so this retry mechanism handles this
# problem
try:
tx_lang = wrap_tx_call(
tx_resource_api.translation(lang).get,
)
except (exceptions.HttpClientError,
exceptions.HttpServerError):
tx_lang = False
cont += 1
if tx_lang:
gh_i18n_path = os.path.join('/', resource['slug'], "i18n")
gh_file_path = os.path.join(gh_i18n_path, lang + '.po')
tx_po_file = polib.pofile(tx_lang['content'])
tx_po_dict = self._load_po_dict(tx_po_file)
gh_file = wrap_gh_call(
gh_repo.contents, [gh_file_path, gh_branch.name],
)
if gh_file:
try:
gh_po_file = polib.pofile(
gh_file.decoded.decode('utf-8'))
except IOError:
print "...ERROR reading %s" % gh_file_path
continue
gh_po_dict = self._load_po_dict(gh_po_file)
unmatched_items = (set(gh_po_dict.items()) ^
set(tx_po_dict.items()))
if not unmatched_items:
print "...no change in %s" % gh_file_path
continue
print '..replacing %s' % gh_file_path
new_file_blob = wrap_gh_call(
gh_repo.create_blob,
args=[tx_lang['content']],
kwargs={'encoding': 'utf-8'},
)
tree_data.append({
'path': gh_file_path[1:],
'mode': '100644',
'type': 'blob',
'sha': new_file_blob,
})
else:
print "ERROR: fetching lang '%s'" % lang
if tree_data:
tree_sha = gh_branch.commit.commit.tree.sha
tree = wrap_gh_call(
gh_repo.create_tree, [tree_data, tree_sha],
)
message = 'OCA Transbot updated translations from Transifex'
if tree:
commit = wrap_gh_call(
gh_repo.create_commit,
kwargs={
'message': message,
'tree': tree.sha,
'parents': [gh_branch.commit.sha],
'author': self.gh_credentials,
'committer': self.gh_credentials,
},
)
print "Pushing to GitHub"
wrap_gh_call(
gh_repo.ref('heads/{}'.format(gh_branch.name)).update,
args=[commit.sha],
)
