Beispiel #1
0
class TransifexPuller(object):
    def __init__(self, target=None, email=None):
        # Read config
        config = read_config()
        self.gh_token = config.get('GitHub', 'token')
        tx_username = (
            config.get('Transifex', 'username') or
            os.environ.get('TRANSIFEX_USER') or
            TX_USERNAME_DEFAULT)
        tx_password = (
            config.get('Transifex', 'password') or
            os.environ.get('TRANSIFEX_PASSWORD'))
        self.tx_num_retries = (
            config.get('Transifex', 'num_retries') or
            os.environ.get('TRANSIFEX_RETRIES'))
        self.tx_org = (
            config.get('Transifex', 'organization') or
            os.environ.get('TRANSIFEX_ORGANIZATION') or
            TX_ORG_DEFAULT)
        self.gh_org = target or self.tx_org
        # Connect to GitHub
        self.github = github_login.login()
        gh_user = wrap_gh_call(self.github.user)
        if not gh_user.email and not email:
            raise Exception(
                'Email required to commit to github. Please provide one on '
                'the command line or make the one of your github profile '
                'public.'
            )
        self.gh_credentials = {
            'name': gh_user.name or str(gh_user),
            'email': gh_user.email or email,
        }
        # Connect to Transifex
        self.tx_api = API(TX_URL, auth=(tx_username, tx_password))

    @classmethod
    def _load_po_dict(cls, po_file):
        po_dict = {}
        for po_entry in po_file:
            if po_entry.msgstr:
                key = u'\n'.join(x[0] for x in po_entry.occurrences)
                key += u'\nmsgid "%s"' % po_entry.msgid
                po_dict[key] = po_entry.msgstr
        return po_dict

    @classmethod
    def _get_oca_project_info(cls, tx_project):
        """Retrieve project and branch on github from transifex project
        information
        """
        # use the project name since it's always formatted using the convention
        # my-project (version)
        # The usage of the name is required since it's hard to find a rule
        # that covers the following cases when using the tx_slug
        # OCA-l10n-xxx-8-0
        # OCA-l10n-xxx-master
        # OCA XXX_xxx-xxx
        tx_name = tx_project['name']
        regex = r'(?P<repo>[^\s]+) \((?P<branch>[^\s]+)\)'
        match_object = re.search(regex, tx_name)
        oca_project = match_object.group('repo')
        oca_branch = match_object.group('branch').replace('-', '.')
        return oca_project, oca_branch

    def process_projects(self, projects=None):
        """For each project, get translations from transifex and push to
        the corresponding project in gihub """
        tx_projects = []
        if projects:
            # Check that provided projects are correct
            for project_slug in projects:
                try:
                    tx_project = wrap_tx_call(
                        self.tx_api.project(project_slug).get
                    )
                    tx_projects.append(tx_project)
                except exceptions.HttpNotFoundError:
                    print "ERROR: Transifex project slug '%s' is invalid" % (
                        project_slug
                    )
        else:
            start = 1
            temp_projects = []
            print "Getting Transifex projects..."
            while temp_projects or start == 1:
                temp_projects = wrap_tx_call(
                    self.tx_api.projects().get, kwargs={'start': start},
                )
                start += len(temp_projects)
                tx_projects += temp_projects
        for tx_project in tx_projects:
            if self.tx_org + '-' in tx_project['slug']:
                self._process_project(tx_project)

    def _process_project(self, tx_project):
        print "Processing project '%s'..." % tx_project['name']
        oca_project, oca_branch = self._get_oca_project_info(tx_project)
        # get a reference to the github repo and branch where to push the
        # the translations
        gh_repo = wrap_gh_call(
            self.github.repository, [self.gh_org, oca_project],
        )
        gh_branch = wrap_gh_call(gh_repo.branch, [oca_branch])
        tree_data = []
        # Check resources on Transifex
        tx_project_api = self.tx_api.project(tx_project['slug'])
        resources = wrap_tx_call(tx_project_api.resources().get)
        for resource in resources:
            print "Checking resource %s..." % resource['name']
            tx_resource_api = tx_project_api.resource(resource['slug'])
            stats = wrap_tx_call(tx_resource_api.stats().get)
            for lang in stats.keys():
                # Discard english (native language in Odoo) or empty langs
                if lang == 'en' or not stats[lang]['translated_words']:
                    continue
                cont = 0
                tx_lang = False
                while cont < self.tx_num_retries and not tx_lang:
                    # for some weird reason, sometimes Transifex fails to
                    # some requests, so this retry mechanism handles this
                    # problem
                    try:
                        tx_lang = wrap_tx_call(
                            tx_resource_api.translation(lang).get,
                        )
                    except (exceptions.HttpClientError,
                            exceptions.HttpServerError):
                        tx_lang = False
                        cont += 1
                if tx_lang:
                    gh_i18n_path = os.path.join('/', resource['slug'], "i18n")
                    gh_file_path = os.path.join(gh_i18n_path, lang + '.po')
                    tx_po_file = polib.pofile(tx_lang['content'])
                    tx_po_dict = self._load_po_dict(tx_po_file)
                    gh_file = wrap_gh_call(
                        gh_repo.contents, [gh_file_path, gh_branch.name],
                    )
                    if gh_file:
                        try:
                            gh_po_file = polib.pofile(
                                gh_file.decoded.decode('utf-8'))
                        except IOError:
                            print "...ERROR reading %s" % gh_file_path
                            continue
                        gh_po_dict = self._load_po_dict(gh_po_file)
                        unmatched_items = (set(gh_po_dict.items()) ^
                                           set(tx_po_dict.items()))
                        if not unmatched_items:
                            print "...no change in %s" % gh_file_path
                            continue
                    print '..replacing %s' % gh_file_path
                    new_file_blob = wrap_gh_call(
                        gh_repo.create_blob,
                        args=[tx_lang['content']],
                        kwargs={'encoding': 'utf-8'},
                    )
                    tree_data.append({
                        'path': gh_file_path[1:],
                        'mode': '100644',
                        'type': 'blob',
                        'sha': new_file_blob,
                    })
                else:
                    print "ERROR: fetching lang '%s'" % lang
        if tree_data:
            tree_sha = gh_branch.commit.commit.tree.sha
            tree = wrap_gh_call(
                gh_repo.create_tree, [tree_data, tree_sha],
            )
            message = '[i18n] New Translations! We need your help ... \n\n to translate more: https://www.it-projects.info/page/translate'
            if tree:
                commit = wrap_gh_call(
                    gh_repo.create_commit,
                    kwargs={
                        'message': message,
                        'tree': tree.sha,
                        'parents': [gh_branch.commit.sha],
                        'author': self.gh_credentials,
                        'committer': self.gh_credentials,
                    },
                )
                print "Pushing to GitHub"
                wrap_gh_call(
                    gh_repo.ref('heads/{}'.format(gh_branch.name)).update,
                    args=[commit.sha],
                )
Beispiel #2
0
class TransifexPuller(object):
    def __init__(self, target=None, email=None):
        # Read config
        config = read_config()
        self.gh_token = config.get('GitHub', 'token')
        tx_username = (
            config.get('Transifex', 'username') or
            os.environ.get('TRANSIFEX_USER') or
            TX_USERNAME_DEFAULT)
        tx_password = (
            config.get('Transifex', 'password') or
            os.environ.get('TRANSIFEX_PASSWORD'))
        self.tx_num_retries = (
            config.get('Transifex', 'num_retries') or
            os.environ.get('TRANSIFEX_RETRIES'))
        self.tx_org = (
            config.get('Transifex', 'organization') or
            os.environ.get('TRANSIFEX_ORGANIZATION') or
            TX_ORG_DEFAULT)
        self.gh_org = target or self.tx_org
        # Connect to GitHub
        self.github = github_login.login()
        gh_user = self.github.user()

        if not gh_user.email and not email:
            raise Exception(
                'Email required to commit to github. Please provide one on '
                'the command line or make the one of your github profile '
                'public.')
        self.gh_credentials = {'name': gh_user.name or str(gh_user),
                               'email': gh_user.email or email}

        # Connect to Transifex
        self.tx_api = API(TX_URL, auth=(tx_username, tx_password))

    @classmethod
    def _load_po_dict(cls, po_file):
        po_dict = {}
        for po_entry in po_file:
            if po_entry.msgstr:
                key = u'\n'.join(x[0] for x in po_entry.occurrences)
                key += u'\nmsgid "%s"' % po_entry.msgid
                po_dict[key] = po_entry.msgstr
        return po_dict

    @classmethod
    def _get_oca_project_info(cls, tx_project):
        """Retrieve project and branch on github from transifex project
        information
        """
        # use the project name since it's always formatted using the convention
        # my-project (version)
        # The usage of the name is required since it's hard to find a rule
        # that covers the following cases when using the tx_slug
        # OCA-l10n-xxx-8-0
        # OCA-l10n-xxx-master
        # OCA XXX_xxx-xxx
        tx_name = tx_project['name']
        regex = r'(?P<repo>[^\s]+) \((?P<branch>[^\s]+)\)'
        match_object = re.search(regex, tx_name)
        oca_project = match_object.group('repo')
        oca_branch = match_object.group('branch').replace('-', '.')
        return oca_project, oca_branch

    def process_projects(self, projects=None):
        """For each project, get translations from transifex and push to
        the corresponding project in gihub """
        tx_projects = []
        if projects:
            # Check that provided projects are correct
            for project_slug in projects:
                try:
                    tx_project = self.tx_api.project(project_slug).get()
                    tx_projects.append(tx_project)
                except (KeyboardInterrupt, SystemExit):
                    raise
                except:
                    print "ERROR: Transifex project slug %s is invalid" % (
                        project_slug)
                    return
        else:
            start = 1
            temp_projects = []
            print "Getting Transifex projects..."
            while temp_projects or start == 1:
                temp_projects = self.tx_api.projects().get(start=start)
                start += len(temp_projects)
                tx_projects += temp_projects
        for tx_project in tx_projects:
            if self.tx_org + '-' in tx_project['slug']:
                self._process_project(tx_project)

    def _process_project(self, tx_project):
        print "Processing project '%s'..." % tx_project['name']
        oca_project, oca_branch = self._get_oca_project_info(tx_project)
        # get a reference to the github repo and branch where to push the
        # the translations
        gh_repo = self.github.repository(self.gh_org, oca_project)
        gh_branch = gh_repo.branch(oca_branch)
        tree_data = []
        # Check resources on Transifex
        tx_project_api = self.tx_api.project(tx_project['slug'])
        resources = tx_project_api.resources().get()
        for resource in resources:
            print "Checking resource %s..." % resource['name']
            if resource['slug'] != 'date_range':
                continue
            tx_resource_api = tx_project_api.resource(resource['slug'])
            stats = tx_resource_api.stats().get()
            for lang in stats.keys():
                # Discard english (native language in Odoo) or empty langs
                if lang == 'en' or not stats[lang]['translated_words']:
                    continue
                cont = 0
                tx_lang = False
                while cont < self.tx_num_retries and not tx_lang:
                    # for some weird reason, sometimes Transifex fails to
                    # some requests, so this retry mechanism handles this
                    # problem
                    try:
                        tx_lang = tx_resource_api.translation(lang).get()
                    except (KeyboardInterrupt, SystemExit):
                        raise
                    except exceptions.HttpClientError:
                        tx_lang = False
                        cont += 1
                if tx_lang:
                    gh_i18n_path = os.path.join('/', resource['slug'], "i18n")
                    gh_file_path = os.path.join(gh_i18n_path, lang + '.po')
                    try:
                        tx_po_file = polib.pofile(tx_lang['content'])
                        tx_po_dict = self._load_po_dict(tx_po_file)
                        gh_file = gh_repo.contents(
                            gh_file_path, gh_branch.name)
                        if gh_file:
                            gh_po_file = polib.pofile(
                                gh_file.decoded.decode('utf-8'))
                            gh_po_dict = self._load_po_dict(gh_po_file)
                            unmatched_items = (set(gh_po_dict.items()) ^
                                               set(tx_po_dict.items()))
                            if not unmatched_items:
                                print "...no change in %s" % gh_file_path
                                continue
                        print '..replacing %s' % gh_file_path
                        new_file_blob = gh_repo.create_blob(
                            tx_lang['content'], encoding='utf-8')
                        tree_data.append({
                            'path': gh_file_path[1:],
                            'mode': '100644',
                            'type': 'blob',
                            'sha': new_file_blob})
                    except (KeyboardInterrupt, SystemExit):
                        raise
                    except:
                        print "ERROR: processing lang '%s'" % lang
                else:
                    print "ERROR: fetching lang '%s'" % lang
            # Wait a minute before the next file to avoid reaching Transifex
            # API limitations
            # TODO: Request the API to get the date for the next request
            # http://docs.rackspace.com/loadbalancers/api/v1.0/clb-devguide/\
            # content/Determining_Limits_Programmatically-d1e1039.html
            print "Sleeping 70 seconds..."
            time.sleep(70)
        if tree_data:
            tree_sha = gh_branch.commit.commit.tree.sha
            tree = gh_repo.create_tree(tree_data, tree_sha)
            message = 'OCA Transbot updated translations from Transifex'
            print "message", message
            commit = gh_repo.create_commit(
                message=message, tree=tree.sha, parents=[gh_branch.commit.sha],
                author=self.gh_credentials, committer=self.gh_credentials)
            print "git pushing"
            gh_repo.ref('heads/{}'.format(gh_branch.name)).update(commit.sha)
        # Wait 5 minutes before the next project to avoid reaching Transifex
        # API limitations
        # TODO: Request the API to get the date for the next request
        # http://docs.rackspace.com/loadbalancers/api/v1.0/clb-devguide/\
        # content/Determining_Limits_Programmatically-d1e1039.html
        print "Sleeping 5 minutes..."
        time.sleep(300)
Beispiel #3
0
def main(argv=None):
    """
    Export translation files and push them to Transifex
    The transifex password should be encrypted in .travis.yml
    If not, export exits early.
    """
    if argv is None:
        argv = sys.argv

    transifex_user = os.environ.get("TRANSIFEX_USER")
    transifex_password = os.environ.get("TRANSIFEX_PASSWORD")

    if not transifex_user:
        print(
            yellow_light("WARNING! Transifex user not defined- "
                         "exiting early."))
        return 1

    if not transifex_password:
        print(
            yellow_light("WARNING! Transifex password not recognized- "
                         "exiting early."))
        return 1

    travis_home = os.environ.get("HOME", "~/")
    travis_build_dir = os.environ.get("TRAVIS_BUILD_DIR", "../..")
    travis_repo_slug = os.environ.get("TRAVIS_REPO_SLUG")
    travis_repo_owner = travis_repo_slug.split("/")[0]
    travis_repo_shortname = travis_repo_slug.split("/")[1]
    odoo_unittest = False
    odoo_exclude = os.environ.get("EXCLUDE")
    odoo_include = os.environ.get("INCLUDE")
    install_options = os.environ.get("INSTALL_OPTIONS", "").split()
    odoo_version = os.environ.get("VERSION")

    if not odoo_version:
        # For backward compatibility, take version from parameter
        # if it's not globally set
        odoo_version = sys.argv[1]
        print(
            yellow_light("WARNING: no env variable set for VERSION. "
                         "Using '%s'" % odoo_version))

    default_project_slug = "%s-%s" % (travis_repo_slug.replace(
        '/', '-'), odoo_version.replace('.', '-'))
    transifex_project_slug = os.environ.get("TRANSIFEX_PROJECT_SLUG",
                                            default_project_slug)
    transifex_project_name = "%s (%s)" % (travis_repo_shortname, odoo_version)
    transifex_organization = os.environ.get("TRANSIFEX_ORGANIZATION",
                                            travis_repo_owner)
    transifex_fill_up_resources = os.environ.get("TRANSIFEX_FILL_UP_RESOURCES",
                                                 "True")
    transifex_team = os.environ.get("TRANSIFEX_TEAM", "23907")
    repository_url = "https://github.com/%s" % travis_repo_slug

    odoo_full = os.environ.get("ODOO_REPO", "odoo/odoo")
    server_path = get_server_path(odoo_full, odoo_version, travis_home)
    addons_path = get_addons_path(travis_home, travis_build_dir, server_path)
    addons_list = get_addons_to_check(travis_build_dir, odoo_include,
                                      odoo_exclude)
    addons = ','.join(addons_list)

    print("\nWorking in %s" % travis_build_dir)
    print("Using repo %s and addons path %s" % (odoo_full, addons_path))

    if not addons:
        print(yellow_light("WARNING! Nothing to translate- exiting early."))
        return 0

    # Create Transifex project if it doesn't exist
    print()
    print(yellow("Creating Transifex project if it doesn't exist"))
    auth = (transifex_user, transifex_password)
    api_url = "https://www.transifex.com/api/2/"
    api = API(api_url, auth=auth)
    project_data = {
        "slug": transifex_project_slug,
        "name": transifex_project_name,
        "source_language_code": "en",
        "description": transifex_project_name,
        "repository_url": repository_url,
        "organization": transifex_organization,
        "license": "permissive_open_source",
        "fill_up_resources": transifex_fill_up_resources,
        "team": transifex_team,
    }
    try:
        api.project(transifex_project_slug).get()
        print('This Transifex project already exists.')
    except exceptions.HttpClientError:
        try:
            api.projects.post(project_data)
            print('Transifex project has been successfully created.')
        except exceptions.HttpClientError:
            print('Transifex organization: %s' % transifex_organization)
            print('Transifex username: %s' % transifex_user)
            print('Transifex project slug: %s' % transifex_project_slug)
            print(
                red('Error: Authentication failed. Please verify that '
                    'Transifex organization, user and password are '
                    'correct. You can change these variables in your '
                    '.travis.yml file.'))
            raise

    print("\nModules to translate: %s" % addons)

    # Install the modules on the database
    database = "openerp_i18n"
    setup_server(database, odoo_unittest, addons, server_path, addons_path,
                 install_options, addons_list)

    # Initialize Transifex project
    print()
    print(yellow('Initializing Transifex project'))
    init_args = [
        '--host=https://www.transifex.com',
        '--user=%s' % transifex_user,
        '--pass=%s' % transifex_password
    ]
    commands.cmd_init(init_args, path_to_tx=None)
    path_to_tx = utils.find_dot_tx()

    connection_context = context_mapping[odoo_version]
    with connection_context(server_path, addons_path, database) \
            as odoo_context:
        for module in addons_list:
            print()
            print(yellow("Downloading PO file for %s" % module))
            source_filename = os.path.join(travis_build_dir, module, 'i18n',
                                           module + ".pot")
            # Create i18n/ directory if doesn't exist
            if not os.path.exists(os.path.dirname(source_filename)):
                os.makedirs(os.path.dirname(source_filename))
            with open(source_filename, 'w') as f:
                f.write(odoo_context.get_pot_contents(module))

            print()
            print(yellow("Linking PO file and Transifex resource"))
            set_args = [
                '-t', 'PO', '--auto-local', '-r',
                '%s.%s' % (transifex_project_slug, module),
                '%s/i18n/<lang>.po' % module, '--source-lang', 'en',
                '--source-file', source_filename, '--execute'
            ]
            commands.cmd_set(set_args, path_to_tx)

    print()
    print(yellow('Pushing translation files to Transifex'))
    push_args = ['-s', '-t', '--skip']
    commands.cmd_push(push_args, path_to_tx)

    return 0
def main(argv=None):
    """
    Export translation files and push them to Transifex
    The transifex password should be encrypted in .travis.yml
    If not, export exits early.
    """
    if argv is None:
        argv = sys.argv

    transifex_user = os.environ.get("TRANSIFEX_USER")
    transifex_password = os.environ.get("TRANSIFEX_PASSWORD")

    if not transifex_user:
        print(yellow_light("WARNING! Transifex user not defined- "
              "exiting early."))
        return 1

    if not transifex_password:
        print(yellow_light("WARNING! Transifex password not recognized- "
              "exiting early."))
        return 1

    travis_home = os.environ.get("HOME", "~/")
    travis_dependencies_dir = os.path.join(travis_home, 'dependencies')
    travis_build_dir = os.environ.get("TRAVIS_BUILD_DIR", "../..")
    travis_repo_slug = os.environ.get("TRAVIS_REPO_SLUG")
    travis_repo_owner = travis_repo_slug.split("/")[0]
    travis_repo_shortname = travis_repo_slug.split("/")[1]
    odoo_unittest = False
    odoo_exclude = os.environ.get("EXCLUDE")
    odoo_include = os.environ.get("INCLUDE")
    install_options = os.environ.get("INSTALL_OPTIONS", "").split()
    odoo_version = os.environ.get("VERSION")

    if not odoo_version:
        # For backward compatibility, take version from parameter
        # if it's not globally set
        odoo_version = argv[1]
        print(yellow_light("WARNING: no env variable set for VERSION. "
              "Using '%s'" % odoo_version))

    default_project_slug = "%s-%s" % (travis_repo_slug.replace('/', '-'),
                                      odoo_version.replace('.', '-'))
    transifex_project_slug = os.environ.get("TRANSIFEX_PROJECT_SLUG",
                                            default_project_slug)
    transifex_project_name = "%s (%s)" % (travis_repo_shortname, odoo_version)
    transifex_organization = os.environ.get("TRANSIFEX_ORGANIZATION",
                                            travis_repo_owner)
    transifex_fill_up_resources = os.environ.get(
        "TRANSIFEX_FILL_UP_RESOURCES", "True"
    )
    transifex_team = os.environ.get(
        "TRANSIFEX_TEAM", "23907"
    )
    repository_url = "https://github.com/%s" % travis_repo_slug

    odoo_full = os.environ.get("ODOO_REPO", "odoo/odoo")
    server_path = get_server_path(odoo_full, odoo_version, travis_home)
    addons_path = get_addons_path(travis_dependencies_dir,
                                  travis_build_dir,
                                  server_path)
    addons_list = get_addons_to_check(travis_build_dir, odoo_include,
                                      odoo_exclude)
    addons = ','.join(addons_list)
    create_server_conf({'addons_path': addons_path}, odoo_version)

    print("\nWorking in %s" % travis_build_dir)
    print("Using repo %s and addons path %s" % (odoo_full, addons_path))

    if not addons:
        print(yellow_light("WARNING! Nothing to translate- exiting early."))
        return 0

    # Create Transifex project if it doesn't exist
    print()
    print(yellow("Creating Transifex project if it doesn't exist"))
    auth = (transifex_user, transifex_password)
    api_url = "https://www.transifex.com/api/2/"
    api = API(api_url, auth=auth)
    project_data = {"slug": transifex_project_slug,
                    "name": transifex_project_name,
                    "source_language_code": "en",
                    "description": transifex_project_name,
                    "repository_url": repository_url,
                    "organization": transifex_organization,
                    "license": "permissive_open_source",
                    "fill_up_resources": transifex_fill_up_resources,
                    "team": transifex_team,
                    }
    try:
        api.project(transifex_project_slug).get()
        print('This Transifex project already exists.')
    except exceptions.HttpClientError:
        try:
            api.projects.post(project_data)
            print('Transifex project has been successfully created.')
        except exceptions.HttpClientError:
            print('Transifex organization: %s' % transifex_organization)
            print('Transifex username: %s' % transifex_user)
            print('Transifex project slug: %s' % transifex_project_slug)
            print(red('Error: Authentication failed. Please verify that '
                      'Transifex organization, user and password are '
                      'correct. You can change these variables in your '
                      '.travis.yml file.'))
            raise

    print("\nModules to translate: %s" % addons)

    # Install the modules on the database
    database = "openerp_i18n"
    setup_server(database, odoo_unittest, addons, server_path, addons_path,
                 install_options, addons_list)

    # Initialize Transifex project
    print()
    print(yellow('Initializing Transifex project'))
    init_args = ['--host=https://www.transifex.com',
                 '--user=%s' % transifex_user,
                 '--pass=%s' % transifex_password]
    commands.cmd_init(init_args, path_to_tx=None)
    path_to_tx = utils.find_dot_tx()

    connection_context = context_mapping[odoo_version]
    with connection_context(server_path, addons_path, database) \
            as odoo_context:
        for module in addons_list:
            print()
            print(yellow("Downloading POT file for %s" % module))
            source_filename = os.path.join(travis_build_dir, module, 'i18n',
                                           module + ".pot")
            # Create i18n/ directory if doesn't exist
            if not os.path.exists(os.path.dirname(source_filename)):
                os.makedirs(os.path.dirname(source_filename))
            with open(source_filename, 'w') as f:
                f.write(odoo_context.get_pot_contents(module))

            print()
            print(yellow("Linking POT file and Transifex resource"))
            set_args = ['-t', 'PO',
                        '--auto-local',
                        '-r', '%s.%s' % (transifex_project_slug, module),
                        '%s/i18n/<lang>.po' % module,
                        '--source-lang', 'en',
                        '--source-file', source_filename,
                        '--execute']
            commands.cmd_set(set_args, path_to_tx)

    print()
    print(yellow('Pushing translation files to Transifex'))
    push_args = ['-s', '-t', '--skip']
    commands.cmd_push(push_args, path_to_tx)

    return 0
def main(argv=None):
    """
    Export translation files and push them to Transifex
    The transifex password should be encrypted in .travis.yml
    If not, export exits early.
    """
    if argv is None:
        argv = sys.argv

    transifex_user = os.environ.get("TRANSIFEX_USER", "*****@*****.**")
    transifex_password = os.environ.get("TRANSIFEX_PASSWORD")

    if not transifex_user:
        print(
            yellow_light("WARNING! Transifex user not defined- "
                         "exiting early."))
        return 1

    if not transifex_password:
        print(
            yellow_light("WARNING! Transifex password not recognized- "
                         "exiting early."))
        return 1

    travis_home = os.environ.get("HOME", "~/")
    travis_build_dir = os.environ.get("TRAVIS_BUILD_DIR", "../..")
    travis_repo_slug = os.environ.get("TRAVIS_REPO_SLUG")
    travis_repo_owner = travis_repo_slug.split("/")[0]
    travis_repo_shortname = travis_repo_slug.split("/")[1]
    odoo_unittest = False
    odoo_exclude = os.environ.get("EXCLUDE")
    odoo_include = os.environ.get("INCLUDE")
    install_options = os.environ.get("INSTALL_OPTIONS", "").split()
    odoo_version = os.environ.get("VERSION")
    langs = parse_list(os.environ.get("LANG_ALLOWED", ""))

    if not odoo_version:
        # For backward compatibility, take version from parameter
        # if it's not globally set
        odoo_version = argv[1]
        print(
            yellow_light("WARNING: no env variable set for VERSION. "
                         "Using '%s'" % odoo_version))

    default_project_slug = "%s-%s" % (travis_repo_slug.replace(
        '/', '-'), odoo_version.replace('.', '-'))
    transifex_project_slug = os.environ.get("TRANSIFEX_PROJECT_SLUG",
                                            default_project_slug)
    transifex_project_name = "%s (%s)" % (travis_repo_shortname, odoo_version)
    transifex_organization = os.environ.get("TRANSIFEX_ORGANIZATION",
                                            travis_repo_owner)
    transifex_fill_up_resources = os.environ.get("TRANSIFEX_FILL_UP_RESOURCES",
                                                 "True")
    transifex_team = os.environ.get("TRANSIFEX_TEAM", "45447")
    repository_url = "https://github.com/%s" % travis_repo_slug

    odoo_full = os.environ.get("ODOO_REPO", "odoo/odoo")
    server_path = get_server_path(odoo_full, odoo_version, travis_home)
    addons_path = get_addons_path(travis_home, travis_build_dir, server_path)
    addons_list = get_addons_to_check(travis_build_dir, odoo_include,
                                      odoo_exclude)
    addons_path_list = parse_list(addons_path)
    all_depends = get_depends(addons_path_list, addons_list)
    main_modules = set(os.listdir(travis_build_dir))
    main_depends = main_modules & all_depends
    addons_list = list(main_depends)
    addons = ','.join(addons_list)
    create_server_conf({'addons_path': addons_path}, odoo_version)

    print("\nWorking in %s" % travis_build_dir)
    print("Using repo %s and addons path %s" % (odoo_full, addons_path))

    if not addons:
        print(yellow_light("WARNING! Nothing to translate- exiting early."))
        return 0

    # Create Transifex project if it doesn't exist
    print()
    print(yellow("Creating Transifex project if it doesn't exist"))
    auth = (transifex_user, transifex_password)
    api_url = "https://www.transifex.com/api/2/"
    api = API(api_url, auth=auth)
    project_data = {
        "slug": transifex_project_slug,
        "name": transifex_project_name,
        "source_language_code": "en",
        "description": transifex_project_name,
        "repository_url": repository_url,
        "organization": transifex_organization,
        "license": "permissive_open_source",
        "fill_up_resources": transifex_fill_up_resources,
        "team": transifex_team,
    }
    try:
        api.project(transifex_project_slug).get()
        print('This Transifex project already exists.')
    except exceptions.HttpClientError:
        try:
            api.projects.post(project_data)
            print('Transifex project has been successfully created.')
        except exceptions.HttpClientError:
            print('Transifex organization: %s' % transifex_organization)
            print('Transifex username: %s' % transifex_user)
            print('Transifex project slug: %s' % transifex_project_slug)
            print(
                red('Error: Authentication failed. Please verify that '
                    'Transifex organization, user and password are '
                    'correct. You can change these variables in your '
                    '.travis.yml file.'))
            raise

    print("\nModules to translate: %s" % addons)

    # Install the modules on the database
    database = "openerp_test"
    script_name = get_server_script(odoo_version)
    setup_server(database, odoo_unittest, addons, server_path, script_name,
                 addons_path, install_options, addons_list)

    # Initialize Transifex project
    print()
    print(yellow('Initializing Transifex project'))
    init_args = [
        '--host=https://www.transifex.com',
        '--user=%s' % transifex_user,
        '--pass=%s' % transifex_password
    ]
    commands.cmd_init(init_args, path_to_tx=None)
    path_to_tx = utils.find_dot_tx()

    # Use by default version 10 connection context
    connection_context = context_mapping.get(odoo_version, Odoo10Context)
    with connection_context(server_path, addons_path, database) \
            as odoo_context:
        for module in addons_list:
            print()
            print(yellow("Obtaining POT file for %s" % module))
            i18n_folder = os.path.join(travis_build_dir, module, 'i18n')
            source_filename = os.path.join(i18n_folder, module + ".pot")
            # Create i18n/ directory if doesn't exist
            if not os.path.exists(os.path.dirname(source_filename)):
                os.makedirs(os.path.dirname(source_filename))
            with open(source_filename, 'w') as f:
                f.write(odoo_context.get_pot_contents(module))
            # Put the correct timestamp for letting known tx client which
            # translations to update
            for po_file_name in os.listdir(i18n_folder):
                if not po_file_name.endswith('.po'):
                    continue
                if langs and os.path.splitext(po_file_name)[0] not in langs:
                    # Limit just allowed languages if is defined
                    os.remove(os.path.join(i18n_folder, po_file_name))
                    continue
                po_file_name = os.path.join(i18n_folder, po_file_name)
                command = [
                    'git', 'log', '--pretty=format:%cd', '-n1', '--date=raw',
                    po_file_name
                ]
                timestamp = float(subprocess.check_output(command).split()[0])
                # This converts to UTC the timestamp
                timestamp = time.mktime(time.gmtime(timestamp))
                os.utime(po_file_name, (timestamp, timestamp))

            print()
            print(yellow("Linking POT file and Transifex resource"))
            set_args = [
                '-t', 'PO', '--auto-local', '-r',
                '%s.%s' % (transifex_project_slug, module),
                '%s/i18n/<lang>.po' % module, '--source-lang', 'en',
                '--source-file', source_filename, '--execute'
            ]
            commands.cmd_set(set_args, path_to_tx)

    print()
    print(yellow('Pushing translation files to Transifex'))
    push_args = ['-s', '-t', '--skip']
    commands.cmd_push(push_args, path_to_tx)

    return 0
class TransifexPuller(object):
    def __init__(self, target=None, email=None):
        # Read config
        config = read_config()
        self.gh_token = config.get('GitHub', 'token')
        tx_username = (
            config.get('Transifex', 'username') or
            os.environ.get('TRANSIFEX_USER') or
            TX_USERNAME_DEFAULT)
        tx_password = (
            config.get('Transifex', 'password') or
            os.environ.get('TRANSIFEX_PASSWORD'))
        self.tx_num_retries = (
            config.get('Transifex', 'num_retries') or
            os.environ.get('TRANSIFEX_RETRIES'))
        self.tx_org = (
            config.get('Transifex', 'organization') or
            os.environ.get('TRANSIFEX_ORGANIZATION') or
            TX_ORG_DEFAULT)
        self.gh_org = target or self.tx_org
        # Connect to GitHub
        self.github = github_login.login()
        gh_user = wrap_gh_call(self.github.user)
        if not gh_user.email and not email:
            raise Exception(
                'Email required to commit to github. Please provide one on '
                'the command line or make the one of your github profile '
                'public.'
            )
        self.gh_credentials = {
            'name': gh_user.name or str(gh_user),
            'email': gh_user.email or email,
        }
        # Connect to Transifex
        self.tx_api = API(TX_URL, auth=(tx_username, tx_password))

    @classmethod
    def _load_po_dict(cls, po_file):
        po_dict = {}
        for po_entry in po_file:
            if po_entry.msgstr:
                key = u'\n'.join(x[0] for x in po_entry.occurrences)
                key += u'\nmsgid "%s"' % po_entry.msgid
                po_dict[key] = po_entry.msgstr
        return po_dict

    @classmethod
    def _get_oca_project_info(cls, tx_project):
        """Retrieve project and branch on github from transifex project
        information
        """
        # use the project name since it's always formatted using the convention
        # my-project (version)
        # The usage of the name is required since it's hard to find a rule
        # that covers the following cases when using the tx_slug
        # OCA-l10n-xxx-8-0
        # OCA-l10n-xxx-master
        # OCA XXX_xxx-xxx
        tx_name = tx_project['name']
        regex = r'(?P<repo>[^\s]+) \((?P<branch>[^\s]+)\)'
        match_object = re.search(regex, tx_name)
        oca_project = match_object.group('repo')
        oca_branch = match_object.group('branch').replace('-', '.')
        return oca_project, oca_branch

    def process_projects(self, projects=None):
        """For each project, get translations from transifex and push to
        the corresponding project in gihub """
        tx_projects = []
        if projects:
            # Check that provided projects are correct
            for project_slug in projects:
                try:
                    tx_project = wrap_tx_call(
                        self.tx_api.project(project_slug).get
                    )
                    tx_projects.append(tx_project)
                except exceptions.HttpNotFoundError:
                    print "ERROR: Transifex project slug '%s' is invalid" % (
                        project_slug
                    )
        else:
            start = 1
            temp_projects = []
            print "Getting Transifex projects..."
            while temp_projects or start == 1:
                temp_projects = wrap_tx_call(
                    self.tx_api.projects().get, kwargs={'start': start},
                )
                start += len(temp_projects)
                tx_projects += temp_projects
        for tx_project in tx_projects:
            if self.tx_org + '-' in tx_project['slug']:
                self._process_project(tx_project)

    def _process_project(self, tx_project):
        print "Processing project '%s'..." % tx_project['name']
        oca_project, oca_branch = self._get_oca_project_info(tx_project)
        # get a reference to the github repo and branch where to push the
        # the translations
        gh_repo = wrap_gh_call(
            self.github.repository, [self.gh_org, oca_project],
        )
        gh_branch = wrap_gh_call(gh_repo.branch, [oca_branch])
        tree_data = []
        # Check resources on Transifex
        tx_project_api = self.tx_api.project(tx_project['slug'])
        resources = wrap_tx_call(tx_project_api.resources().get)
        for resource in resources:
            print "Checking resource %s..." % resource['name']
            tx_resource_api = tx_project_api.resource(resource['slug'])
            stats = wrap_tx_call(tx_resource_api.stats().get)
            for lang in stats.keys():
                # Discard english (native language in Odoo) or empty langs
                if lang == 'en' or not stats[lang]['translated_words']:
                    continue
                cont = 0
                tx_lang = False
                while cont < self.tx_num_retries and not tx_lang:
                    # for some weird reason, sometimes Transifex fails to
                    # some requests, so this retry mechanism handles this
                    # problem
                    try:
                        tx_lang = wrap_tx_call(
                            tx_resource_api.translation(lang).get,
                        )
                    except (exceptions.HttpClientError,
                            exceptions.HttpServerError):
                        tx_lang = False
                        cont += 1
                if tx_lang:
                    gh_i18n_path = os.path.join('/', resource['slug'], "i18n")
                    gh_file_path = os.path.join(gh_i18n_path, lang + '.po')
                    tx_po_file = polib.pofile(tx_lang['content'])
                    tx_po_dict = self._load_po_dict(tx_po_file)
                    gh_file = wrap_gh_call(
                        gh_repo.contents, [gh_file_path, gh_branch.name],
                    )
                    if gh_file:
                        try:
                            gh_po_file = polib.pofile(
                                gh_file.decoded.decode('utf-8'))
                        except IOError:
                            print "...ERROR reading %s" % gh_file_path
                            continue
                        gh_po_dict = self._load_po_dict(gh_po_file)
                        unmatched_items = (set(gh_po_dict.items()) ^
                                           set(tx_po_dict.items()))
                        if not unmatched_items:
                            print "...no change in %s" % gh_file_path
                            continue
                    print '..replacing %s' % gh_file_path
                    new_file_blob = wrap_gh_call(
                        gh_repo.create_blob,
                        args=[tx_lang['content']],
                        kwargs={'encoding': 'utf-8'},
                    )
                    tree_data.append({
                        'path': gh_file_path[1:],
                        'mode': '100644',
                        'type': 'blob',
                        'sha': new_file_blob,
                    })
                else:
                    print "ERROR: fetching lang '%s'" % lang
        if tree_data:
            tree_sha = gh_branch.commit.commit.tree.sha
            tree = wrap_gh_call(
                gh_repo.create_tree, [tree_data, tree_sha],
            )
            message = 'OCA Transbot updated translations from Transifex'
            if tree:
                commit = wrap_gh_call(
                    gh_repo.create_commit,
                    kwargs={
                        'message': message,
                        'tree': tree.sha,
                        'parents': [gh_branch.commit.sha],
                        'author': self.gh_credentials,
                        'committer': self.gh_credentials,
                    },
                )
                print "Pushing to GitHub"
                wrap_gh_call(
                    gh_repo.ref('heads/{}'.format(gh_branch.name)).update,
                    args=[commit.sha],
                )