Example #1
0
    def delete(self, resources=[], languages=[], skip=False):
        """Delete translations."""
        resource_list = self.get_chosen_resources(resources)
        for resource in resources:
            delete_languages = []
            files = self.get_resource_files(resource)
            project_slug, resource_slug = resource.split('.')
            lang_map = self.get_resource_lang_mapping(resource)
            host = self.get_resource_host(resource)

            url_info = {
                'host': host,
                'project': project_slug,
                'resource': resource_slug
            }
            logger.debug("URL data are: %s" % url_info)

            MSG("Deleting translations for resource %s:" % resource)
            if not languages:
                logger.warning("No languages specified.")
                return
            for language in languages:
                try:
                    self.do_url_request(
                        'delete_translation', url_info, language=language, method="DELETE"
                    )
                    msg = "Deleted language %s from resource %s in project %s."
                    MSG(msg % (language, resource_slug, project_slug))
                except Exception, e:
                    msg = "ERROR: Unable to delete translation %s.%s.%s"
                    MSG(msg % (project_slug, resource_slug, language))
                    if not skip:
                        raise
Example #2
0
    def delete(self, resources=[], languages=[], skip=False, force=False):
        """Delete translations."""
        resource_list = self.get_chosen_resources(resources)
        self.skip = skip
        self.force = force

        if not languages:
            delete_func = self._delete_resource
        else:
            delete_func = self._delete_translations

        for resource in resource_list:
            project_slug, resource_slug = resource.split('.', 1)
            host = self.get_resource_host(resource)
            self.url_info = {
                'host': host,
                'project': project_slug,
                'resource': resource_slug
            }
            logger.debug("URL data are: %s" % self.url_info)
            json, _ = self.do_url_request('project_details', project=self)
            project_details = parse_json(json)
            teams = project_details['teams']
            stats = self._get_stats_for_resource()
            delete_func(project_details, resource, stats, languages)
Example #3
0
    def _get_git_dir(self, path):
        """Check if path lies within a git directory, and return that

            Args:
                path a complete filepath
            Returns:
                (<repotype>, dirname, basename)
        """
        d, t = os.path.split(path)
        h = d

        while h:
            if h in ('/', '/home', '/net'):
                # don't even try these!
                break
            # rQ : are we allowed to go up from self.root ?

            hit = self._git_dirs_cache.get(h,None)
            if hit is False:
                # /negative/ cache
                raise KeyError(h)
            elif hit is None:
                # not cached, we must check further
                if os.path.isdir(os.path.join(h, '.git')):
                    hit = 'git'
                if hit:
                    self._git_dirs_cache[h] = hit
            if hit:
                logger.debug("Hit!, %s is a %s dir", h, hit)
                return hit, d, t
            else:
                h = os.path.split(h)[0]
        
        raise KeyError(d)
Example #4
0
def get_details(api_call, username, password, *args, **kwargs):
    """
    Get the tx project info through the API.

    This function can also be used to check the existence of a project.
    """
    url = (API_URLS[api_call] % (kwargs)).encode('UTF-8')
    conn = urllib3.connection_from_url(kwargs['hostname'])
    headers = urllib3.util.make_headers(
        basic_auth='{0}:{1}'.format(username, password),
        accept_encoding=True,
        user_agent=user_agent_identifier(),
    )
    try:
        r = conn.request('GET', url, headers=headers)
        if r.status < 200 or r.status >= 400:
            raise Exception(r.data)
        remote_project = parse_json(r.data)
        return remote_project
    except ssl.SSLError:
        logger.error("Invalid SSL certificate")
        raise
    except Exception, e:
        logger.debug(unicode(e))
        raise
    def _should_push_translation(self, lang, stats, local_file, force=False):
        """Return whether a local translation file should be
        pushed to Trasnifex.

        We use the following criteria for that:
        - If user requested to force the upload.
        - If language exists in Transifex.
        - If local file is younger than the remote file.

        Args:
            lang: The language code to check.
            stats: The (global) statistics object.
            local_file: The local translation file.
            force: A boolean flag.
        Returns:
            True or False.
        """
        if force:
            logger.debug("Push translation due to -f.")
            return True
        try:
            lang_stats = stats[lang]
        except KeyError, e:
            logger.debug("Language %s does not exist in Transifex." % lang)
            return True
Example #6
0
    def get_chosen_resources(self, resources):
        """Get the resources the user selected.

        Support wildcards in the resources specified by the user.

        Args:
            resources: A list of resources as specified in command-line or
                an empty list.
        Returns:
            A list of resources.
        """
        configured_resources = self.get_resource_list()
        if not resources:
            return configured_resources

        selected_resources = []
        for resource in resources:
            found = False
            for full_name in configured_resources:
                if fnmatch.fnmatch(full_name, resource):
                    selected_resources.append(full_name)
                    found = True
            if not found:
                msg = "Specified resource '%s' does not exist."
                raise Exception(msg % resource)
        logger.debug("Operating on resources: %s" % selected_resources)
        return selected_resources
Example #7
0
    def _init(self, path_to_tx=None):
        # The path to the root of the project, where .tx lives!
        self.root = path_to_tx or find_dot_tx()
        logger.debug("Path to tx is %s." % self.root)
        if not self.root:
            MSG("Cannot find any .tx directory!")
            MSG("Run 'tx init' to initialize your project first!")
            raise ProjectNotInit()

        # The path to the config file (.tx/config)
        self.config_file = os.path.join(self.root, ".tx", "config")
        logger.debug("Config file is %s" % self.config_file)
        # Touch the file if it doesn't exist
        if not os.path.exists(self.config_file):
            MSG("Cannot find the config file (.tx/config)!")
            MSG("Run 'tx init' to fix this!")
            raise ProjectNotInit()

        # The dictionary which holds the config parameters after deser/tion.
        # Read the config in memory
        self.config = OrderedRawConfigParser()
        try:
            self.config.read(self.config_file)
        except Exception, err:
            MSG("WARNING: Cannot open/parse .tx/config file", err)
            MSG("Run 'tx init' to fix this!")
            raise ProjectNotInit()
Example #8
0
 def _get_config_file_path(self, root_path):
     """Check the .tx/config file exists."""
     config_file = os.path.join(root_path, ".tx", "config")
     logger.debug("Config file is %s" % config_file)
     if not os.path.exists(config_file):
         msg = "Cannot find the config file (.tx/config)!"
         raise ProjectNotInit(msg)
     return config_file
Example #9
0
 def _get_tx_dir_path(self, path_to_tx):
     """Check the .tx directory exists."""
     root_path = path_to_tx or find_dot_tx()
     logger.debug("Path to tx is %s." % root_path)
     if not root_path:
         msg = "Cannot find any .tx directory!"
         raise ProjectNotInit(msg)
     return root_path
Example #10
0
    def do_url_request(self, api_call, multipart=False, data=None,
                       files=[], method="GET", **kwargs):
        """
        Issues a url request.
        """
        # Read the credentials from the config file (.transifexrc)
        host = self.url_info['host']
        try:
            username = self.txrc.get(host, 'username')
            passwd = self.txrc.get(host, 'password')
            token = self.txrc.get(host, 'token')
            hostname = self.txrc.get(host, 'hostname')
        except ConfigParser.NoSectionError:
            raise Exception("No user credentials found for host %s. Edit"
                " ~/.transifexrc and add the appropriate info in there." %
                host)

        # Create the Url
        kwargs['hostname'] = hostname
        kwargs.update(self.url_info)
        url = (API_URLS[api_call] % kwargs).encode('UTF-8')
        logger.debug(url)

        if multipart:
            for info, filename in files:
                name = os.path.basename(filename)
                data = {
                    "resource": info.split(';')[0],
                    "language": info.split(';')[1],
                    "uploaded_file": (name, open(filename, 'rb').read())
                }
            headers = urllib3.util.make_headers(
                basic_auth='{0}:{1}'.format(username, passwd),
                accept_encoding=True,
                user_agent=user_agent_identifier(),
                keep_alive=True
            )
            r = self.conn.request(
                method, url, fields=data, headers=headers
            )
        else:
            headers = urllib3.util.make_headers(
                basic_auth='{0}:{1}'.format(username, passwd),
                accept_encoding=True,
                user_agent=user_agent_identifier(),
                keep_alive=True
            )
            r = self.conn.request(
                method, url, fields=data, headers=headers
            )

        r.close()
        if r.status < 200 or r.status >= 400:
            if r.status == 404:
                raise HttpNotFound(r.data)
            else:
                raise Exception(r.data)
        return r.data
Example #11
0
    def do_url_request(self, api_call, multipart=False, data=None,
                       files=[], encoding=None, method="GET", **kwargs):
        """
        Issues a url request.
        """
        # Read the credentials from the config file (.transifexrc)
        host = self.url_info['host']
        try:
            username = self.txrc.get(host, 'username')
            passwd = self.txrc.get(host, 'password')
            token = self.txrc.get(host, 'token')
            hostname = self.txrc.get(host, 'hostname')
        except ConfigParser.NoSectionError:
            raise Exception("No user credentials found for host %s. Edit"
                " ~/.transifexrc and add the appropriate info in there." %
                host)

        # Create the Url
        kwargs['hostname'] = hostname
        kwargs.update(self.url_info)
        url = (API_URLS[api_call] % kwargs).encode('UTF-8')
        logger.debug(url)

        opener = None
        headers = None
        req = None

        if multipart:
            opener = urllib2.build_opener(MultipartPostHandler)
            for info,filename in files:
                data = { "resource" : info.split(';')[0],
                         "language" : info.split(';')[1],
                         "uploaded_file" :  open(filename,'rb') }

            urllib2.install_opener(opener)
            req = RequestWithMethod(url=url, data=data, method=method)
        else:
            req = RequestWithMethod(url=url, data=data, method=method)
            if encoding:
                req.add_header("Content-Type",encoding)

        base64string = base64.encodestring('%s:%s' % (username, passwd))[:-1]
        authheader = "Basic %s" % base64string
        req.add_header("Authorization", authheader)
        req.add_header("Accept-Encoding", "gzip,deflate")
        req.add_header("User-Agent", user_agent_identifier())

        try:
            response = urllib2.urlopen(req, timeout=300)
            return http_response(response)
        except urllib2.HTTPError, e:
            if e.code in [401, 403, 404]:
                raise e
            elif 200 <= e.code < 300:
                return None
            else:
                # For other requests, we should print the message as well
                raise Exception("Remote server replied: %s" % e.read())
 def _get_stats_for_resource(self):
     """Get the statistics information for a resource."""
     try:
         r = self.do_url_request('resource_stats')
         logger.debug("Statistics response is %s" % r)
         stats = parse_json(r)
     except urllib2.HTTPError, e:
         logger.debug("Resource not found: %s" % e)
         stats = {}
Example #13
0
 def _get_stats_for_resource(self, url_info):
     """Get the statistics information for a resource."""
     try:
         r = self.do_url_request('resource_stats', url_info)
         logger.debug("Statistics response is %s" % r)
         stats = parse_json(r)
     except Exception,e:
         logger.debug("Empty statistics: %s" % e)
         stats = {}
Example #14
0
 def _get_stats_for_resource(self):
     """Get the statistics information for a resource."""
     try:
         r = self.do_url_request('resource_stats')
         logger.debug("Statistics response is %s" % r)
         stats = parse_json(r)
     except ssl.SSLError:
         logger.error("Invalid SSL certificate")
         raise
     except Exception, e:
         logger.debug(unicode(e))
         raise
    def _should_download(self, lang, stats, local_file=None, force=False,
                         mode=None):
        """Return whether a translation should be downloaded.

        If local_file is None, skip the timestamps check (the file does
        not exist locally).
        """
        try:
            lang_stats = stats[lang]
        except KeyError, e:
            logger.debug("No lang %s in statistics" % lang)
            return False
Example #16
0
    def _get_transifex_file(self, directory=None):
        """Fetch the path of the .transifexrc file.

        It is in the home directory ofthe user by default.
        """
        if directory is None:
            directory = os.path.expanduser('~')
        txrc_file = os.path.join(directory, ".transifexrc")
        logger.debug(".transifexrc file is at %s" % directory)
        if not os.path.exists(txrc_file):
            msg = "No authentication data found."
            raise ProjectNotInit(msg)
        return txrc_file
Example #17
0
def get_details(api_call, username, password, *args, **kwargs):
    """
    Get the tx project info through the API.

    This function can also be used to check the existence of a project.
    """
    url = API_URLS[api_call] % kwargs
    try:
        data, charset = make_request('GET', kwargs['hostname'], url, username, password)
        return parse_json(data)
    except Exception as e:
        logger.debug(six.u(str(e)))
        raise
def _set_project_option(resource, name, value, path_to_tx, func_name):
    """Save the option to the project config file."""
    if value is None:
        return
    if not resource:
        logger.debug("Setting the %s for all resources." % name)
        resources = []
    else:
        logger.debug("Setting the %s for resource %s." % (name, resource))
        resources = [resource, ]
    prj = project.Project(path_to_tx)
    getattr(prj, func_name)(resources, value)
    prj.save()
    def _get_transifex_file(self, directory=None):
        """Fetch the path of the .transifexrc file.

        It is in the home directory ofthe user by default.
        """
        if directory is None:
            directory = os.path.expanduser('~')
        txrc_file = os.path.join(directory, ".transifexrc")
        logger.debug(".transifexrc file is at %s" % directory)
        if not os.path.exists(txrc_file):
            msg = "No authentication data found."
            logger.info(msg)
            mask = os.umask(077)
            open(txrc_file, 'w').close()
            os.umask(mask)
        return txrc_file
Example #20
0
    def _new_translations_to_add(self, files, slang, lang_map,
                                 stats, force=False):
        """Return a list of translations which are
        new to the local installation.
        """
        new_translations = []
        langs = list(stats.keys())
        logger.debug("Available languages are: %s" % langs)

        for lang in langs:
            lang_exists = lang in list(files.keys())
            lang_is_source = lang == slang
            mapped_lang_exists = (
                lang in lang_map and lang_map[lang] in list(files.keys())
            )
            if lang_exists or lang_is_source or mapped_lang_exists:
                continue
            if self._should_add_translation(lang, stats, force):
                new_translations.append(lang)
        return set(new_translations)
Example #21
0
    def get(self, api_call, *args, **kwargs):
        """
        Performs the GET API call specified by api_call and
        parses the response
        """
        # mock response
        if api_call not in self.VALID_CALLS:
            raise Exception(
                "Tried to perform unsupported API call {}".format(
                    api_call
                )
            )

        hostname = self.hostnames[api_call]
        url = API_URLS[api_call] % kwargs
        url = "{}{}".format(hostname, url)

        try:
            response = requests.get(
                url, auth=HTTPBasicAuth(self.username, self.token)
            )
            response.raise_for_status()
            all_data = response.json()
        except Exception as e:
            logger.debug(six.u(str(e)))
            raise

        next_page = response.links.get('next')
        while next_page:
            try:
                response = requests.get(
                    next_page['url'],
                    auth=HTTPBasicAuth(self.USERNAME, self.token)
                )
                response.raise_for_status()
                all_data.extend(response.json())
                next_page = response.links.get('next')
            except Exception as e:
                logger.debug(six.u(str(e)))
                raise
        return all_data
Example #22
0
    def _should_download(self, lang, stats, local_file=None, force=False,
                         mode=None):
        """Return whether a translation should be downloaded.

        If local_file is None, skip the timestamps check (the file does
        not exist locally).
        """
        try:
            lang_stats = stats[lang]
        except KeyError as e:
            logger.debug("No lang %s in statistics" % lang)
            return False

        satisfies_min = self._satisfies_min_translated(lang_stats, mode)
        if not satisfies_min:
            return False

        if force:
            logger.debug("Downloading translation due to -f")
            return True

        if local_file is not None:
            remote_update = self._extract_updated(lang_stats)
            if not self._remote_is_newer(remote_update, local_file):
                logger.debug("Local is newer than remote for lang %s" % lang)
                return False
        return True
Example #23
0
    def _new_translations_to_add(self,
                                 files,
                                 slang,
                                 lang_map,
                                 stats,
                                 force=False):
        """Return a list of translations which are
        new to the local installation.
        """
        new_translations = []
        langs = list(stats.keys())
        logger.debug("Available languages are: %s" % langs)

        for lang in langs:
            lang_exists = lang in list(files.keys())
            lang_is_source = lang == slang
            mapped_lang_exists = (lang in lang_map
                                  and lang_map[lang] in list(files.keys()))
            if lang_exists or lang_is_source or mapped_lang_exists:
                continue
            if self._should_add_translation(lang, stats, force):
                new_translations.append(lang)
        return set(new_translations)
Example #24
0
    def _remote_is_newer(self, remote_updated, local_file):
        """Check whether the remote translation is newer that the local file.

        Args:
            remote_updated: The date and time the translation was last
                updated remotely.
            local_file: The local file.
        Returns:
            True or False.
        """
        if remote_updated is None:
            logger.debug("No remote time")
            return False
        remote_time = self._generate_timestamp(remote_updated)
        local_time = self._get_time_of_local_file(
            self.get_full_path(local_file)
        )
        logger.debug(
            "Remote time is %s and local %s" % (remote_time, local_time)
        )
        if local_time is not None and remote_time < local_time:
            return False
        return True
Example #25
0
    def get(self, api_call, *args, **kwargs):
        """
        Performs the GET API call specified by api_call and
        parses the response
        """
        # mock response
        if api_call not in self.VALID_CALLS:
            raise Exception(
                "Tried to perform unsupported API call {}".format(api_call))

        hostname = self.hostnames[api_call]
        url = API_URLS[api_call] % kwargs
        url = "{}{}".format(hostname, url)

        try:
            response = requests.get(url,
                                    auth=HTTPBasicAuth(self.username,
                                                       self.token))
            response.raise_for_status()
            all_data = response.json()
        except Exception as e:
            logger.debug(six.u(str(e)))
            raise

        next_page = response.links.get('next')
        while next_page:
            try:
                response = requests.get(next_page['url'],
                                        auth=HTTPBasicAuth(
                                            self.USERNAME, self.token))
                response.raise_for_status()
                all_data.extend(response.json())
                next_page = response.links.get('next')
            except Exception as e:
                logger.debug(six.u(str(e)))
                raise
        return all_data
Example #26
0
    def delete(self, resources=[], languages=[], skip=False, force=False):
        """Delete translations."""
        resource_list = self.get_chosen_resources(resources)
        self.skip = skip
        self.force = force

        if not languages:
            delete_func = self._delete_resource
        else:
            delete_func = self._delete_translations

        for resource in resource_list:
            project_slug, resource_slug = resource.split('.', 1)
            host = self.get_resource_host(resource)
            self.url_info = {
                'host': host,
                'project': project_slug,
                'resource': resource_slug
            }
            logger.debug("URL data are: %s" % self.url_info)
            json, _ = self.do_url_request('project_details', project=self)
            project_details = utils.parse_json(json)
            stats = self._get_stats_for_resource()
            delete_func(project_details, resource, stats, languages)
Example #27
0
 def _get_stats_for_resource(self):
     """Get the statistics information for a resource."""
     try:
         r, charset = self.do_url_request('resource_stats')
         logger.debug("Statistics response is %s" % r)
         stats = parse_json(r)
     except HttpNotFound:
         logger.debug("Resource not found, creating...")
         stats = {}
     except Exception as e:
         logger.debug(six.u(str(e)))
         raise
     return stats
Example #28
0
 def _get_stats_for_resource(self):
     """Get the statistics information for a resource."""
     try:
         r, charset = self.do_url_request('resource_stats')
         logger.debug("Statistics response is %s" % r)
         stats = parse_json(r)
     except HttpNotFound:
         logger.debug("Resource not found, creating...")
         stats = {}
     except Exception as e:
         logger.debug(six.u(str(e)))
         raise
     return stats
Example #29
0
            if translations:
                # Check if given language codes exist
                if not languages:
                    push_languages = files.keys()
                else:
                    push_languages = []
                    f_langs = files.keys()
                    for l in languages:
                        if l in lang_map.keys():
                            l = lang_map[l]
                        push_languages.append(l)
                        if l not in f_langs:
                            ERRMSG(
                                "Warning: No mapping found for language code '%s'."
                                % color_text(l, "RED"))
                logger.debug("Languages to push are %s" % push_languages)

                # Push translation files one by one
                for lang in push_languages:
                    local_lang = lang
                    if lang in lang_map.values():
                        remote_lang = lang_map.flip[lang]
                    else:
                        remote_lang = lang

                    local_file = files[local_lang]

                    kwargs = {
                        'lang': remote_lang,
                        'stats': stats,
                        'local_file': local_file,
Example #30
0
            if translations:
                # Check if given language codes exist
                if not languages:
                    push_languages = files.keys()
                else:
                    push_languages = []
                    f_langs = files.keys()
                    for l in languages:
                        if l in lang_map.keys():
                            l = lang_map[l]
                        push_languages.append(l)
                        if l not in f_langs:
                            msg = "Warning: No mapping found for language code '%s'."
                            logger.error(msg % color_text(l, "RED"))
                logger.debug("Languages to push are %s" % push_languages)

                # Push translation files one by one
                for lang in push_languages:
                    local_lang = lang
                    if lang in lang_map.values():
                        remote_lang = lang_map.flip[lang]
                    else:
                        remote_lang = lang

                    local_file = files[local_lang]

                    kwargs = {
                        'lang': remote_lang,
                        'stats': stats,
                        'local_file': local_file,
Example #31
0
            if translations:
                # Check if given language codes exist
                if not languages:
                    push_languages = files.keys()
                else:
                    push_languages = []
                    f_langs = files.keys()
                    for l in languages:
                        if l in lang_map.keys():
                            l = lang_map[l]
                        push_languages.append(l)
                        if l not in f_langs:
                            ERRMSG("Warning: No mapping found for language code '%s'." %
                                color_text(l,"RED"))
                logger.debug("Languages to push are %s" % push_languages)

                # Push translation files one by one
                for lang in push_languages:
                    local_lang = lang
                    if lang in lang_map.values():
                        remote_lang = lang_map.flip[lang]
                    else:
                        remote_lang = lang

                    local_file = files[local_lang]

                    kwargs = {
                        'lang': remote_lang,
                        'stats': stats,
                        'local_file': local_file,
Example #32
0
    def single_pull(self, languages, resource, overwrite, fetchall, fetchsource, force, skip):
        logger.debug("Handling resource %s" % resource)
        project_slug, resource_slug = resource.split('.')
        files = self.get_resource_files(resource)
        slang = self.get_resource_option(resource, 'source_lang')
        sfile = self.get_resource_option(resource, 'source_file')
        lang_map = self.get_resource_lang_mapping(resource)
        host = self.get_resource_host(resource)
        logger.debug("Language mapping is: %s" % lang_map)
        url_info = {
            'host': host,
            'project': project_slug,
            'resource': resource_slug
        }
        logger.debug("URL data are: %s" % url_info)

        stats = self._get_stats_for_resource(url_info)

        try:
            file_filter = self.config.get(resource, 'file_filter')
        except ConfigParser.NoOptionError:
            file_filter = None

        # Pull source file
        pull_languages = set([])
        new_translations = set([])

        if fetchall:
            new_translations = self._new_translations_to_add(
                files, slang, lang_map, resource, stats, force
            )
            if new_translations:
                MSG("New translations found for the following languages: %s" %
                    ', '.join(new_translations))

        existing, new = self._languages_to_pull(
            languages, files, lang_map, resource, stats, force
        )
        pull_languages |= existing
        new_translations |= new
        logger.debug("Adding to new translations: %s" % new)

        if fetchsource:
            if sfile and slang not in pull_languages:
                pull_languages.append(slang)
            elif slang not in new_translations:
                new_translations.append(slang)

        for lang in pull_languages:
            local_lang = lang
            if lang in lang_map.values():
                remote_lang = lang_map.flip[lang]
            else:
                remote_lang = lang
            if languages and lang not in pull_languages:
                logger.debug("Skipping language %s" % lang)
                continue
            if lang != slang:
                local_file = files.get(lang, None) or files[lang_map[lang]]
            else:
                local_file = sfile
            logger.debug("Using file %s" % local_file)

            kwargs = {
                'lang': remote_lang,
                'stats': stats,
                'local_file': local_file,
                'force': force,
                'resource': resource
            }
            if not self._should_update_translation(**kwargs):
                msg = "Skipping '%s' translation (file: %s)."
                MSG(msg % (color_text(remote_lang, "RED"), local_file))
                continue

            if not overwrite:
                local_file = ("%s.new" % local_file)
            MSG(" -> %s: %s" % (color_text(remote_lang,"RED"), local_file))
            try:
                r = self.do_url_request('pull_file', url_info, language=remote_lang)
            except Exception,e:
                if not skip:
                    raise e
                else:
                    ERRMSG(e)
                    continue
            base_dir = os.path.split(local_file)[0]
            mkdir_p(base_dir)
            fd = open(local_file, 'wb')
            fd.write(r)
            fd.close()
Example #33
0
    def push(self, source=False, translations=False, force=False, resources=[], languages=[],
        skip=False, no_interactive=False):
        """
        Push all the resources
        """
        resource_list = self.get_chosen_resources(resources)
        self.skip = skip
        self.force = force
        for resource in resource_list:
            push_languages = []
            project_slug, resource_slug = resource.split('.', 1)
            files = self.get_resource_files(resource)
            slang = self.get_resource_option(resource, 'source_lang')
            sfile = self.get_source_file(resource)
            lang_map = self.get_resource_lang_mapping(resource)
            host = self.get_resource_host(resource)
            logger.debug("Language mapping is: %s" % lang_map)
            logger.debug("Using host %s" % host)
            self.url_info = {
                'host': host,
                'project': project_slug,
                'resource': resource_slug
            }

            logger.info("Pushing translations for resource %s:" % resource)

            stats = self._get_stats_for_resource()

            if force and not no_interactive:
                answer = input("Warning: By using --force, the uploaded"
                    " files will overwrite remote translations, even if they"
                    " are newer than your uploaded files.\nAre you sure you"
                    " want to continue? [y/N] ")

                if not answer in ["", 'Y', 'y', "yes", 'YES']:
                    return

            if source:
                if sfile is None:
                    logger.error("You don't seem to have a proper source file"
                        " mapping for resource %s. Try without the --source"
                        " option or set a source file first and then try again." %
                        resource)
                    continue
                # Push source file
                try:
                    logger.warning("Pushing source file (%s)" % sfile)
                    if not self._resource_exists(stats):
                        logger.info("Resource does not exist.  Creating...")
                        fileinfo = "%s;%s" % (resource_slug, slang)
                        filename = self.get_full_path(sfile)
                        self._create_resource(resource, project_slug, fileinfo, filename)
                    self.do_url_request(
                        'push_source', multipart=True, method="PUT",
                        files=[(
                                "%s;%s" % (resource_slug, slang)
                                , self.get_full_path(sfile)
                        )],
                    )
                except Exception as e:
                    if isinstance(e, SSLError) or not skip:
                        raise
                    else:
                        logger.error(e)
            else:
                try:
                    self.do_url_request('resource_details')
                except Exception as e:
                    if isinstance(e, SSLError):
                        raise
                    code = getattr(e, 'code', None)
                    if code == 404:
                        msg = "Resource %s doesn't exist on the server."
                        logger.error(msg % resource)
                        continue

            if translations:
                # Check if given language codes exist
                if not languages:
                    push_languages = list(files.keys())
                else:
                    push_languages = []
                    f_langs = list(files.keys())
                    for l in languages:
                        if l in list(lang_map.keys()):
                            l = lang_map[l]
                        push_languages.append(l)
                        if l not in f_langs:
                            msg = "Warning: No mapping found for language code '%s'."
                            logger.error(msg % color_text(l,"RED"))
                logger.debug("Languages to push are %s" % push_languages)

                # Push translation files one by one
                for lang in push_languages:
                    local_lang = lang
                    if lang in list(lang_map.values()):
                        remote_lang = lang_map.flip[lang]
                    else:
                        remote_lang = lang

                    local_file = files[local_lang]

                    kwargs = {
                        'lang': remote_lang,
                        'stats': stats,
                        'local_file': local_file,
                        'force': force,
                    }
                    if not self._should_push_translation(**kwargs):
                        msg = "Skipping '%s' translation (file: %s)."
                        logger.info(msg % (color_text(lang, "RED"), local_file))
                        continue

                    msg = "Pushing '%s' translations (file: %s)"
                    logger.warning(
                         msg % (color_text(remote_lang, "RED"), local_file)
                    )
                    try:
                        self.do_url_request(
                            'push_translation', multipart=True, method='PUT',
                            files=[(
                                    "%s;%s" % (resource_slug, remote_lang),
                                    self.get_full_path(local_file)
                            )], language=remote_lang
                        )
                        logger.debug("Translation %s pushed." % remote_lang)
                    except HttpNotFound:
                        if not source:
                            logger.error("Resource hasn't been created. Try pushing source file.")
                    except Exception as e:
                        if isinstance(e, SSLError) or not skip:
                            raise
                        else:
                            logger.error(e)
Example #34
0
    def pull(self, languages=[], resources=[], overwrite=True, fetchall=False,
        fetchsource=False, force=False, skip=False, minimum_perc=0, mode=None,
        pseudo=False):
        """Pull all translations file from transifex server."""
        self.minimum_perc = minimum_perc
        resource_list = self.get_chosen_resources(resources)

        if mode == 'reviewed':
            url = 'pull_reviewed_file'
        elif mode == 'translator':
            url = 'pull_translator_file'
        elif mode == 'developer':
            url = 'pull_developer_file'
        else:
            url = 'pull_file'

        for resource in resource_list:
            logger.debug("Handling resource %s" % resource)
            self.resource = resource
            project_slug, resource_slug = resource.split('.', 1)
            files = self.get_resource_files(resource)
            slang = self.get_resource_option(resource, 'source_lang')
            sfile = self.get_source_file(resource)
            lang_map = self.get_resource_lang_mapping(resource)
            host = self.get_resource_host(resource)
            logger.debug("Language mapping is: %s" % lang_map)
            if mode is None:
                mode = self._get_option(resource, 'mode')
            self.url_info = {
                'host': host,
                'project': project_slug,
                'resource': resource_slug
            }
            logger.debug("URL data are: %s" % self.url_info)

            stats = self._get_stats_for_resource()

            try:
                file_filter = self.config.get(resource, 'file_filter')
            except configparser.NoOptionError:
                file_filter = None

            # Pull source file
            pull_languages = set([])
            new_translations = set([])

            if pseudo:
                pseudo_file = self._get_pseudo_file(
                    slang, resource, file_filter
                )
                if self._should_download(slang, stats, local_file=pseudo_file):
                    logger.info("Pulling pseudo file for resource %s (%s)." % (
                        resource,
                        color_text(pseudo_file, "RED")
                    ))
                    self._download_pseudo(
                        project_slug, resource_slug, pseudo_file
                    )
                if not languages:
                    continue

            if fetchall:
                new_translations = self._new_translations_to_add(
                    files, slang, lang_map, stats, force
                )
                if new_translations:
                    msg = "New translations found for the following languages: %s"
                    logger.info(msg % ', '.join(new_translations))

            existing, new = self._languages_to_pull(
                languages, files, lang_map, stats, force
            )
            pull_languages |= existing
            new_translations |= new
            logger.debug("Adding to new translations: %s" % new)

            if fetchsource:
                if sfile and slang not in pull_languages:
                    pull_languages.add(slang)
                elif slang not in new_translations:
                    new_translations.add(slang)

            if pull_languages:
                logger.debug("Pulling languages for: %s" % pull_languages)
                msg = "Pulling translations for resource %s (source: %s)"
                logger.info(msg % (resource, sfile))

            for lang in pull_languages:
                local_lang = lang
                if lang in list(lang_map.values()):
                    remote_lang = lang_map.flip[lang]
                else:
                    remote_lang = lang
                if languages and lang not in pull_languages:
                    logger.debug("Skipping language %s" % lang)
                    continue
                if lang != slang:
                    local_file = files.get(lang, None) or files[lang_map[lang]]
                else:
                    local_file = sfile
                logger.debug("Using file %s" % local_file)

                kwargs = {
                    'lang': remote_lang,
                    'stats': stats,
                    'local_file': local_file,
                    'force': force,
                    'mode': mode,
                }
                if not self._should_update_translation(**kwargs):
                    msg = "Skipping '%s' translation (file: %s)."
                    logger.info(
                        msg % (color_text(remote_lang, "RED"), local_file)
                    )
                    continue

                if not overwrite:
                    local_file = ("%s.new" % local_file)
                logger.warning(
                    " -> %s: %s" % (color_text(remote_lang, "RED"), local_file)
                )
                try:
                    r, charset = self.do_url_request(url, language=remote_lang)
                except Exception as e:
                    if isinstance(e, SSLError) or not skip:
                        raise
                    else:
                        logger.error(e)
                        continue
                base_dir = os.path.split(local_file)[0]
                mkdir_p(base_dir)
                fd = open(local_file, 'wb')
                fd.write(r.encode(charset))
                fd.close()

            if new_translations:
                msg = "Pulling new translations for resource %s (source: %s)"
                logger.info(msg % (resource, sfile))
                for lang in new_translations:
                    if lang in list(lang_map.keys()):
                        local_lang = lang_map[lang]
                    else:
                        local_lang = lang
                    remote_lang = lang
                    if file_filter:
                        local_file = os.path.relpath(
                            os.path.join(
                                self.root, native_path(
                                    file_filter.replace('<lang>', local_lang)
                                )
                            ), os.curdir
                        )
                    else:
                        trans_dir = os.path.join(self.root, ".tx", resource)
                        if not os.path.exists(trans_dir):
                            os.mkdir(trans_dir)
                        local_file = os.path.relpath(os.path.join(trans_dir, '%s_translation' %
                            local_lang, os.curdir))

                    if lang != slang:
                        satisfies_min = self._satisfies_min_translated(
                            stats[remote_lang], mode
                        )
                        if not satisfies_min:
                            msg = "Skipping language %s due to used options."
                            logger.info(msg % lang)
                            continue
                    logger.warning(
                        " -> %s: %s" % (color_text(remote_lang, "RED"), local_file)
                    )

                    r, charset = self.do_url_request(url, language=remote_lang)
                    base_dir = os.path.split(local_file)[0]
                    mkdir_p(base_dir)
                    fd = open(local_file, 'wb')
                    fd.write(r.encode(charset))
                    fd.close()
            if translations:
                # Check if given language codes exist
                if not languages:
                    push_languages = files.keys()
                else:
                    push_languages = []
                    f_langs = files.keys()
                    for l in languages:
                        if l in lang_map.keys():
                            l = lang_map[l]
                        push_languages.append(l)
                        if l not in f_langs:
                            msg = "Warning: No mapping found for language code '%s'."
                            logger.error(msg % color_text(l,"RED"))
                logger.debug("Languages to push are %s" % push_languages)

                # Push translation files one by one
                for lang in push_languages:
                    local_lang = lang
                    if lang in lang_map.values():
                        remote_lang = lang_map.flip[lang]
                    else:
                        remote_lang = lang

                    local_file = files[local_lang]

                    kwargs = {
                        'lang': remote_lang,
                        'stats': stats,
                        'local_file': local_file,
Example #36
0
    def push(self,
             source=False,
             translations=False,
             force=False,
             resources=[],
             languages=[],
             skip=False,
             no_interactive=False):
        """
        Push all the resources
        """
        resource_list = self.get_chosen_resources(resources)
        self.skip = skip
        self.force = force
        for resource in resource_list:
            push_languages = []
            project_slug, resource_slug = resource.split('.')
            files = self.get_resource_files(resource)
            slang = self.get_resource_option(resource, 'source_lang')
            sfile = self.get_source_file(resource)
            lang_map = self.get_resource_lang_mapping(resource)
            host = self.get_resource_host(resource)
            verify_ssl(host)
            logger.debug("Language mapping is: %s" % lang_map)
            logger.debug("Using host %s" % host)
            self.url_info = {
                'host': host,
                'project': project_slug,
                'resource': resource_slug
            }

            logger.info("Pushing translations for resource %s:" % resource)

            stats = self._get_stats_for_resource()

            if force and not no_interactive:
                answer = raw_input(
                    "Warning: By using --force, the uploaded"
                    " files will overwrite remote translations, even if they"
                    " are newer than your uploaded files.\nAre you sure you"
                    " want to continue? [y/N] ")

                if not answer in ["", 'Y', 'y', "yes", 'YES']:
                    return

            if source:
                if sfile is None:
                    logger.error(
                        "You don't seem to have a proper source file"
                        " mapping for resource %s. Try without the --source"
                        " option or set a source file first and then try again."
                        % resource)
                    continue
                # Push source file
                try:
                    logger.warning("Pushing source file (%s)" % sfile)
                    if not self._resource_exists(stats):
                        logger.info("Resource does not exist.  Creating...")
                        fileinfo = "%s;%s" % (resource_slug, slang)
                        filename = self.get_full_path(sfile)
                        self._create_resource(resource, project_slug, fileinfo,
                                              filename)
                    self.do_url_request(
                        'push_source',
                        multipart=True,
                        method="PUT",
                        files=[("%s;%s" % (resource_slug, slang),
                                self.get_full_path(sfile))],
                    )
                except Exception, e:
                    if not skip:
                        raise
                    else:
                        logger.error(e)
            else:
                try:
                    self.do_url_request('resource_details')
                except Exception, e:
                    code = getattr(e, 'code', None)
                    if code == 404:
                        msg = "Resource %s doesn't exist on the server."
                        logger.error(msg % resource)
                        continue
Example #37
0
    def pull(self,
             languages=[],
             resources=[],
             overwrite=True,
             fetchall=False,
             fetchsource=False,
             force=False,
             skip=False,
             minimum_perc=0,
             mode=None,
             pseudo=False):
        """Pull all translations file from transifex server."""
        self.minimum_perc = minimum_perc
        resource_list = self.get_chosen_resources(resources)
        skip_decode = False

        if mode == 'reviewed':
            url = 'pull_reviewed_file'
        elif mode == 'translator':
            url = 'pull_translator_file'
        elif mode == 'developer':
            url = 'pull_developer_file'
        elif mode == 'onlytranslated':
            url = 'pull_onlytranslated_file'
        elif mode == 'onlyreviewed':
            url = 'pull_onlyreviewed_file'
        else:
            url = 'pull_file'

        for resource in resource_list:
            logger.debug("Handling resource %s" % resource)
            self.resource = resource
            project_slug, resource_slug = resource.split('.', 1)
            files = self.get_resource_files(resource)
            slang = self.get_resource_option(resource, 'source_lang')
            sfile = self.get_source_file(resource)
            lang_map = self.get_resource_lang_mapping(resource)
            host = self.get_resource_host(resource)
            logger.debug("Language mapping is: %s" % lang_map)
            if mode is None:
                mode = self._get_option(resource, 'mode')
            self.url_info = {
                'host': host,
                'project': project_slug,
                'resource': resource_slug
            }
            logger.debug("URL data are: %s" % self.url_info)

            stats = self._get_stats_for_resource()
            details_response, _ = self.do_url_request('resource_details')
            details = utils.parse_json(details_response)
            if details['i18n_type'] in self.SKIP_DECODE_I18N_TYPES:
                skip_decode = True
            try:
                file_filter = self.config.get(resource, 'file_filter')
            except configparser.NoOptionError:
                file_filter = None

            # Pull source file
            pull_languages = set([])
            new_translations = set([])

            if pseudo:
                pseudo_file = self._get_pseudo_file(slang, resource,
                                                    file_filter)
                if self._should_download(slang, stats, local_file=pseudo_file):
                    logger.info(
                        "Pulling pseudo file for resource %s (%s)." %
                        (resource, utils.color_text(pseudo_file, "RED")))
                    self._download_pseudo(project_slug, resource_slug,
                                          pseudo_file)
                if not languages:
                    continue

            if fetchall:
                new_translations = self._new_translations_to_add(
                    files, slang, lang_map, stats, force)
                if new_translations:
                    msg = ("New translations found "
                           "for the following languages:%s")
                    logger.info(msg % ', '.join(new_translations))

            existing, new = self._languages_to_pull(languages, files, lang_map,
                                                    stats, force)
            pull_languages |= existing
            new_translations |= new
            logger.debug("Adding to new translations: %s" % new)

            if fetchsource:
                if sfile and slang not in pull_languages:
                    pull_languages.add(slang)
                elif slang not in new_translations:
                    new_translations.add(slang)

            if pull_languages:
                logger.debug("Pulling languages for: %s" % pull_languages)
                msg = "Pulling translations for resource %s (source: %s)"
                logger.info(msg % (resource, sfile))

            for lang in pull_languages:
                local_lang = lang
                if lang in list(lang_map.values()):
                    remote_lang = lang_map.flip[lang]
                else:
                    remote_lang = lang
                if languages and lang not in pull_languages:
                    logger.debug("Skipping language %s" % lang)
                    continue
                if lang != slang:
                    local_file = files.get(lang, None) or files[lang_map[lang]]
                else:
                    local_file = sfile
                logger.debug("Using file %s" % local_file)

                kwargs = {
                    'lang': remote_lang,
                    'stats': stats,
                    'local_file': local_file,
                    'force': force,
                    'mode': mode,
                }
                if not self._should_update_translation(**kwargs):
                    msg = "Skipping '%s' translation (file: %s)."
                    logger.info(
                        msg %
                        (utils.color_text(remote_lang, "RED"), local_file))
                    continue

                if not overwrite:
                    local_file = ("%s.new" % local_file)
                logger.warning(
                    " -> %s: %s" %
                    (utils.color_text(remote_lang, "RED"), local_file))
                try:
                    r, charset = self.do_url_request(url,
                                                     language=remote_lang,
                                                     skip_decode=skip_decode)
                except Exception as e:
                    if isinstance(e, SSLError) or not skip:
                        raise
                    else:
                        logger.error(e)
                        continue
                self._save_file(local_file, charset, r)

            if new_translations:
                msg = "Pulling new translations for resource %s (source: %s)"
                logger.info(msg % (resource, sfile))
                for lang in new_translations:
                    if lang in list(lang_map.keys()):
                        local_lang = lang_map[lang]
                    else:
                        local_lang = lang
                    remote_lang = lang
                    if file_filter:
                        local_file = os.path.relpath(
                            os.path.join(
                                self.root,
                                native_path(
                                    file_filter.replace('<lang>',
                                                        local_lang))),
                            os.curdir)
                    else:
                        trans_dir = os.path.join(self.root, ".tx", resource)
                        if not os.path.exists(trans_dir):
                            os.mkdir(trans_dir)
                        local_file = os.path.relpath(
                            os.path.join(trans_dir,
                                         '%s_translation' % local_lang,
                                         os.curdir))

                    if lang != slang:
                        satisfies_min = self._satisfies_min_translated(
                            stats[remote_lang], mode)
                        if not satisfies_min:
                            msg = "Skipping language %s due to used options."
                            logger.info(msg % lang)
                            continue
                    logger.warning(
                        " -> %s: %s" %
                        (utils.color_text(remote_lang, "RED"), local_file))

                    r, charset = self.do_url_request(url,
                                                     language=remote_lang,
                                                     skip_decode=skip_decode)
                    self._save_file(local_file, charset, r)
    def push(self, source=False, translations=False, force=False, resources=[], languages=[],
        skip=False, no_interactive=False):
        """
        Push all the resources
        """
        resource_list = self.get_chosen_resources(resources)
        self.skip = skip
        self.force = force
        for resource in resource_list:
            push_languages = []
            project_slug, resource_slug = resource.split('.')
            files = self.get_resource_files(resource)
            slang = self.get_resource_option(resource, 'source_lang')
            sfile = self.get_resource_option(resource, 'source_file')
            lang_map = self.get_resource_lang_mapping(resource)
            host = self.get_resource_host(resource)
            logger.debug("Language mapping is: %s" % lang_map)
            logger.debug("Using host %s" % host)
            self.url_info = {
                'host': host,
                'project': project_slug,
                'resource': resource_slug
            }

            logger.info("Pushing translations for resource %s:" % resource)

            stats = self._get_stats_for_resource()

            if force and not no_interactive:
                answer = raw_input("Warning: By using --force, the uploaded"
                    " files will overwrite remote translations, even if they"
                    " are newer than your uploaded files.\nAre you sure you"
                    " want to continue? [y/N] ")

                if not answer in ["", 'Y', 'y', "yes", 'YES']:
                    return

            if source:
                if sfile == None:
                    logger.error("You don't seem to have a proper source file"
                        " mapping for resource %s. Try without the --source"
                        " option or set a source file first and then try again." %
                        resource)
                    continue
                # Push source file
                try:
                    logger.warning("Pushing source file (%s)" % sfile)
                    if not self._resource_exists(stats):
                        logger.info("Resource does not exist.  Creating...")
                        fileinfo = "%s;%s" % (resource_slug, slang)
                        filename = self.get_full_path(sfile)
                        self._create_resource(resource, project_slug, fileinfo, filename)
                    self.do_url_request(
                        'push_source', multipart=True, method="PUT",
                        files=[(
                                "%s;%s" % (resource_slug, slang)
                                , self.get_full_path(sfile)
                        )],
                    )
                except Exception, e:
                    if not skip:
                        raise e
                    else:
                        logger.error(e)
            else:
                try:
                    self.do_url_request('resource_details')
                except Exception, e:
                    code = getattr(e, 'code', None)
                    if code == 404:
                        msg = "Resource %s doesn't exist on the server."
                        logger.error(msg % resource)
                        continue
Example #39
0
    def push(self,
             source=False,
             translations=False,
             force=False,
             resources=[],
             languages=[],
             skip=False,
             no_interactive=False):
        """
        Push all the resources
        """
        resource_list = self.get_chosen_resources(resources)
        self.skip = skip
        self.force = force
        for resource in resource_list:
            push_languages = []
            project_slug, resource_slug = resource.split('.', 1)
            files = self.get_resource_files(resource)
            slang = self.get_resource_option(resource, 'source_lang')
            sfile = self.get_source_file(resource)
            lang_map = self.get_resource_lang_mapping(resource)
            host = self.get_resource_host(resource)
            logger.debug("Language mapping is: %s" % lang_map)
            logger.debug("Using host %s" % host)
            self.url_info = {
                'host': host,
                'project': project_slug,
                'resource': resource_slug
            }

            logger.info("Pushing translations for resource %s:" % resource)

            stats = self._get_stats_for_resource()

            if force and not no_interactive:
                answer = input(
                    "Warning: By using --force, the uploaded"
                    " files will overwrite remote translations, even if they"
                    " are newer than your uploaded files.\nAre you sure you"
                    " want to continue? [y/N] ")

                if not answer in ["", 'Y', 'y', "yes", 'YES']:
                    return

            if source:
                if sfile is None:
                    logger.error(
                        "You don't seem to have a proper source file"
                        " mapping for resource %s. Try without the --source"
                        " option or set a source file first and then try again."
                        % resource)
                    continue
                # Push source file
                try:
                    logger.warning("Pushing source file (%s)" % sfile)
                    if not self._resource_exists(stats):
                        logger.info("Resource does not exist.  Creating...")
                        fileinfo = "%s;%s" % (resource_slug, slang)
                        filename = self.get_full_path(sfile)
                        self._create_resource(resource, project_slug, fileinfo,
                                              filename)
                    self.do_url_request(
                        'push_source',
                        multipart=True,
                        method="PUT",
                        files=[("%s;%s" % (resource_slug, slang),
                                self.get_full_path(sfile))],
                    )
                except Exception as e:
                    if isinstance(e, SSLError) or not skip:
                        raise
                    else:
                        logger.error(e)
            else:
                try:
                    self.do_url_request('resource_details')
                except Exception as e:
                    if isinstance(e, SSLError):
                        raise
                    code = getattr(e, 'code', None)
                    if code == 404:
                        msg = "Resource %s doesn't exist on the server."
                        logger.error(msg % resource)
                        continue

            if translations:
                # Check if given language codes exist
                if not languages:
                    push_languages = list(files.keys())
                else:
                    push_languages = []
                    f_langs = list(files.keys())
                    for l in languages:
                        if l in list(lang_map.keys()):
                            l = lang_map[l]
                        push_languages.append(l)
                        if l not in f_langs:
                            msg = "Warning: No mapping found for language code '%s'."
                            logger.error(msg % color_text(l, "RED"))
                logger.debug("Languages to push are %s" % push_languages)

                # Push translation files one by one
                for lang in push_languages:
                    local_lang = lang
                    if lang in list(lang_map.values()):
                        remote_lang = lang_map.flip[lang]
                    else:
                        remote_lang = lang

                    local_file = files[local_lang]

                    kwargs = {
                        'lang': remote_lang,
                        'stats': stats,
                        'local_file': local_file,
                        'force': force,
                    }
                    if not self._should_push_translation(**kwargs):
                        msg = "Skipping '%s' translation (file: %s)."
                        logger.info(msg %
                                    (color_text(lang, "RED"), local_file))
                        continue

                    msg = "Pushing '%s' translations (file: %s)"
                    logger.warning(
                        msg % (color_text(remote_lang, "RED"), local_file))
                    try:
                        self.do_url_request(
                            'push_translation',
                            multipart=True,
                            method='PUT',
                            files=[("%s;%s" % (resource_slug, remote_lang),
                                    self.get_full_path(local_file))],
                            language=remote_lang)
                        logger.debug("Translation %s pushed." % remote_lang)
                    except HttpNotFound:
                        if not source:
                            logger.error(
                                "Resource hasn't been created. Try pushing source file."
                            )
                    except Exception as e:
                        if isinstance(e, SSLError) or not skip:
                            raise
                        else:
                            logger.error(e)
Example #40
0
    def pull(self,
             languages=[],
             resources=[],
             overwrite=True,
             fetchall=False,
             fetchsource=False,
             force=False,
             skip=False,
             minimum_perc=0,
             mode=None):
        """Pull all translations file from transifex server."""
        self.minimum_perc = minimum_perc
        resource_list = self.get_chosen_resources(resources)

        if mode == 'reviewed':
            url = 'pull_reviewed_file'
        elif mode == 'translator':
            url = 'pull_translator_file'
        elif mode == 'developer':
            url = 'pull_developer_file'
        else:
            url = 'pull_file'

        for resource in resource_list:
            logger.debug("Handling resource %s" % resource)
            self.resource = resource
            project_slug, resource_slug = resource.split('.')
            files = self.get_resource_files(resource)
            slang = self.get_resource_option(resource, 'source_lang')
            sfile = self.get_source_file(resource)
            lang_map = self.get_resource_lang_mapping(resource)
            host = self.get_resource_host(resource)
            verify_ssl(host)
            logger.debug("Language mapping is: %s" % lang_map)
            if mode is None:
                mode = self._get_option(resource, 'mode')
            self.url_info = {
                'host': host,
                'project': project_slug,
                'resource': resource_slug
            }
            logger.debug("URL data are: %s" % self.url_info)

            stats = self._get_stats_for_resource()

            try:
                file_filter = self.config.get(resource, 'file_filter')
            except ConfigParser.NoOptionError:
                file_filter = None

            # Pull source file
            pull_languages = set([])
            new_translations = set([])

            if fetchall:
                new_translations = self._new_translations_to_add(
                    files, slang, lang_map, stats, force)
                if new_translations:
                    msg = "New translations found for the following languages: %s"
                    logger.info(msg % ', '.join(new_translations))

            existing, new = self._languages_to_pull(languages, files, lang_map,
                                                    stats, force)
            pull_languages |= existing
            new_translations |= new
            logger.debug("Adding to new translations: %s" % new)

            if fetchsource:
                if sfile and slang not in pull_languages:
                    pull_languages.add(slang)
                elif slang not in new_translations:
                    new_translations.add(slang)

            if pull_languages:
                logger.debug("Pulling languages for: %s" % pull_languages)
                msg = "Pulling translations for resource %s (source: %s)"
                logger.info(msg % (resource, sfile))

            for lang in pull_languages:
                local_lang = lang
                if lang in lang_map.values():
                    remote_lang = lang_map.flip[lang]
                else:
                    remote_lang = lang
                if languages and lang not in pull_languages:
                    logger.debug("Skipping language %s" % lang)
                    continue
                if lang != slang:
                    local_file = files.get(lang, None) or files[lang_map[lang]]
                else:
                    local_file = sfile
                logger.debug("Using file %s" % local_file)

                kwargs = {
                    'lang': remote_lang,
                    'stats': stats,
                    'local_file': local_file,
                    'force': force,
                    'mode': mode,
                }
                if not self._should_update_translation(**kwargs):
                    msg = "Skipping '%s' translation (file: %s)."
                    logger.info(msg %
                                (color_text(remote_lang, "RED"), local_file))
                    continue

                if not overwrite:
                    local_file = ("%s.new" % local_file)
                logger.warning(" -> %s: %s" %
                               (color_text(remote_lang, "RED"), local_file))
                try:
                    r = self.do_url_request(url, language=remote_lang)
                except Exception, e:
                    if not skip:
                        raise e
                    else:
                        logger.error(e)
                        continue
                base_dir = os.path.split(local_file)[0]
                mkdir_p(base_dir)
                fd = open(local_file, 'wb')
                fd.write(r)
                fd.close()

            if new_translations:
                msg = "Pulling new translations for resource %s (source: %s)"
                logger.info(msg % (resource, sfile))
                for lang in new_translations:
                    if lang in lang_map.keys():
                        local_lang = lang_map[lang]
                    else:
                        local_lang = lang
                    remote_lang = lang
                    if file_filter:
                        local_file = os.path.relpath(
                            os.path.join(
                                self.root,
                                native_path(
                                    file_filter.replace('<lang>',
                                                        local_lang))),
                            os.curdir.decode('utf-8'))
                    else:
                        trans_dir = os.path.join(self.root, ".tx", resource)
                        if not os.path.exists(trans_dir):
                            os.mkdir(trans_dir)
                        local_file = os.path.relpath(
                            os.path.join(trans_dir,
                                         '%s_translation' % local_lang,
                                         os.curdir))

                    if lang != slang:
                        satisfies_min = self._satisfies_min_translated(
                            stats[remote_lang], mode)
                        if not satisfies_min:
                            msg = "Skipping language %s due to used options."
                            logger.info(msg % lang)
                            continue
                    logger.warning(
                        " -> %s: %s" %
                        (color_text(remote_lang, "RED"), local_file))
                    r = self.do_url_request(url, language=remote_lang)

                    base_dir = os.path.split(local_file)[0]
                    mkdir_p(base_dir)
                    fd = open(local_file, 'wb')
                    fd.write(r)
                    fd.close()
Example #41
0
    def single_pull(self, languages, resource, overwrite, fetchall,
                    fetchsource, force, skip):
        logger.debug("Handling resource %s" % resource)
        project_slug, resource_slug = resource.split('.')
        files = self.get_resource_files(resource)
        slang = self.get_resource_option(resource, 'source_lang')
        sfile = self.get_resource_option(resource, 'source_file')
        lang_map = self.get_resource_lang_mapping(resource)
        host = self.get_resource_host(resource)
        logger.debug("Language mapping is: %s" % lang_map)
        url_info = {
            'host': host,
            'project': project_slug,
            'resource': resource_slug
        }
        logger.debug("URL data are: %s" % url_info)

        stats = self._get_stats_for_resource(url_info)

        try:
            file_filter = self.config.get(resource, 'file_filter')
        except ConfigParser.NoOptionError:
            file_filter = None

        # Pull source file
        pull_languages = set([])
        new_translations = set([])

        if fetchall:
            new_translations = self._new_translations_to_add(
                files, slang, lang_map, resource, stats, force)
            if new_translations:
                MSG("New translations found for the following languages: %s" %
                    ', '.join(new_translations))

        existing, new = self._languages_to_pull(languages, files, lang_map,
                                                resource, stats, force)
        pull_languages |= existing
        new_translations |= new
        logger.debug("Adding to new translations: %s" % new)

        if fetchsource:
            if sfile and slang not in pull_languages:
                pull_languages.append(slang)
            elif slang not in new_translations:
                new_translations.append(slang)

        for lang in pull_languages:
            local_lang = lang
            if lang in lang_map.values():
                remote_lang = lang_map.flip[lang]
            else:
                remote_lang = lang
            if languages and lang not in pull_languages:
                logger.debug("Skipping language %s" % lang)
                continue
            if lang != slang:
                local_file = files.get(lang, None) or files[lang_map[lang]]
            else:
                local_file = sfile
            logger.debug("Using file %s" % local_file)

            kwargs = {
                'lang': remote_lang,
                'stats': stats,
                'local_file': local_file,
                'force': force,
                'resource': resource
            }
            if not self._should_update_translation(**kwargs):
                msg = "Skipping '%s' translation (file: %s)."
                MSG(msg % (color_text(remote_lang, "RED"), local_file))
                continue

            if not overwrite:
                local_file = ("%s.new" % local_file)
            MSG(" -> %s: %s" % (color_text(remote_lang, "RED"), local_file))
            try:
                r = self.do_url_request('pull_file',
                                        url_info,
                                        language=remote_lang)
            except Exception, e:
                if not skip:
                    raise e
                else:
                    ERRMSG(e)
                    continue
            base_dir = os.path.split(local_file)[0]
            mkdir_p(base_dir)
            fd = open(local_file, 'wb')
            fd.write(r)
            fd.close()
Example #42
0
    def do_url_request(self,
                       api_call,
                       multipart=False,
                       data=None,
                       files=[],
                       encoding=None,
                       method="GET",
                       **kwargs):
        """
        Issues a url request.
        """
        # Read the credentials from the config file (.transifexrc)
        host = self.url_info['host']
        try:
            username = self.txrc.get(host, 'username')
            passwd = self.txrc.get(host, 'password')
            token = self.txrc.get(host, 'token')
            hostname = self.txrc.get(host, 'hostname')
        except ConfigParser.NoSectionError:
            raise Exception(
                "No user credentials found for host %s. Edit"
                " ~/.transifexrc and add the appropriate info in there." %
                host)

        # Create the Url
        kwargs['hostname'] = hostname
        kwargs.update(self.url_info)
        url = (API_URLS[api_call] % kwargs).encode('UTF-8')
        logger.debug(url)

        opener = None
        headers = None
        req = None

        if multipart:
            opener = urllib2.build_opener(MultipartPostHandler)
            for info, filename in files:
                data = {
                    "resource": info.split(';')[0],
                    "language": info.split(';')[1],
                    "uploaded_file": open(filename, 'rb')
                }

            urllib2.install_opener(opener)
            req = RequestWithMethod(url=url, data=data, method=method)
        else:
            req = RequestWithMethod(url=url, data=data, method=method)
            if encoding:
                req.add_header("Content-Type", encoding)

        base64string = base64.encodestring('%s:%s' % (username, passwd))[:-1]
        authheader = "Basic %s" % base64string
        req.add_header("Authorization", authheader)
        req.add_header("Accept-Encoding", "gzip,deflate")
        req.add_header("User-Agent", user_agent_identifier())

        try:
            response = urllib2.urlopen(req, timeout=300)
            return http_response(response)
        except urllib2.HTTPError, e:
            if e.code in [401, 403, 404]:
                raise e
            elif 200 <= e.code < 300:
                return None
            else:
                # For other requests, we should print the message as well
                raise Exception("Remote server replied: %s" % e.read())