Beispiel #1
0
def store_l20n(entities,
               resource,
               key,
               string,
               comment,
               locale,
               order,
               string_plural="",
               plural_form=None):
    """Store .l20n entity or translation."""

    if entities:
        save_entity(resource=resource,
                    string=string,
                    string_plural=string_plural,
                    comment=comment,
                    key=key,
                    order=order)

    else:
        try:
            e = Entity.objects.get(resource=resource, key=key)
            save_translation(entity=e,
                             locale=locale,
                             string=string,
                             plural_form=plural_form)
        except Entity.DoesNotExist:
            pass
Beispiel #2
0
def extract_lang(project, locale, path, entities=False):
    """Extract .lang file with path and save or update in DB."""

    lang = parse_lang(path)
    relative_path = get_relative_path(path, locale)

    resource, created = Resource.objects.get_or_create(
        project=project, path=relative_path, format='lang')

    if entities:
        for order, (key, value) in enumerate(lang):
            save_entity(
                resource=resource, string=key, comment=value[1], order=order)

        update_entity_count(resource)

    else:
        for key, value in lang:
            if key != value[2] or '{ok}' in value[3]:
                try:
                    e = Entity.objects.get(resource=resource, string=key)
                    save_translation(
                        entity=e, locale=locale, string=value[2])

                except Entity.DoesNotExist:
                    continue

        update_stats(resource, locale)

    log.debug("[" + locale.code + "]: " + path + " saved to DB.")
Beispiel #3
0
def extract_ini(project, path):
    """Extract .ini file with path and save or update in DB."""

    config = configparser.ConfigParser()
    with codecs.open(path, 'r', 'utf-8') as f:
        try:
            config.read_file(f)
        except Exception as e:
            log.debug("INI configparser: " + str(e))

    sections = config.sections()

    source_locale = None
    for s in ('templates', 'en-US', 'en-GB', 'en'):
        if s in sections:
            source_locale = s
            break
    if source_locale is None:
        log.error("Unable to detect source locale")
        raise Exception("error")

    # Move source locale on top, so we save entities first, then translations
    sections.insert(0, sections.pop(sections.index(source_locale)))

    resource, created = Resource.objects.get_or_create(project=project,
                                                       path=path,
                                                       format='ini')

    for section in sections:
        try:
            locale = Locale.objects.get(code=section)
        except Locale.DoesNotExist:
            log.debug("Locale not supported: " + section)
            break

        order = 0
        for item in config.items(section):
            if section == source_locale:
                save_entity(resource=resource,
                            string=item[1],
                            key=item[0],
                            order=order)
                order += 1
            else:
                try:
                    e = Entity.objects.get(resource=resource, key=item[0])
                    save_translation(entity=e, locale=locale, string=item[1])
                except Entity.DoesNotExist:
                    log.debug("[" + section + "]: line ID " + item[0] +
                              " is obsolete.")
                    continue

        if section == source_locale:
            update_entity_count(resource)
        else:
            update_stats(resource, locale)

        log.debug("[" + section + "]: saved to DB.")
Beispiel #4
0
def extract_ini(project, path):
    """Extract .ini file with path and save or update in DB."""

    config = configparser.ConfigParser()
    with codecs.open(path, 'r', 'utf-8') as f:
        try:
            config.read_file(f)
        except Exception as e:
            log.debug("INI configparser: " + str(e))

    sections = config.sections()

    source_locale = None
    for s in ('templates', 'en-US', 'en-GB', 'en'):
        if s in sections:
            source_locale = s
            break
    if source_locale is None:
        log.error("Unable to detect source locale")
        raise Exception("error")

    # Move source locale on top, so we save entities first, then translations
    sections.insert(0, sections.pop(sections.index(source_locale)))

    resource, created = Resource.objects.get_or_create(
        project=project, path=path, format='ini')

    for section in sections:
        try:
            locale = Locale.objects.get(code=section)
        except Locale.DoesNotExist:
            log.debug("Locale not supported: " + section)
            break

        order = 0
        for item in config.items(section):
            if section == source_locale:
                save_entity(resource=resource, string=item[1],
                            key=item[0], order=order)
                order += 1
            else:
                try:
                    e = Entity.objects.get(
                        resource=resource, key=item[0])
                    save_translation(
                        entity=e, locale=locale, string=item[1])
                except Entity.DoesNotExist:
                    log.debug("[" + section + "]: line ID " +
                              item[0] + " is obsolete.")
                    continue

        if section == source_locale:
            update_entity_count(resource)
        else:
            update_stats(resource, locale)

        log.debug("[" + section + "]: saved to DB.")
Beispiel #5
0
def extract_inc(project, locale, path, entities=False):
    """Extract .inc file with path and save or update in DB."""

    with codecs.open(path, 'r', 'utf-8', errors='replace') as inc_file:

        comment = []
        order = 0
        relative_path = get_relative_path(path, locale)
        resource, created = Resource.objects.get_or_create(
            project=project, path=relative_path, format='inc')

        for line in inc_file:

            # Uncomment MOZ_LANGPACK_CONTRIBUTORS (commented out)
            # http://hg.mozilla.org/releases/mozilla-aurora/file/572c8f4c8fed/browser/locales/en-US/defines.inc#l10
            if entities and line.startswith('# #define'):
                line = line.lstrip('#').strip()

            # Comments
            if entities and line.startswith('# '):
                comment.append(line.lstrip('# ').strip())

            # Strings
            elif line.startswith('#define'):
                parts = line.lstrip('#define').strip().split(None, 1)

                if not parts:
                    continue
                if len(parts) == 1:
                    key, string = parts[0], ""
                else:
                    key, string = parts

                if entities:
                    save_entity(resource=resource, string=string, key=key,
                                comment=" ".join(comment), order=order)
                    comment = []
                    order += 1

                else:
                    try:
                        e = Entity.objects.get(resource=resource, key=key)
                        save_translation(
                            entity=e, locale=locale, string=string)
                    except Entity.DoesNotExist:
                        continue

            elif entities:
                comment = []

        if entities:
            update_entity_count(resource)
        else:
            update_stats(resource, locale)

        log.debug("[" + locale.code + "]: " + path + " saved to DB.")
Beispiel #6
0
def extract_silme(parser, project, locale, path, entities=False):
    """Extract file with path using silme and save or update in DB."""

    try:
        f = open(path)
        structure = parser.get_structure(f.read())
        format = str(parser).split('.')[-1].split('Format')[0].lower()

        comment = ""
        order = 0
        relative_path = get_relative_path(path, locale)
        resource, created = Resource.objects.get_or_create(project=project,
                                                           path=relative_path,
                                                           format=format)

        for obj in structure:
            if isinstance(obj, silme.core.entity.Entity):
                if entities:
                    save_entity(resource=resource,
                                string=obj.value,
                                key=obj.id,
                                comment=comment,
                                order=order)
                    comment = ""
                    order += 1
                else:
                    try:
                        e = Entity.objects.get(resource=resource, key=obj.id)
                        save_translation(entity=e,
                                         locale=locale,
                                         string=obj.value)

                    except Entity.DoesNotExist:
                        continue

            elif isinstance(obj, silme.core.structure.Comment):
                if entities:
                    comment = str(obj)

        if entities:
            update_entity_count(resource)
        else:
            update_stats(resource, locale)

        log.debug("[" + locale.code + "]: " + path + " saved to DB.")
        f.close()
    except IOError:
        log.debug("[" + locale.code + "]: " + path +
                  " doesn't exist. Skipping.")
Beispiel #7
0
def extract_silme(parser, project, locale, path, entities=False):
    """Extract file with path using silme and save or update in DB."""

    try:
        f = open(path)
        structure = parser.get_structure(f.read())
        format = str(parser).split('.')[-1].split('Format')[0].lower()

        comment = ""
        order = 0
        relative_path = get_relative_path(path, locale)
        resource, created = Resource.objects.get_or_create(
            project=project, path=relative_path, format=format)

        for obj in structure:
            if isinstance(obj, silme.core.entity.Entity):
                if entities:
                    save_entity(resource=resource, string=obj.value,
                                key=obj.id, comment=comment, order=order)
                    comment = ""
                    order += 1
                else:
                    try:
                        e = Entity.objects.get(resource=resource, key=obj.id)
                        save_translation(
                            entity=e,
                            locale=locale,
                            string=obj.value)

                    except Entity.DoesNotExist:
                        continue

            elif isinstance(obj, silme.core.structure.Comment):
                if entities:
                    comment = str(obj)

        if entities:
            update_entity_count(resource)
        else:
            update_stats(resource, locale)

        log.debug("[" + locale.code + "]: " + path + " saved to DB.")
        f.close()
    except IOError:
        log.debug("[" + locale.code + "]: " +
                  path + " doesn't exist. Skipping.")
Beispiel #8
0
def store_l20n(
        entities, resource, key, string, comment, locale, order,
        string_plural="", plural_form=None):
    """Store .l20n entity or translation."""

    if entities:
        save_entity(
            resource=resource, string=string, string_plural=string_plural,
            comment=comment, key=key, order=order)

    else:
        try:
            e = Entity.objects.get(resource=resource, key=key)
            save_translation(
                entity=e, locale=locale, string=string,
                plural_form=plural_form)
        except Entity.DoesNotExist:
            pass
Beispiel #9
0
def extract_silme(parser, project, locale, path, entities=False):
    """Extract file with path using silme and save or update in DB."""

    with codecs.open(path, 'r', 'utf-8') as f:
        structure = parser.get_structure(f.read())
        format = str(parser).split('.')[2]

        comment = ""
        order = 0
        relative_path = get_relative_path(path, locale)
        resource, created = Resource.objects.get_or_create(project=project,
                                                           path=relative_path,
                                                           format=format)

        for obj in structure:
            if isinstance(obj, silme.core.entity.Entity):
                if entities:
                    save_entity(resource=resource,
                                string=obj.value,
                                key=obj.id,
                                comment=comment,
                                order=order)
                    comment = ""
                    order += 1
                else:
                    try:
                        e = Entity.objects.get(resource=resource, key=obj.id)
                        save_translation(entity=e,
                                         locale=locale,
                                         string=obj.value)

                    except Entity.DoesNotExist:
                        continue

            elif isinstance(obj, silme.core.structure.Comment):
                if entities:
                    comment = ''.join(unicode(i) for i in obj)

        if entities:
            update_entity_count(resource)
        else:
            update_stats(resource, locale)

        log.debug("[" + locale.code + "]: " + path + " saved to DB.")
Beispiel #10
0
def extract_silme(parser, project, locale, path, entities=False):
    """Extract file with path using silme and save or update in DB."""

    with codecs.open(path, 'r', 'utf-8') as f:
        structure = parser.get_structure(f.read())
        format = str(parser).split('.')[2]

        comment = ""
        order = 0
        relative_path = get_relative_path(path, locale)
        resource, created = Resource.objects.get_or_create(
            project=project, path=relative_path, format=format)

        for obj in structure:
            if isinstance(obj, silme.core.entity.Entity):
                if entities:
                    save_entity(resource=resource, string=obj.value,
                                key=obj.id, comment=comment, order=order)
                    comment = ""
                    order += 1
                else:
                    try:
                        e = Entity.objects.get(resource=resource, key=obj.id)
                        save_translation(
                            entity=e,
                            locale=locale,
                            string=obj.value)

                    except Entity.DoesNotExist:
                        continue

            elif isinstance(obj, silme.core.structure.Comment):
                if entities:
                    comment = ''.join(unicode(i) for i in obj)

        if entities:
            update_entity_count(resource)
        else:
            update_stats(resource, locale)

        log.debug("[" + locale.code + "]: " + path + " saved to DB.")
Beispiel #11
0
def extract_xliff(project, locale, path, entities=False):
    """Extract .xliff file with path and save or update in DB."""

    with open(path) as f:
        xf = xliff.xlifffile(f)

        relative_path = get_relative_path(path, locale)
        resource, created = Resource.objects.get_or_create(project=project,
                                                           path=relative_path,
                                                           format='xliff')

        if entities:
            for order, unit in enumerate(xf.units):
                save_entity(resource=resource,
                            string=unicode(unit.get_rich_source()[0]),
                            key=unit.getid(),
                            comment=unit.getnotes(),
                            order=order)

            update_entity_count(resource)

        else:
            for unit in xf.units:
                translation = unicode(unit.get_rich_target()[0])
                if translation:
                    try:
                        e = Entity.objects.get(resource=resource,
                                               key=unit.getid())
                        save_translation(entity=e,
                                         locale=locale,
                                         string=translation)

                    except Entity.DoesNotExist:
                        continue

            update_stats(resource, locale)

        log.debug("[" + locale.code + "]: " + path + " saved to DB.")
Beispiel #12
0
def extract_xliff(project, locale, path, entities=False):
    """Extract .xliff file with path and save or update in DB."""

    with open(path) as f:
        xf = xliff.xlifffile(f)

        relative_path = get_relative_path(path, locale)
        resource, created = Resource.objects.get_or_create(
            project=project, path=relative_path, format='xliff')

        if entities:
            for order, unit in enumerate(xf.units):
                save_entity(
                    resource=resource,
                    string=unicode(unit.get_rich_source()[0]),
                    key=unit.getid(),
                    comment=unit.getnotes(),
                    order=order)

            update_entity_count(resource)

        else:
            for unit in xf.units:
                translation = unicode(unit.get_rich_target()[0])
                if translation:
                    try:
                        e = Entity.objects.get(
                            resource=resource, key=unit.getid())
                        save_translation(
                            entity=e, locale=locale, string=translation)

                    except Entity.DoesNotExist:
                        continue

            update_stats(resource, locale)

        log.debug("[" + locale.code + "]: " + path + " saved to DB.")
Beispiel #13
0
def extract_po(project, locale, path, entities=False):
    """Extract .po (gettext) file with path and save or update in DB."""

    try:
        po = polib.pofile(path)

        relative_path = get_relative_path(path, locale)
        if relative_path[-1] == 't':
            relative_path = relative_path[:-1]

        resource, created = Resource.objects.get_or_create(
            project=project, path=relative_path, format='po')

        if entities:
            for order, entry in enumerate(po):
                if not entry.obsolete:
                    save_entity(resource=resource,
                                string=entry.msgid,
                                string_plural=entry.msgid_plural,
                                comment=entry.comment,
                                order=order,
                                source=entry.occurrences)

            update_entity_count(resource)

        else:
            for entry in (po.translated_entries() + po.fuzzy_entries()):
                if not entry.obsolete:

                    # Entities without plurals
                    if len(entry.msgstr) > 0:
                        try:
                            e = Entity.objects.get(
                                resource=resource,
                                string=entry.msgid)
                            save_translation(
                                entity=e,
                                locale=locale,
                                string=entry.msgstr,
                                fuzzy='fuzzy' in entry.flags)

                        except Entity.DoesNotExist:
                            continue

                    # Pluralized entities
                    elif len(entry.msgstr_plural) > 0:
                        try:
                            e = Entity.objects.get(
                                resource=resource,
                                string=entry.msgid)
                            for k in entry.msgstr_plural:
                                save_translation(
                                    entity=e,
                                    locale=locale,
                                    string=entry.msgstr_plural[k],
                                    plural_form=k,
                                    fuzzy='fuzzy' in entry.flags)

                        except Entity.DoesNotExist:
                            continue

            update_stats(resource, locale)

        log.debug("[" + locale.code + "]: " + path + " saved to DB.")
    except Exception as e:
        log.critical('PoExtractError for %s: %s' % (path, e))