Exemplo n.º 1
0
    def import_from_icecat(cls):
        from iceparser.models import Vocabulary

        # request = urllib2.Request(
        #     'http://data.icecat.biz/export/freexml/refs/LanguageList.xml.gz')
        # base64string = base64.encodestring(
        # '%s:%s' % ('username', 'password')).replace('\n', '')
        # request.add_header('Authorization', "Basic %s" % base64string)
        # result = urllib2.urlopen(request)
        #
        # local_file = open('languages.gz', "wb")
        # local_file.write(result.read())
        # local_file.close()

        tree = etree.parse('LanguageList.xml')

        current_languages = make_model_dict(Language)
        current_vocabularies = make_model_dict(Vocabulary)
        languages = {}

        # Save languages
        language_tags = tree.findall('.//Language')
        language_count = len(language_tags)
        for idx, language_tag in enumerate(language_tags):
            print '{0} de {1}'.format(idx + 1, language_count)

            d = language_tag.attrib

            language = Language(id=int(d['ID']),
                                code=d['Code'],
                                short_code=d['ShortCode'],
                                published='Y')

            try:
                current_language = current_languages[language.id]

                if model_to_dict(language) != model_to_dict(current_language):
                    language.save()
            except KeyError:
                language.save()

            languages[language.id] = language

        # Save translations
        Vocabulary.batch_import(tree.findall('.//Name'), current_vocabularies,
                                languages)
Exemplo n.º 2
0
    def import_from_icecat(cls):
        from iceparser.models import Language, Measure, Tex, Vocabulary

        tree = etree.parse("FeaturesList.xml")

        current_measures = make_model_dict(Measure)
        current_features = make_model_dict(Feature)
        current_languages = make_model_dict(Language)
        current_texes = make_model_dict(Tex)
        current_vocabularies = make_model_dict(Vocabulary)

        # Save features
        feature_tags = tree.findall(".//Feature")
        features_count = len(feature_tags)
        for idx, feature_tag in enumerate(feature_tags):
            print "{0} de {1}".format(idx + 1, features_count)

            d = feature_tag.attrib

            measure_tag = feature_tag.find("Measure")
            if not measure_tag:
                continue

            measure_id = int(measure_tag.attrib["ID"])
            measure = current_measures[measure_id]

            feature = Feature(id=int(d["ID"]), klass=int(d["Class"]), type=d["Type"], measure=measure)

            try:
                if model_to_dict(feature) != model_to_dict(current_features[feature.id]):
                    feature.save()
            except KeyError:
                feature.save()

            descs = Tex.batch_import(feature_tag.iterfind(".//Description"), current_texes, current_languages)

            current_descriptions = set([d.id for d in measure.descriptions.all()])
            new_descriptions = set([d.id for d in descs])
            if current_descriptions != new_descriptions:
                measure.descriptions = descs

            names = Vocabulary.batch_import(feature_tag.findall(".//Name"), current_vocabularies, current_languages)

            current_names = set([d.id for d in measure.names.all()])
            new_names = set([d.id for d in names])
            if current_names != new_names:
                measure.names = names
Exemplo n.º 3
0
    def import_from_icecat(cls):
        from iceparser.models import Language, Measure, Tex, Vocabulary

        tree = etree.parse('FeaturesList.xml')

        current_measures = make_model_dict(Measure)
        current_features = make_model_dict(Feature)
        current_languages = make_model_dict(Language)
        current_texes = make_model_dict(Tex)
        current_vocabularies = make_model_dict(Vocabulary)

        # Save features
        feature_tags = tree.findall('.//Feature')
        features_count = len(feature_tags)
        for idx, feature_tag in enumerate(feature_tags):
            print '{0} de {1}'.format(idx+1, features_count)

            d = feature_tag.attrib

            measure_tag = feature_tag.find('Measure')
            if not measure_tag:
                continue

            measure_id = int(measure_tag.attrib['ID'])
            measure = current_measures[measure_id]

            feature = Feature(
                id=int(d['ID']),
                klass=int(d['Class']),
                type=d['Type'],
                measure=measure
            )

            try:
                if model_to_dict(feature) != model_to_dict(current_features[feature.id]):
                    feature.save()
            except KeyError:
                feature.save()

            descs = Tex.batch_import(
                feature_tag.iterfind('.//Description'),
                current_texes,
                current_languages
            )

            current_descriptions = set(
                [d.id for d in measure.descriptions.all()])
            new_descriptions = set([d.id for d in descs])
            if current_descriptions != new_descriptions:
                measure.descriptions = descs

            names = Vocabulary.batch_import(
                feature_tag.findall('.//Name'),
                current_vocabularies,
                current_languages
            )

            current_names = set([d.id for d in measure.names.all()])
            new_names = set([d.id for d in names])
            if current_names != new_names:
                measure.names = names
Exemplo n.º 4
0
    def import_from_icecat(cls):
        from iceparser.models import Language, MeasureSign, Vocabulary, Tex

        # request = urllib2.Request(
        #     'http://data.icecat.biz/export/freexml/refs/MeasuresList.xml.gz')
        # base64string = base64.encodestring(
        # '%s:%s' % ('username', 'password')).replace('\n', '')
        # request.add_header('Authorization', "Basic %s" % base64string)
        # result = urllib2.urlopen(request)
        #
        # local_file = open('measures.gz', "wb")
        # local_file.write(result.read())
        # local_file.close()

        tree = etree.parse('MeasuresList.xml')

        languages = make_model_dict(Language)
        texes = make_model_dict(Tex)
        vocabularies = make_model_dict(Vocabulary)
        measure_signs = make_model_dict(MeasureSign)

        measures = tree.findall('.//Measure')
        measure_count = len(measures)

        # Save measures
        for idx, measure_tag in enumerate(measures):
            print '{0} de {1}'.format(idx + 1, measure_count)

            if measure_tag.find('Sign') is not None:
                sign_value = measure_tag.find('Sign').text
            else:
                sign_value = ''

            measure = Measure(
                id=measure_tag.attrib['ID'],
                sign=sign_value
            )

            measure.save()

            descs = Tex.batch_import(
                measure_tag.iterfind('.//Description'),
                texes,
                languages
            )

            current_descriptions = set([d.id for d in measure.descriptions.all()])
            new_descriptions = set([d.id for d in descs])
            if current_descriptions != new_descriptions:
                measure.descriptions = descs

            names = Vocabulary.batch_import(
                measure_tag.findall('.//Name'),
                vocabularies,
                languages
            )

            current_names = set([d.id for d in measure.names.all()])
            new_names = set([d.id for d in names])
            if current_names != new_names:
                measure.names = names

            # Save measure signs
            for measure_sign_tag in measure_tag.iterfind('Signs//Sign'):
                ms_dict = measure_sign_tag.attrib

                if ms_dict['ID'] == '0':
                    continue

                value = measure_sign_tag.text
                if not value:
                    value = ''

                measure_sign = MeasureSign(
                    id=int(ms_dict['ID']),
                    measure=measure,
                    language=languages[int(ms_dict['langid'])],
                    value=value
                )

                d1 = model_to_dict(measure_sign)

                try:
                    d2 = model_to_dict(measure_signs[measure_sign.id])

                    if d1 != d2 and int(d1['measure']) != d2['measure']:
                        measure_sign.save()
                except KeyError:
                    measure_sign.save()