def import_from_icecat(cls):
from iceparser.models import Vocabulary
# request = urllib2.Request(
# 'http://data.icecat.biz/export/freexml/refs/LanguageList.xml.gz')
# base64string = base64.encodestring(
# '%s:%s' % ('username', 'password')).replace('\n', '')
# request.add_header('Authorization', "Basic %s" % base64string)
# result = urllib2.urlopen(request)
#
# local_file = open('languages.gz', "wb")
# local_file.write(result.read())
# local_file.close()
tree = etree.parse('LanguageList.xml')
current_languages = make_model_dict(Language)
current_vocabularies = make_model_dict(Vocabulary)
languages = {}
# Save languages
language_tags = tree.findall('.//Language')
language_count = len(language_tags)
for idx, language_tag in enumerate(language_tags):
print '{0} de {1}'.format(idx + 1, language_count)
d = language_tag.attrib
language = Language(id=int(d['ID']),
code=d['Code'],
short_code=d['ShortCode'],
published='Y')
try:
current_language = current_languages[language.id]
if model_to_dict(language) != model_to_dict(current_language):
language.save()
except KeyError:
language.save()
languages[language.id] = language
# Save translations
Vocabulary.batch_import(tree.findall('.//Name'), current_vocabularies,
languages)
def import_from_icecat(cls):
from iceparser.models import Language, Measure, Tex, Vocabulary
tree = etree.parse("FeaturesList.xml")
current_measures = make_model_dict(Measure)
current_features = make_model_dict(Feature)
current_languages = make_model_dict(Language)
current_texes = make_model_dict(Tex)
current_vocabularies = make_model_dict(Vocabulary)
# Save features
feature_tags = tree.findall(".//Feature")
features_count = len(feature_tags)
for idx, feature_tag in enumerate(feature_tags):
print "{0} de {1}".format(idx + 1, features_count)
d = feature_tag.attrib
measure_tag = feature_tag.find("Measure")
if not measure_tag:
continue
measure_id = int(measure_tag.attrib["ID"])
measure = current_measures[measure_id]
feature = Feature(id=int(d["ID"]), klass=int(d["Class"]), type=d["Type"], measure=measure)
try:
if model_to_dict(feature) != model_to_dict(current_features[feature.id]):
feature.save()
except KeyError:
feature.save()
descs = Tex.batch_import(feature_tag.iterfind(".//Description"), current_texes, current_languages)
current_descriptions = set([d.id for d in measure.descriptions.all()])
new_descriptions = set([d.id for d in descs])
if current_descriptions != new_descriptions:
measure.descriptions = descs
names = Vocabulary.batch_import(feature_tag.findall(".//Name"), current_vocabularies, current_languages)
current_names = set([d.id for d in measure.names.all()])
new_names = set([d.id for d in names])
if current_names != new_names:
measure.names = names
def import_from_icecat(cls):
from iceparser.models import Language, Measure, Tex, Vocabulary
tree = etree.parse('FeaturesList.xml')
current_measures = make_model_dict(Measure)
current_features = make_model_dict(Feature)
current_languages = make_model_dict(Language)
current_texes = make_model_dict(Tex)
current_vocabularies = make_model_dict(Vocabulary)
# Save features
feature_tags = tree.findall('.//Feature')
features_count = len(feature_tags)
for idx, feature_tag in enumerate(feature_tags):
print '{0} de {1}'.format(idx+1, features_count)
d = feature_tag.attrib
measure_tag = feature_tag.find('Measure')
if not measure_tag:
continue
measure_id = int(measure_tag.attrib['ID'])
measure = current_measures[measure_id]
feature = Feature(
id=int(d['ID']),
klass=int(d['Class']),
type=d['Type'],
measure=measure
)
try:
if model_to_dict(feature) != model_to_dict(current_features[feature.id]):
feature.save()
except KeyError:
feature.save()
descs = Tex.batch_import(
feature_tag.iterfind('.//Description'),
current_texes,
current_languages
)
current_descriptions = set(
[d.id for d in measure.descriptions.all()])
new_descriptions = set([d.id for d in descs])
if current_descriptions != new_descriptions:
measure.descriptions = descs
names = Vocabulary.batch_import(
feature_tag.findall('.//Name'),
current_vocabularies,
current_languages
)
current_names = set([d.id for d in measure.names.all()])
new_names = set([d.id for d in names])
if current_names != new_names:
measure.names = names
def import_from_icecat(cls):
from iceparser.models import Language, MeasureSign, Vocabulary, Tex
# request = urllib2.Request(
# 'http://data.icecat.biz/export/freexml/refs/MeasuresList.xml.gz')
# base64string = base64.encodestring(
# '%s:%s' % ('username', 'password')).replace('\n', '')
# request.add_header('Authorization', "Basic %s" % base64string)
# result = urllib2.urlopen(request)
#
# local_file = open('measures.gz', "wb")
# local_file.write(result.read())
# local_file.close()
tree = etree.parse('MeasuresList.xml')
languages = make_model_dict(Language)
texes = make_model_dict(Tex)
vocabularies = make_model_dict(Vocabulary)
measure_signs = make_model_dict(MeasureSign)
measures = tree.findall('.//Measure')
measure_count = len(measures)
# Save measures
for idx, measure_tag in enumerate(measures):
print '{0} de {1}'.format(idx + 1, measure_count)
if measure_tag.find('Sign') is not None:
sign_value = measure_tag.find('Sign').text
else:
sign_value = ''
measure = Measure(
id=measure_tag.attrib['ID'],
sign=sign_value
)
measure.save()
descs = Tex.batch_import(
measure_tag.iterfind('.//Description'),
texes,
languages
)
current_descriptions = set([d.id for d in measure.descriptions.all()])
new_descriptions = set([d.id for d in descs])
if current_descriptions != new_descriptions:
measure.descriptions = descs
names = Vocabulary.batch_import(
measure_tag.findall('.//Name'),
vocabularies,
languages
)
current_names = set([d.id for d in measure.names.all()])
new_names = set([d.id for d in names])
if current_names != new_names:
measure.names = names
# Save measure signs
for measure_sign_tag in measure_tag.iterfind('Signs//Sign'):
ms_dict = measure_sign_tag.attrib
if ms_dict['ID'] == '0':
continue
value = measure_sign_tag.text
if not value:
value = ''
measure_sign = MeasureSign(
id=int(ms_dict['ID']),
measure=measure,
language=languages[int(ms_dict['langid'])],
value=value
)
d1 = model_to_dict(measure_sign)
try:
d2 = model_to_dict(measure_signs[measure_sign.id])
if d1 != d2 and int(d1['measure']) != d2['measure']:
measure_sign.save()
except KeyError:
measure_sign.save()