def country2kw(row, kw):
# for both PAR and ADR
if row.has_key('PROF'):
activity = row['PROF']
if activity:
try:
activity = int(activity)
except ValueError:
dblogger.debug("Ignored invalid value PROF = %r", activity)
else:
if activity:
try:
activity = Activity.objects.get(pk=activity)
except Activity.DoesNotExist:
activity = Activity(id=activity,
name=unicode(activity))
activity.save(force_insert=True)
kw.update(activity=activity)
country = row['PAYS']
if country:
try:
country = Country.objects.get(short_code__exact=country)
except Country.DoesNotExist:
country = Country(isocode=country,
name=country,
short_code=country)
country.save()
kw.update(country=country)
store(
kw,
phone=row['TEL'],
fax=row['FAX'],
email=row['EMAIL'],
)
kw.update(street2kw(join_words(row['RUE'], row['RUENUM'], row['RUEBTE'])))
zip_code = row['CP']
if zip_code:
kw.update(zip_code=zip_code)
try:
city = Place.objects.get(
country=country,
zip_code__exact=zip_code,
)
kw.update(city=city)
except Place.DoesNotExist, e:
city = Place(zip_code=zip_code, name=zip_code, country=country)
city.save()
kw.update(city=city)
#~ logger.warning("%s-%s : %s",row['PAYS'],row['CP'],e)
except Place.MultipleObjectsReturned, e:
dblogger.warning("%s-%s : %s", row['PAYS'], row['CP'], e)
def country2kw(row, kw):
# for both PAR and ADR
if row.has_key('PROF'):
activity = row['PROF']
if activity:
try:
activity = int(activity)
except ValueError:
dblogger.debug("Ignored invalid value PROF = %r", activity)
else:
if activity:
try:
activity = Activity.objects.get(pk=activity)
except Activity.DoesNotExist:
activity = Activity(
id=activity, name=unicode(activity))
activity.save(force_insert=True)
kw.update(activity=activity)
country = row['PAYS']
if country:
try:
country = Country.objects.get(short_code__exact=country)
except Country.DoesNotExist:
country = Country(isocode=country, name=country,
short_code=country)
country.save()
kw.update(country=country)
email = row['EMAIL']
if email and is_valid_email(email):
kw.update(email=email)
store(kw,
phone=row['TEL'],
fax=row['FAX'],
)
kw.update(street2kw(join_words(row['RUE'], row['RUENUM'], row['RUEBTE'])))
zip_code = row['CP']
if zip_code:
kw.update(zip_code=zip_code)
try:
city = Place.objects.get(
country=country,
zip_code__exact=zip_code,
)
kw.update(city=city)
except Place.DoesNotExist, e:
city = Place(zip_code=zip_code, name=zip_code, country=country)
city.save()
kw.update(city=city)
#~ dblogger.warning("%s-%s : %s",row['PAYS'],row['CP'],e)
except Place.MultipleObjectsReturned, e:
dblogger.warning("%s-%s : %s", row['PAYS'], row['CP'], e)
def objects():
n = 0
for code, kw in LANGUAGES.items():
iso2 = kw['iso2']
kw = dd.babel_values('name', **kw)
if kw.get('name'):
n += 1
kw.update(iso2=iso2)
yield Language(id=code, **kw)
else:
logger.debug("%r : no name for default site language %s", code,
settings.SITE.DEFAULT_LANGUAGE.django_code)
def objects():
n = 0
for code, kw in LANGUAGES.items():
iso2 = kw['iso2']
kw = babel_values('name', **kw)
if kw.get('name'):
n += 1
kw.update(iso2=iso2)
yield Language(id=code, **kw)
else:
logger.debug("%r : no name for default site language %s",
code, settings.SITE.DEFAULT_LANGUAGE.django_code)
def watch(data_dir):
"Deserves more documentation."
infile = os.path.join(data_dir, "changelog.json")
if not os.path.exists(infile):
# ~ print "Nothing to do."
return
watching = os.path.join(data_dir, "changelog.watching.json")
if not os.path.exists(watching):
try:
os.rename(infile, watching)
except Exception, e:
dblogger.debug("Could not rename %s to %s: %s", infile, watching, e)
return
def plp2lino(plptype, p, c):
if plptype.endswith('R'):
return
if plptype == '01':
role = R_CHILD
elif plptype == '02':
role = R_COHABITANT
elif plptype == '03':
role = R_ADOPTED
elif plptype == '04':
role = R_RELATIVE
elif plptype == '10':
role = R_MARRIED
elif plptype == '11':
role = R_PARTNER
else:
raise Exception("Invalid link type %r" % plptype)
# Is there a household where the parent of this relation is no a
# child?
members = Member.objects.filter(
person=p.person).exclude(role__in=child_roles)
if members.count() == 0:
hh = Household(type=T_FAMILY, name=p.person.last_name)
hh.full_clean()
hh.save()
dblogger.debug(
"Created household %s from PLP `%s is %s of %s`",
hh, p, role, c)
obj = Member(household=hh, role=R_CHEF, person=p.person)
obj.full_clean()
obj.save()
elif members.count() == 1:
hh = members[0].household
else:
msg = "Found more than 1 household for parent %r" % p
# raise Exception(msg)
dblogger.warning(msg)
return
obj = Member(household=hh, role=role, person=c.person)
obj.full_clean()
obj.save()
dblogger.info("Created %s as %s", obj.person, obj.role)
def objects():
n = 0
"""
"""
fn = os.path.join(os.path.dirname(__file__),'countries.xml')
dom = minidom.parse(fn)
#~ print dom.documentElement.__class__
#~ print dom.documentElement
for coun in dom.documentElement.getElementsByTagName('coun:country'):
#~ print coun.toxml()
#~ print c.attributes['coun:alpha2']
names = {}
for name in coun.getElementsByTagName('coun:name'):
assert len(name.childNodes) == 1
#~ print [n.data for n in ]
#~ print name.firstChild.data
names[str(name.attributes['lang'].value)] = name.firstChild.data
kw = babel_values('name',**names)
kw.update(
isocode = coun.getElementsByTagName('coun:alpha2')[0].childNodes[0].data,
iso3 = coun.getElementsByTagName('coun:alpha3')[0].childNodes[0].data,
)
if kw['name']:
#~ kw.update(iso3=iso3)
n += 1
yield Country(**kw)
else:
logger.debug("%r : no name for default site language %s",
code,settings.SITE.DEFAULT_LANGUAGE.django_code)
for ln in TABLE2.splitlines():
ln = ln.strip()
if ln:
code1, code2, code3, name = ln.split(None,3)
n += 1
yield Country(isocode=code1,name=name)
rec = ln.split('\t')
if len(rec) != 8:
logger.warning("Ignored line %d (len(rec) is %d)", n, len(rec))
elif len(rec[0]) != 3:
logger.warning("Ignored line %d", n)
else:
language_type = rec[5]
if language_type == 'L':
ref_name = rec[6]
if ref_name:
code = rec[0]
if len(rec[1]) == 3:
code = rec[1]
LANGUAGES[code] = dict(en=ref_name, iso2=rec[3])
else:
logger.debug("Ignored line %d : language type is %r", n,
language_type)
"""
"""
german = u'''
Abchasisch abk
Aceh-Sprache ace
Acholi-Sprache ach
Adangme-Sprache ada
Adygisch ady
Ägyptisch egy
Afrihili afh
Afrikaans afr
Ainu-Sprache ain
Akan-Sprache aka
Akkadisch akk
rec = ln.split('\t')
if len(rec) != 8:
logger.warning("Ignored line %d (len(rec) is %d)", n, len(rec))
elif len(rec[0]) != 3:
logger.warning("Ignored line %d", n)
else:
language_type = rec[5]
if language_type == 'L':
ref_name = rec[6]
if ref_name:
code = rec[0]
if len(rec[1]) == 3:
code = rec[1]
LANGUAGES[code] = dict(en=ref_name, iso2=rec[3])
else:
logger.debug("Ignored line %d : language type is %r",
n, language_type)
"""
"""
german = u'''
Abchasisch abk
Aceh-Sprache ace
Acholi-Sprache ach
Adangme-Sprache ada
Adygisch ady
Ägyptisch egy
Afrihili afh
Afrikaans afr
Ainu-Sprache ain
if not os.path.exists(watching):
try:
os.rename(infile, watching)
except Exception, e:
dblogger.debug("Could not rename %s to %s: %s", infile, watching, e)
return
dblogger.info("Processing file %s", watching)
fd_watching = codecs.open(watching, "r", encoding="cp850")
failed = os.path.join(data_dir, "changelog.failed.json")
fd_failed = codecs.open(failed, "a", encoding="cp850")
# ~ log = open(os.path.join(data_dir,'changelog.done.log'),'a')
i = 0
for ln in fd_watching.readlines():
i += 1
dblogger.debug("process_line(%r,%r)", i, ln)
try:
process_line(ln)
except Exception, e:
# ~ raise
fd_failed.write("// %s %r\n%s\n\n" % (time.strftime("%Y-%m-%d %H:%M:%S"), e, ln))
# ~ fd_failed.write(ln+'\n')
# ~ dblogger.warning("%s:%d: %r\nin changelog line %s", watching,i,e,ln)
dblogger.warning("Exception '%r' while processing changelog line:\n%s", e, ln)
# for ValidationError we don't want a full traceback with mail to
# the admins.
if not isinstance(e, IGNORABLE_ERRORS):
dblogger.exception(e)
# ~ raise
fd_watching.close()
fd_failed.close()
