def handle(self, path, **options):
path = pathlib.Path(path)
base = path.parent
printer = helpers.Printer(self.stdout, options['verbosity'])
printer.info("Importing political parties...")
with path.open() as f:
parties = yaml.safe_load(f)
for party in parties:
printer.info(' %s' % party['name'])
names = [party['name']] + party.get('alternate_names', [])
try:
actor = models.Actor.objects.get(first_name__in=names,
group=True)
except models.Actor.DoesNotExist:
printer.info((
" error: can't find existing party entry in database with Actor.first_name=%r."
) % party['name'])
else:
logo = (base / party['logo']) if party['logo'] else None
if logo and logo.exists():
if not actor.photo:
with logo.open('rb') as f:
actor.photo.save(path.name, File(f), save=True)
elif logo:
printer.info(" error: logo %r can't be found" %
party['logo'])
printer.info("done.")
def handle(self, slug, path, **options):
printer = helpers.Printer(self.stdout, options['verbosity'])
user = services.get_bot_user('importbot')
topic = models.Topic.objects.get(slug=slug)
with open(path) as f:
posts = yaml.safe_load(f)
for post in posts:
if post['type'] == 'event':
exists = (models.Event.objects.filter(
post__topic=topic,
source_link=post['event']['source_link']).exists())
if exists:
printer.info('"%s" already exists.' %
post['event']['title'])
else:
form = forms.EventForm(topic, post['event'])
if form.is_valid():
services.create_event(user, topic, form.cleaned_data)
printer.info('"%s" imported.' % post['event']['title'])
else:
printer.info('Error while importing %r' %
post['event'])
printer.info(form.errors.as_text())
def handle(self, path, **options):
path = pathlib.Path(path)
printer = helpers.Printer(self.stdout, options['verbosity'])
seimas, created = models.Body.objects.get_or_create(name='Seimas')
typemap = {
'SEI': seimas,
}
printer.info('Importing terms...')
with (path / 'kadencijos.csv').open() as f:
reader = csv.DictReader(f)
import_terms(reader, typemap)
def todt(value):
return datetime.datetime.strptime(value, '%Y-%m-%d')
printer.info('Importing candidates and parties...')
with (path / 'kadencijos.csv').open() as f:
Term = collections.namedtuple('Term', 'id body since until group')
terms = {
row['KADENCIJOS_ID']:
Term(id=row['KADENCIJOS_ID'],
body=typemap[row['RUSIS']],
since=todt(row['PRADZIA']),
until=todt(row['PABAIGA']) if row['PABAIGA'] else None,
group=models.Group.objects.get_or_create(
title='Kandidatai į %s metų Seimą' %
todt(row['PRADZIA']).year,
defaults={'timestamp': todt(row['PRADZIA'])},
)[0])
for row in csv.DictReader(f) if row['RUSIS'] == 'SEI'
}
with (path / 'rinkimai.csv').open() as f:
Election = collections.namedtuple('Election', 'term date')
elections = {
row['VYKDOMU_RINKIMU_TURO_ID']: Election(
term=terms[row['KADENCIJOS_ID']],
date=todt(row['RINKIMU_TURO_DATA']),
)
for row in csv.DictReader(f) if row['KADENCIJOS_ID'] in terms
}
with (path / 'organizacijos.csv').open() as f:
organisations = {
row['ORGANIZACIJOS_ID']: row['ORGANIZACIJOS_PAVADINIMAS']
for row in csv.DictReader(f)
}
def name(name):
return ' '.join([x.title() for x in name.split()])
with (path / 'kandidatai.csv').open() as f:
Candidate = collections.namedtuple(
'Candidate', 'birth_date last_name first_name')
CandidateElection = collections.namedtuple(
'CandidateElection',
'elected election vienmandate daugiamandate')
candidates = collections.defaultdict(list)
for row in csv.DictReader(f):
if row['VR_TURO_ID'] in elections:
candidate = Candidate(row['GIMIMO_DATA'],
name(row['PAVARDE']),
name(row['VARDAS']))
vienmandate = row['VIENMANDATE_ORGANIZACIJA']
daugiamandate = row['DAUGIAMANDATE_ORGANIZACIJA']
candidates[candidate].append(
CandidateElection(
elected=row['AR_ISRINKTAS'] == 'T',
election=elections[row['VR_TURO_ID']],
vienmandate=organisations[vienmandate]
if vienmandate else None,
daugiamandate=organisations[daugiamandate]
if daugiamandate else None,
))
for candidate, rounds in candidates.items():
printer.info('')
printer.info('%s %s %s' % candidate)
membership = {}
rounds = sorted(rounds, key=lambda x: x.election.term.since)
rounds_by_term = itertools.groupby(
rounds, key=lambda x: x.election.term.id)
times_elected = sum(1 for x in rounds if x.elected)
times_candidate = 0
actor_terms = []
for term_id, term_rounds in rounds_by_term:
times_candidate += 1
term_rounds = list(term_rounds)
term = terms[term_id]
actor_terms.append(term)
elected = sorted(
[x.election.date for x in term_rounds if x.elected])
result = ('elected: %s' %
elected[0].strftime('%Y-%m-%d')) if elected else ''
printer.info(' %s-%s %s' % (
term.since.strftime('%Y'),
term.until.strftime('%Y') if term.until else ' ',
result,
))
orgs = set()
orgs.update(x.vienmandate for x in term_rounds
if x.vienmandate)
orgs.update(x.daugiamandate for x in term_rounds
if x.daugiamandate)
if orgs:
for org in sorted(orgs):
printer.info(' %s' % org)
since, until = membership.get(
org,
(min(elected, default=term.since), term.until))
membership[org] = (
min(since, term.since),
max(until, term.until)
if until and term.until else term.until,
)
printer.info(' Member of:')
for org, (since, until) in membership.items():
printer.info(' %s (%s - %s)' % (
org,
since.strftime('%Y-%m-%d'),
until.strftime('%Y-%m-%d') if until else '',
))
actor_title = 'seimo narys' if times_elected > 0 else ''
actor, created = models.Actor.objects.get_or_create(
birth_date=candidate.birth_date,
first_name=candidate.first_name,
last_name=candidate.last_name,
defaults={
'title': actor_title,
'times_elected': times_elected,
'times_candidate': times_candidate,
},
)
if not created:
modified = [
actor.times_elected != times_elected,
actor.times_candidate != times_candidate,
]
if any(modified):
actor.times_elected = times_elected
actor.times_candidate = times_candidate
actor.save()
for org, (since, until) in membership.items():
party, created = models.Actor.objects.get_or_create(
first_name=org,
group=True,
defaults={
'title': 'plitinė partija',
'body': seimas,
})
models.Member.objects.get_or_create(actor=actor,
group=party,
defaults={
'since': since,
'until': until,
})
# Add this actor to all term groups
in_groups = actor.ingroups.values_list('pk', flat=True)
for term in actor_terms:
if term.group.pk not in in_groups:
term.group.members.add(actor)
printer.info('done.')
def handle(self, path, **options):
printer = helpers.Printer(self.stdout, options['verbosity'])
path = pathlib.Path(path)
# ------------------------------------------
# 'fname': <first name>,
# 'lname': <last name>,
# 'bdate': <birth date>,
# 'title': <title or profession>,
# 'partial': <bool: true if data covers only one or few elections, but not all of them>,
# 'elections': [
# {
# 'body': <body, for example 'Seimas'>,
# 'since': <date, when elections started>,
# 'until': <date, until next elections>,
# 'party': <party>,
# 'photo': {
# 'ext': <photo file extension>,
# 'data': <base64 encoded photo file>,
# },
# 'elected': <bool: elected or not>,
# }
# ]
now = datetime.datetime.now()
strptime = datetime.datetime.strptime
Photo = collections.namedtuple('Photo', 'type, data')
Election = collections.namedtuple(
'Term', 'body, since, until, party, photo, elected')
Candidate = collections.namedtuple(
'Candidate', 'fname, lname, bdate, partial, elections')
with path.open(encoding='utf-8') as f:
for line in f:
row = json.loads(line)
row['elections'] = [
Election(**dict(
x,
since=strptime(x['since'], '%Y-%m-%d'),
until=strptime(x['until'], '%Y-%m-%d'),
photo=Photo(**x['photo']),
)) for x in row['elections']
]
candidate = Candidate(**dict(
row,
bdate=strptime(row['bdate']),
))
# Update candidate
printer.info('')
printer.info('{bdate} {fname} {lname}'.format(**row))
actor, created = models.Actor.objects.get_or_create(
birth_date=candidate.bdate,
first_name=candidate.fname,
last_name=candidate.lname,
defaults={
'title': candidate.title,
},
)
# Update candidate photo
if not actor.photo and candidate.elections:
photo = candidate.elections[-1].photo
filename = '%s-%s.%s' % (candidate.fname, candidate.lname,
photo.ext)
content = io.BytesIO(
base64.b64decode(photo.data.encode('ascii')))
actor.photo.save(filename, File(content), save=True)
actor_groups = actor.ingroups.values_list('pk', flat=True)
actor_parties = {
x.pk: x
for x in models.Member.objects.filter(actor=actor)
}
for election in candidate.elections:
# Get election body (must be created manually)
body = models.Body.objects.get(name=election.body)
# Get or create election term
term, created = models.Term.objects.get_or_create(
body=body,
title='%d-%d metų kadencija' % (
election.since.year,
election.until.year
if election.until else now.year,
))
# Get or create group for this election
group_title = {
'Seimas': 'Seimą',
}
group, created = models.Group.objects.get_or_create(
title='Kandidatai į %d metų %s' % (
election.since.year,
group_title[election.body],
),
defaults={
'timestamp': election.since,
},
)
# Add current candidate to the election group
if group.pk not in actor_groups:
group.members.add(actor)
# Get or create party
party, created = models.Actor.objects.get_or_create(
first_name=election.party,
group=True,
defaults={
'body': body,
'title': 'plitinė partija',
},
)
# Update party membership information
membership = actor_parties.get(party.pk, None)
if membership is None:
models.Member.objects.get_or_create(actor=actor,
group=party,
defaults={
'since':
election.since,
'until':
election.until,
})
elif (
(election.since
and membership.since.date() != election.since.date())
or
(election.until and
membership.until.date() != election.until.date())):
if election.since:
membership.since = min(membership.since,
election.since)
if election.until:
membership.until = max(membership.until,
election.until)
membership.save()
printer.info('done.')
def handle(self, path, base, **options):
printer = helpers.Printer(self.stdout, options['verbosity'])
path = pathlib.Path(path)
base = pathlib.Path(base)
with path.open() as f:
photos = yaml.safe_load(f)
def clean_name(name):
return unidecode.unidecode(name).lower()
Person = collections.namedtuple('Person',
'id first_name last_name birth_date')
names = collections.defaultdict(list)
for x in models.Actor.objects.filter(birth_date__isnull=False,
group=False):
names[clean_name(x.first_name + ' ' + x.last_name)].append(
Person(
id=x.id,
first_name=x.first_name,
last_name=x.last_name,
birth_date=x.birth_date,
))
for photo in photos:
cleaned_name = clean_name(photo['name'])
if cleaned_name in names:
matches = [
x for x in names[cleaned_name]
if (photo.get('born') is None or photo['born'].strftime(
'%Y-%m-%d') == x.birth_date.strftime('%Y-%m-%d'))
]
if len(matches) == 1:
x = matches[0]
printer.info('- name: %s %s' % (x.first_name, x.last_name))
printer.info(' born: %s' %
(x.birth_date.strftime('%Y-%m-%d')))
printer.info(' photo: %s' % photo['photo'])
photo_path = (base /
photo['photo']) if photo['photo'] else None
if photo_path and photo_path.exists():
actor = models.Actor.objects.get(pk=x.id)
if not actor.photo:
with photo_path.open('rb') as f:
actor.photo.save(path.name, File(f), save=True)
elif photo_path:
printer.info(" error: logo %r can't be found" %
photo['photo'])
else:
printer.info('# edit: multiple matches')
for x in matches:
printer.info('- name: %s %s' %
(x.first_name, x.last_name))
printer.info(' born: %s' %
(x.birth_date.strftime('%Y-%m-%d')))
printer.info(' photo: %s' % photo['photo'])
else:
printer.info('# edit: no match found')
printer.info('- name: %s' % photo['name'])
printer.info(' photo: %s' % photo['photo'])