def handle(self, path, **options): path = pathlib.Path(path) base = path.parent printer = helpers.Printer(self.stdout, options['verbosity']) printer.info("Importing political parties...") with path.open() as f: parties = yaml.safe_load(f) for party in parties: printer.info(' %s' % party['name']) names = [party['name']] + party.get('alternate_names', []) try: actor = models.Actor.objects.get(first_name__in=names, group=True) except models.Actor.DoesNotExist: printer.info(( " error: can't find existing party entry in database with Actor.first_name=%r." ) % party['name']) else: logo = (base / party['logo']) if party['logo'] else None if logo and logo.exists(): if not actor.photo: with logo.open('rb') as f: actor.photo.save(path.name, File(f), save=True) elif logo: printer.info(" error: logo %r can't be found" % party['logo']) printer.info("done.")
def handle(self, slug, path, **options): printer = helpers.Printer(self.stdout, options['verbosity']) user = services.get_bot_user('importbot') topic = models.Topic.objects.get(slug=slug) with open(path) as f: posts = yaml.safe_load(f) for post in posts: if post['type'] == 'event': exists = (models.Event.objects.filter( post__topic=topic, source_link=post['event']['source_link']).exists()) if exists: printer.info('"%s" already exists.' % post['event']['title']) else: form = forms.EventForm(topic, post['event']) if form.is_valid(): services.create_event(user, topic, form.cleaned_data) printer.info('"%s" imported.' % post['event']['title']) else: printer.info('Error while importing %r' % post['event']) printer.info(form.errors.as_text())
def handle(self, path, **options): path = pathlib.Path(path) printer = helpers.Printer(self.stdout, options['verbosity']) seimas, created = models.Body.objects.get_or_create(name='Seimas') typemap = { 'SEI': seimas, } printer.info('Importing terms...') with (path / 'kadencijos.csv').open() as f: reader = csv.DictReader(f) import_terms(reader, typemap) def todt(value): return datetime.datetime.strptime(value, '%Y-%m-%d') printer.info('Importing candidates and parties...') with (path / 'kadencijos.csv').open() as f: Term = collections.namedtuple('Term', 'id body since until group') terms = { row['KADENCIJOS_ID']: Term(id=row['KADENCIJOS_ID'], body=typemap[row['RUSIS']], since=todt(row['PRADZIA']), until=todt(row['PABAIGA']) if row['PABAIGA'] else None, group=models.Group.objects.get_or_create( title='Kandidatai į %s metų Seimą' % todt(row['PRADZIA']).year, defaults={'timestamp': todt(row['PRADZIA'])}, )[0]) for row in csv.DictReader(f) if row['RUSIS'] == 'SEI' } with (path / 'rinkimai.csv').open() as f: Election = collections.namedtuple('Election', 'term date') elections = { row['VYKDOMU_RINKIMU_TURO_ID']: Election( term=terms[row['KADENCIJOS_ID']], date=todt(row['RINKIMU_TURO_DATA']), ) for row in csv.DictReader(f) if row['KADENCIJOS_ID'] in terms } with (path / 'organizacijos.csv').open() as f: organisations = { row['ORGANIZACIJOS_ID']: row['ORGANIZACIJOS_PAVADINIMAS'] for row in csv.DictReader(f) } def name(name): return ' '.join([x.title() for x in name.split()]) with (path / 'kandidatai.csv').open() as f: Candidate = collections.namedtuple( 'Candidate', 'birth_date last_name first_name') CandidateElection = collections.namedtuple( 'CandidateElection', 'elected election vienmandate daugiamandate') candidates = collections.defaultdict(list) for row in csv.DictReader(f): if row['VR_TURO_ID'] in elections: candidate = Candidate(row['GIMIMO_DATA'], name(row['PAVARDE']), name(row['VARDAS'])) vienmandate = row['VIENMANDATE_ORGANIZACIJA'] daugiamandate = row['DAUGIAMANDATE_ORGANIZACIJA'] candidates[candidate].append( CandidateElection( elected=row['AR_ISRINKTAS'] == 'T', election=elections[row['VR_TURO_ID']], vienmandate=organisations[vienmandate] if vienmandate else None, daugiamandate=organisations[daugiamandate] if daugiamandate else None, )) for candidate, rounds in candidates.items(): printer.info('') printer.info('%s %s %s' % candidate) membership = {} rounds = sorted(rounds, key=lambda x: x.election.term.since) rounds_by_term = itertools.groupby( rounds, key=lambda x: x.election.term.id) times_elected = sum(1 for x in rounds if x.elected) times_candidate = 0 actor_terms = [] for term_id, term_rounds in rounds_by_term: times_candidate += 1 term_rounds = list(term_rounds) term = terms[term_id] actor_terms.append(term) elected = sorted( [x.election.date for x in term_rounds if x.elected]) result = ('elected: %s' % elected[0].strftime('%Y-%m-%d')) if elected else '' printer.info(' %s-%s %s' % ( term.since.strftime('%Y'), term.until.strftime('%Y') if term.until else ' ', result, )) orgs = set() orgs.update(x.vienmandate for x in term_rounds if x.vienmandate) orgs.update(x.daugiamandate for x in term_rounds if x.daugiamandate) if orgs: for org in sorted(orgs): printer.info(' %s' % org) since, until = membership.get( org, (min(elected, default=term.since), term.until)) membership[org] = ( min(since, term.since), max(until, term.until) if until and term.until else term.until, ) printer.info(' Member of:') for org, (since, until) in membership.items(): printer.info(' %s (%s - %s)' % ( org, since.strftime('%Y-%m-%d'), until.strftime('%Y-%m-%d') if until else '', )) actor_title = 'seimo narys' if times_elected > 0 else '' actor, created = models.Actor.objects.get_or_create( birth_date=candidate.birth_date, first_name=candidate.first_name, last_name=candidate.last_name, defaults={ 'title': actor_title, 'times_elected': times_elected, 'times_candidate': times_candidate, }, ) if not created: modified = [ actor.times_elected != times_elected, actor.times_candidate != times_candidate, ] if any(modified): actor.times_elected = times_elected actor.times_candidate = times_candidate actor.save() for org, (since, until) in membership.items(): party, created = models.Actor.objects.get_or_create( first_name=org, group=True, defaults={ 'title': 'plitinė partija', 'body': seimas, }) models.Member.objects.get_or_create(actor=actor, group=party, defaults={ 'since': since, 'until': until, }) # Add this actor to all term groups in_groups = actor.ingroups.values_list('pk', flat=True) for term in actor_terms: if term.group.pk not in in_groups: term.group.members.add(actor) printer.info('done.')
def handle(self, path, **options): printer = helpers.Printer(self.stdout, options['verbosity']) path = pathlib.Path(path) # ------------------------------------------ # 'fname': <first name>, # 'lname': <last name>, # 'bdate': <birth date>, # 'title': <title or profession>, # 'partial': <bool: true if data covers only one or few elections, but not all of them>, # 'elections': [ # { # 'body': <body, for example 'Seimas'>, # 'since': <date, when elections started>, # 'until': <date, until next elections>, # 'party': <party>, # 'photo': { # 'ext': <photo file extension>, # 'data': <base64 encoded photo file>, # }, # 'elected': <bool: elected or not>, # } # ] now = datetime.datetime.now() strptime = datetime.datetime.strptime Photo = collections.namedtuple('Photo', 'type, data') Election = collections.namedtuple( 'Term', 'body, since, until, party, photo, elected') Candidate = collections.namedtuple( 'Candidate', 'fname, lname, bdate, partial, elections') with path.open(encoding='utf-8') as f: for line in f: row = json.loads(line) row['elections'] = [ Election(**dict( x, since=strptime(x['since'], '%Y-%m-%d'), until=strptime(x['until'], '%Y-%m-%d'), photo=Photo(**x['photo']), )) for x in row['elections'] ] candidate = Candidate(**dict( row, bdate=strptime(row['bdate']), )) # Update candidate printer.info('') printer.info('{bdate} {fname} {lname}'.format(**row)) actor, created = models.Actor.objects.get_or_create( birth_date=candidate.bdate, first_name=candidate.fname, last_name=candidate.lname, defaults={ 'title': candidate.title, }, ) # Update candidate photo if not actor.photo and candidate.elections: photo = candidate.elections[-1].photo filename = '%s-%s.%s' % (candidate.fname, candidate.lname, photo.ext) content = io.BytesIO( base64.b64decode(photo.data.encode('ascii'))) actor.photo.save(filename, File(content), save=True) actor_groups = actor.ingroups.values_list('pk', flat=True) actor_parties = { x.pk: x for x in models.Member.objects.filter(actor=actor) } for election in candidate.elections: # Get election body (must be created manually) body = models.Body.objects.get(name=election.body) # Get or create election term term, created = models.Term.objects.get_or_create( body=body, title='%d-%d metų kadencija' % ( election.since.year, election.until.year if election.until else now.year, )) # Get or create group for this election group_title = { 'Seimas': 'Seimą', } group, created = models.Group.objects.get_or_create( title='Kandidatai į %d metų %s' % ( election.since.year, group_title[election.body], ), defaults={ 'timestamp': election.since, }, ) # Add current candidate to the election group if group.pk not in actor_groups: group.members.add(actor) # Get or create party party, created = models.Actor.objects.get_or_create( first_name=election.party, group=True, defaults={ 'body': body, 'title': 'plitinė partija', }, ) # Update party membership information membership = actor_parties.get(party.pk, None) if membership is None: models.Member.objects.get_or_create(actor=actor, group=party, defaults={ 'since': election.since, 'until': election.until, }) elif ( (election.since and membership.since.date() != election.since.date()) or (election.until and membership.until.date() != election.until.date())): if election.since: membership.since = min(membership.since, election.since) if election.until: membership.until = max(membership.until, election.until) membership.save() printer.info('done.')
def handle(self, path, base, **options): printer = helpers.Printer(self.stdout, options['verbosity']) path = pathlib.Path(path) base = pathlib.Path(base) with path.open() as f: photos = yaml.safe_load(f) def clean_name(name): return unidecode.unidecode(name).lower() Person = collections.namedtuple('Person', 'id first_name last_name birth_date') names = collections.defaultdict(list) for x in models.Actor.objects.filter(birth_date__isnull=False, group=False): names[clean_name(x.first_name + ' ' + x.last_name)].append( Person( id=x.id, first_name=x.first_name, last_name=x.last_name, birth_date=x.birth_date, )) for photo in photos: cleaned_name = clean_name(photo['name']) if cleaned_name in names: matches = [ x for x in names[cleaned_name] if (photo.get('born') is None or photo['born'].strftime( '%Y-%m-%d') == x.birth_date.strftime('%Y-%m-%d')) ] if len(matches) == 1: x = matches[0] printer.info('- name: %s %s' % (x.first_name, x.last_name)) printer.info(' born: %s' % (x.birth_date.strftime('%Y-%m-%d'))) printer.info(' photo: %s' % photo['photo']) photo_path = (base / photo['photo']) if photo['photo'] else None if photo_path and photo_path.exists(): actor = models.Actor.objects.get(pk=x.id) if not actor.photo: with photo_path.open('rb') as f: actor.photo.save(path.name, File(f), save=True) elif photo_path: printer.info(" error: logo %r can't be found" % photo['photo']) else: printer.info('# edit: multiple matches') for x in matches: printer.info('- name: %s %s' % (x.first_name, x.last_name)) printer.info(' born: %s' % (x.birth_date.strftime('%Y-%m-%d'))) printer.info(' photo: %s' % photo['photo']) else: printer.info('# edit: no match found') printer.info('- name: %s' % photo['name']) printer.info(' photo: %s' % photo['photo'])