def handle(self, *args, **options): date_of = datetime.now().date() date_ = date_of.strftime(API_DATE_FORMAT) for org_id in getParentOrganizationsWithVoters(): sw = WordAnalysis(organization_id=org_id, count_of='groups', date_=date_) #if not sw.isNewSpeech: # return JsonResponse({'alliswell': True, # 'msg': 'Na ta dan ni bilo govorov'}) # Vocabolary size all_score = sw.getVocabularySize() max_score, maxPGid = sw.getMaxVocabularySize() avg_score = sw.getAvgVocabularySize() date_of = sw.getDate() maxPG = Organization.objects.get(id_parladata=maxPGid) print '[INFO] saving vocabulary size' for p in all_score: self.stdout.write('Settings organisation %s' % str(p['counter_id'])) org = Organization.objects.get( id_parladata=int(p['counter_id'])) saveOrAbortNew(model=VocabularySize, organization=org, created_for=date_of, score=int(p['coef']), maxOrg=maxPG, average=avg_score, maximum=max_score) self.stdout.write('DONE') return 0
def setPGMismatch(commander, pg_id, date=None): """ Setter for analysis mismatch of parlament group """ if date: date_of = datetime.strptime(date, API_DATE_FORMAT) else: date_of = datetime.now().date() date = '' org = Organization.objects.get(id_parladata=pg_id) memsOfPGs = getOrganizationsWithVotersList(date_=date_of) data = [] for member in memsOfPGs[int(pg_id)]: try: mismatch = getPersonCardModelNew(MismatchOfPG, int(member), date) data.append({'id': member, 'ratio': mismatch.data}) except: commander.stderr.write('No MismatchOfPG card for person %s' % str(member)) pass saveOrAbortNew(model=PGMismatch, organization=org, created_for=date_of, data=data) return
def handle(self, *args, **options): date_ = '' if options['date']: date_of = datetime.strptime(options['date'], API_DATE_FORMAT).date() date_ = options['date'] else: # dirty work around, TODO: fix findDatesFromLastCard for input without person_id #date_of = findDatesFromLastCard(Presence, '11', datetime.now().strftime(API_DATE_FORMAT))[0] date_of = datetime.now().date() date_ = date_of.strftime(API_DATE_FORMAT) data = getQuestions() self.stdout.write('Get voters') for org_id in getParentOrganizationsWithVoters(): mps_ids = getVotersIDs(date_=date_of, organization_id=org_id) print(mps_ids) authors = [] for question in data: for author in question['authors']: if author in mps_ids: authors.append(author) print(org_id, len(authors), float(len(mps_ids))) try: avg = len(authors) / float(len(mps_ids)) except ZeroDivisionError as error: avg = 0 question_count = Counter(authors) max_value = 0 max_persons = [] for maxi in question_count.most_common(90): if max_value == 0: max_value = maxi[1] if maxi[1] == max_value: max_persons.append(maxi[0]) else: break for person_id in mps_ids: person = Person.objects.get(id_parladata=person_id) saveOrAbortNew(model=NumberOfQuestions, created_for=date_of, person=person, score=question_count[person_id], average=avg, maximum=max_value, maxMPs=max_persons) self.stdout.write('Done setting NumberOfQuestions to MP %s' % str(person.id_parladata)) return 0
def setTfidfOfMP(commander, mp_id): url = '%s/tfidf/person?id=%s' % (ISCI_URL, mp_id) commander.stdout.write('About to fetch %s' % url) r = requests.get(url) commander.stdout.write('Saving speaker %s' % str(mp_id)) output = r.json() date_of = datetime.now().date() person = Person.objects.get(id_parladata=mp_id) data = output.get('tfidf', []) saveOrAbortNew(Tfidf, person=person, created_for=date_of, is_visible=False, data=data)
def setTfidfOfPG(commander, pg_id): url = '%s/tfidf/party?id=%s' % (ISCI_URL, pg_id) commander.stdout.write('About to fetch %s' % url) r = requests.get(url) commander.stdout.write('Saving PG %s' % str(pg_id)) output = r.json() date_of = datetime.now().date() organization = Organization.objects.get(id_parladata=pg_id) data = output.get('tfidf', []) saveOrAbortNew(Tfidf, organization=organization, created_for=date_of, is_visible=False, data=data)
def setTfidfOfSession(commander, session_id): url = '%s/tfidf/session?id=%s' % (ISCI_URL, session_id) commander.stdout.write('About to fetch %s' % url) r = requests.get(url) commander.stdout.write('Saving session %s' % str(session_id)) output = r.json() date_of = datetime.now().date() session = Session.objects.get(id_parladata=session_id) data = output.get('tfidf', []) saveOrAbortNew(Tfidf, session=session, created_for=date_of, is_visible=False, data=data)
def setPresenceOfPGs(self): """ set presence for all Partys """ table = self.presencePGs averageSessions = table['sessions'].mean() averageVotes = table['votes'].mean() maxVotes = table['votes'].max() maxVotesOrgIdx = table['votes'].idxmax() maxSessions = table['sessions'].max() maxSessionOrgIdx = table['sessions'].idxmax() for pg in self.pgs: try: thisSessions = table[table.pg == pg].sessions[pg] thisVotes = table[table.pg == pg].votes[pg] except: continue thisOrg = Organization.objects.get(id_parladata=pg) result = saveOrAbortNew(model=PercentOFAttendedSession, created_for=self.date_of, organization=thisOrg, organization_value_sessions=thisSessions, maxPG_sessions=[maxSessionOrgIdx], average_sessions=averageSessions, maximum_sessions=maxSessions, organization_value_votes=thisVotes, maxPG_votes=[maxVotesOrgIdx], average_votes=averageVotes, maximum_votes=maxVotes)
def setPercentOFAttendedSessionPG(pg_id, date_of, parenet_org): date_ = date_of.strftime(API_DATE_FORMAT) allSum = {} data = {} voters = getVotersPairsWithOrg(date_=date_of, organization_id=parenet_org) membersOfPG = {} for i, value in sorted(voters.iteritems()): membersOfPG.setdefault(value, []).append(i) data = getNumberOfAllMPAttendedSessions(date_of, getVotersIDs(date_=date_of, organization_id=parenet_org)) sessions = {pg: [] for pg in membersOfPG if membersOfPG[pg]} votes = {pg: [] for pg in membersOfPG if membersOfPG[pg]} for pg in membersOfPG: if not membersOfPG[pg]: continue for member in membersOfPG[pg]: if str(member) in data['sessions'].keys(): sessions[pg].append(data['sessions'][str(member)]) votes[pg].append(data['votes'][str(member)]) sessions[pg] = sum(sessions[pg])/len(sessions[pg]) votes[pg] = sum(votes[pg])/len(votes[pg]) thisMPSessions = sessions[pg_id] maximumSessions = max(sessions.values()) maximumPGSessions = [pgId for pgId in sessions if sessions[pgId] == maximumSessions][:5] averageSessions = sum(data['sessions'].values()) / len(data['sessions']) thisMPVotes = votes[pg_id] maximumVotes = max(votes.values()) maximumPGVotes = [pgId for pgId in votes if votes[pgId] == maximumVotes][:5] averageVotes = sum(data['votes'].values()) / len(data['votes']) org = Organization.objects.get(id_parladata=int(pg_id)) result = saveOrAbortNew(model=PercentOFAttendedSession, created_for=date_of, organization=org, organization_value_sessions=thisMPSessions, maxPG_sessions=maximumPGSessions, average_sessions=averageSessions, maximum_sessions=maximumSessions, organization_value_votes=thisMPVotes, maxPG_votes=maximumPGVotes, average_votes=averageVotes, maximum_votes=maximumVotes)
def setMPsOfPG(commander, pg_id, date_=None): if date_: date_of = datetime.strptime(date_, API_DATE_FORMAT).date() commander.stdout.write('Setting for date %s' % str(date_of)) else: date_of = datetime.now().date() date_ = date_of.strftime(API_DATE_FORMAT) commander.stdout.write('Setting for today (%s)' % str(date_of)) pairs = getVotersPairsWithOrg() membersOfPG = {i: [] for i in set(pairs.values())} for mem, org in pairs.items(): membersOfPG[org].append(mem) org = Organization.objects.get(id_parladata=pg_id) commander.stdout.write('Setting for organisation %s' % str(pg_id)) saveOrAbortNew(model=MPOfPg, organization=org, id_parladata=pg_id, MPs=membersOfPG[pg_id], created_for=date_of)
def setPresenceThroughTime(commander, person_id, date_=None): if date_: fdate = datetime.strptime(date_, '%d.%m.%Y').date() else: fdate = datetime.now().date() person = Person.objects.filter(id_parladata=person_id) if not person: commander.stdout.write('Person with id %s doesnt exist' % str(pg)) return data = getBallotsCounter(person[0], fdate) data_for_save = [] commander.stdout.write('Iterating through months for person %s' % str(person_id)) for month in data: options = YES + NOT_PRESENT + AGAINST + ABSTAIN stats = sum( [month[option] for option in options if option in month.keys()]) not_member = month['total'] - stats not_member = float(not_member) / month['total'] if not_member else 0 presence = float(stats - sum([ month[option] for option in NOT_PRESENT if option in month.keys() ])) / month['total'] if stats else 0 data_for_save.append({ 'date_ts': month['date_ts'], 'presence': presence * 100, 'not_member': not_member * 100, 'vote_count': month['total'] }) saveOrAbortNew(model=PresenceThroughTime, person=Person.objects.get(id_parladata=person_id), created_for=fdate, data=data_for_save) commander.stdout.write('Set PresenceThroughTime for person id %s' % str(person_id))
def setPresenceOfPG(commander, session_id): """ Stores presence of PGs on specific session """ PGs = getOrganizationsWithVoters() commander.stdout.write( 'About to get ballots for session') votes = getBallotsForSession(session_id) counters_in = Counter([vote['voterparty'] for vote in votes if vote['option'] not in NOT_PRESENT]) counters_out = Counter([vote['voterparty'] for vote in votes if vote['option'] in NOT_PRESENT]) pgs = list(set(counters_in.keys() + counters_out.keys())) results = {} for pg in pgs: if not pg in PGs: continue try: results[pg] = counters_in[pg] * 100 / \ (counters_in[pg] + counters_out[pg]) except: if pg in counters_in.keys(): results[pg] = 100 elif pg in counters_out.keys(): results[pg] = 0 else: commander.stderr.write( 'Something is wrong with PG %s' % str(pg)) session = Session.objects.get(id_parladata=session_id) saveOrAbortNew(model=PresenceOfPG, created_for=session.start_time, presence=[results], session=session) commander.stdout.write( 'Successfully saved presence of PGs for session %s' % str(session_id))
def setMembershipsOfMember(commander, person_id, date_=None): if date_: date_of = datetime.strptime(date_, API_DATE_FORMAT) else: date_of = datetime.now().date() organizations = {org['id']: org for org in getOrganizations()} memberships = getMembershipsOfMember(person_id=person_id, date_=date_of) data = defaultdict(list) person = Person.objects.get(id_parladata=int(person_id)) for mem in memberships: if mem['role'] != 'voter': continue organization = organizations[mem['organization']] org_links = getLinks(organization=organization['id']) if org_links: org_link = org_links[0]['url'] else: org_link = None data[organization['classification']].append({ 'url': org_link, 'org_type': organization['classification'], 'org_id': organization['id'], 'name': organization['_name'] }) saveOrAbortNew(MembershipsOfMember, created_for=date_of, person=person, data=dict(data)) commander.stdout.write('Set MembershipsOfMember for person id %s' % str(person_id))
def setPGPresenceThroughTime(commander, pg, date): if date: date_of = datetime.strptime(date, API_DATE_FORMAT).date() else: date_of = datetime.now().date() org = Organization.objects.filter(id_parladata=pg) if not org: commander.stdout.write('Organization with id %s doesnt exist' % str(pg)) return else: org = org[0] data = getBallotsCounter(org, date_of) data_for_save = [] for month in data: options = YES + NOT_PRESENT + AGAINST + ABSTAIN stats = sum( [month[option] for option in options if option in month.keys()]) presence = float(stats - sum([ month[option] for option in NOT_PRESENT if option in month.keys() ])) / stats if stats else 0 data_for_save.append({ 'date_ts': month['date_ts'], 'presence': presence * 100, }) org = Organization.objects.get(id_parladata=pg) commander.stdout.write('Saving presence for organization %s' % str(pg)) saveOrAbortNew(model=PresenceThroughTime, organization=org, created_for=date_of, data=data_for_save) return
def setPercentOFAttendedSession(commander, members, date_of=datetime.now().date()): data = getNumberOfAllMPAttendedSessions(date_of, members) for person_id in members: if not data["votes"].values(): commander.stdout.write('Member with id: %s has not votes' % str(person_id)) return if person_id in data["sessions"].keys(): thisMP = data["sessions"][person_id] else: # ta member se ni obstajal commander.stdout.write('Member with id %s didn\'t exist' % str(person_id)) return org_sessions_values = [value for key, value in data["sessions"].items() if int(key) in members] maximum = max(org_sessions_values) maximumMP = [pId for pId in data["sessions"] if data["sessions"][pId] == maximum and int(pId) in members] average = sum(org_sessions_values) / len(org_sessions_values) if person_id in data["votes"].keys(): thisMPVotes = data["votes"][person_id] else: thisMPVotes = 0 org_votes_values = [value for key, value in data["votes"].items() if int(key) in members] maximumVotes = max(org_votes_values) maximumMPVotes = [pId for pId in data["votes"] if data["votes"][pId] == maximumVotes and int(pId) in members] averageVotes = sum(org_votes_values) / len(org_votes_values) person = Person.objects.get(id_parladata=int(person_id)) result = saveOrAbortNew(model=Presence, created_for=date_of, person=person, person_value_sessions=thisMP, maxMP_sessions=maximumMP, average_sessions=average, maximum_sessions=maximum, person_value_votes=thisMPVotes, maxMP_votes=maximumMPVotes, average_votes=averageVotes, maximum_votes=maximumVotes) commander.stdout.write('Set Presence for member with id %s' % str(person_id))
def setPresenceMPs(self): """ set presence for all Members """ votes = self.presenceMP_V sessions = self.presenceMP_S actualVotes = votes[votes.voter.isin(self.members)] avgVote = actualVotes[['attendance']].mean().attendance maxVote = actualVotes.max() max_vote_value = actualVotes['attendance'].max() max_vote_voters = actualVotes[actualVotes.attendance == max_vote_value]['voter'].tolist() actualSession = sessions[sessions.voter.isin(self.members)] avgSession = actualSession[['attendance']].mean().attendance max_session_value = actualSession['attendance'].max() max_ses_voters = actualSession[actualSession.attendance == max_session_value]['voter'].tolist() for mp in self.members: try: person = Person.objects.get(id_parladata=int(mp)) thisVotes = votes[votes.voter == mp].reset_index().at[ 0, 'attendance'] tempS = sessions[sessions.voter == mp].reset_index() thisSession = tempS.at[0, 'attendance'] print mp result = saveOrAbortNew(model=Presence, created_for=self.date_of, person=person, person_value_sessions=thisSession, maxMP_sessions=max_ses_voters, average_sessions=avgSession, maximum_sessions=max_session_value, person_value_votes=thisVotes, maxMP_votes=max_vote_voters, average_votes=avgVote, maximum_votes=max_vote_value) except: print mp, 'fail set presence'
def setEqualVoters(self): """ set cards how equal are voters """ for mp in self.members: print '.:' + str(mp) + ':.' person = Person.objects.get(id_parladata=int(mp)) try: r = self.equalVotersData[mp] except: print mp, 'fail set equal voters' continue mps_data = r.reset_index().sort_values(mp, ascending=False) # most equal most_data = [] data = mps_data for idx in range(len(data) - 1): member_id = int(data.iloc[idx]['voter']) if member_id in self.members: member = Person.objects.get(id_parladata=member_id) most_data.append({ 'member': member, 'score': data.iloc[idx][mp] }) most_data = list(reversed(most_data)) result = saveOrAbortNew(model=EqualVoters, created_for=self.date_of, person=person, person1=most_data[0]['member'], votes1=most_data[0]['score'], person2=most_data[1]['member'], votes2=most_data[1]['score'], person3=most_data[2]['member'], votes3=most_data[2]['score'], person4=most_data[3]['member'], votes4=most_data[3]['score'], person5=most_data[4]['member'], votes5=most_data[4]['score']) # less equal less_data = [] data = mps_data #for idx in range(5): idx = 0 while len(less_data) < 5: member_id = int(data.iloc[idx]['voter']) if member_id in self.members: member = Person.objects.get(id_parladata=member_id) less_data.append({ 'member': member, 'score': data.iloc[idx][mp] }) idx += 1 result = saveOrAbortNew(model=LessEqualVoters, created_for=self.date_of, person=person, person1=less_data[0]['member'], votes1=less_data[0]['score'], person2=less_data[1]['member'], votes2=less_data[1]['score'], person3=less_data[2]['member'], votes3=less_data[2]['score'], person4=less_data[3]['member'], votes4=less_data[3]['score'], person5=less_data[4]['member'], votes5=less_data[4]['score'])
def setMotionOfSession(commander, session_id): """Stores all motions with detiled data of specific sesison. """ if commander: commander.stdout.write('Beginning setMotionOfSession for session %s' % str(session_id)) commander.stdout.write('Getting for motion for session %s' % (str(session_id))) votes = parladata_api.getVotesForSession(session_id) session = Session.objects.get(id_parladata=session_id) laws = [] for vote in votes: if commander: commander.stdout.write('Handling vote %s' % str(vote['id'])) if not vote['name']: commander.stdout.write('Skipping vote because it has no name: %s' % str(vote['id'])) motion = parladata_api.getMotion(vote['motion']) if vote['results']: votes_for = vote['results']['for'] votes_against = vote['results']['against'] votes_abstain = vote['results']['abstain'] votes_absent = vote['results']['absent'] if vote['counter']: # this is for votes without ballots counter = json.loads(vote['counter']) opts_set = set(counter.keys()) if opts_set.intersection(YES): votes_for = counter['for'] if opts_set.intersection(AGAINST): votes_against = counter['against'] if opts_set.intersection(ABSTAIN): votes_abstain = counter['abstain'] # hardcoded croations number of member votes_absent = 151 - sum([int(v) for v in counter.values()]) result = motion['result'] documents = [{ 'url': link['url'], 'name': link['url'] } for link in motion['links']] # TODO: replace try with: "if vote['epa']" try: law = Legislation.objects.get(epa=motion['epa']) laws.append(law) except: law = None classification = getMotionClassification(vote['name']) vote_obj = Vote.objects.filter(id_parladata=vote['id']) agendaItems = list( AgendaItem.objects.filter(id_parladata__in=motion['agenda_item'])) if vote_obj: if commander: commander.stdout.write('Updating vote %s' % str(vote['id'])) commander.stdout.write('Updating data %s' % str(vote)) prev_result = vote_obj[0].result vote_obj.update( created_for=session.start_time, start_time=vote['start_time'], session=session, motion=vote['name'], tags=vote['tags'], votes_for=votes_for, against=votes_against, abstain=votes_abstain, not_present=votes_absent, result=result, id_parladata=vote['id'], document_url=documents, epa=motion['epa'], law=law, classification=classification, ) vote_obj = vote_obj[0] vote_obj.agenda_item.add(*agendaItems) if prev_result != vote_obj.result: if commander: commander.stdout.write( 'Running finish_legislation_by_final_vote(vote[0])') finish_legislation_by_final_vote(vote_obj) else: if commander: commander.stdout.write('Saving new vote %s' % str(vote['id'])) result = saveOrAbortNew( model=Vote, created_for=session.start_time, start_time=vote['start_time'], session=session, motion=vote['name'], tags=vote['tags'], votes_for=votes_for, against=votes_against, abstain=votes_abstain, not_present=votes_absent, result=result, id_parladata=vote['id'], document_url=documents, epa=motion['epa'], law=law, classification=classification, ) vote_obj = Vote.objects.get(id_parladata=vote['id']) commander.stdout.write( 'Running finish_legislation_by_final_vote(vote[0])') finish_legislation_by_final_vote(vote_obj) owners = getOwnersOfAmendment(vote_obj) if owners['orgs']: a_orgs = list( Organization.objects.filter(id_parladata__in=owners['orgs'])) else: a_orgs = [] if owners['people']: a_people = Person.objects.filter(id_parladata__in=owners['people']) else: a_people = [] if a_orgs: vote_obj.amendment_of.clear() for org in a_orgs: AmendmentOfOrg(vote=vote_obj, organization=org).save() vote_obj.amendment_of_person.add(*a_people) # set motion analize if commander: commander.stdout.write('Running setMotionAnalize for %s' % str(session_id)) setMotionAnalize(session_id) # TODO figure out what to do with this # if laws: # recacheLegislationsOnSession(session_id) if commander: commander.stdout.write('Running deleteRendersOfSessionVotes for %s' % str(session_id)) deleteRendersOfSession([session_id]) return 0
def handle(self, *args, **options): if options['date']: date_of = datetime.strptime(options['date'], API_DATE_FORMAT) else: date_of = datetime.now().date() date_ = date_of.strftime(API_DATE_FORMAT) for org_id in getParentOrganizationsWithVoters(): sw = WordAnalysis(organization_id=org_id, count_of='members', date_=date_) #if not sw.isNewSpeech: # return JsonResponse({'alliswell': False}) #Vocabolary size all_score = sw.getVocabularySize() max_score, maxMPid = sw.getMaxVocabularySize() avg_score = sw.getAvgVocabularySize() date_of = sw.getDate() maxMP = Person.objects.get(id_parladata=maxMPid) self.stdout.write('[INFO] saving vocabulary size') for p in all_score: self.stdout.write( '[INFO] saving vocabulary size for person %s' % str(p['counter_id'])) saveOrAbortNew(model=VocabularySize, person=Person.objects.get( id_parladata=int(p['counter_id'])), created_for=date_of, score=int(p['coef']), maxMP=maxMP, average=avg_score, maximum=max_score) #Unique words all_score = sw.getUniqueWords() max_score, maxMPid = sw.getMaxUniqueWords() avg_score = sw.getAvgUniqueWords() date_of = sw.getDate() maxMP = Person.objects.get(id_parladata=maxMPid) self.stdout.write('[INFO] saving unique words') for p in all_score: self.stdout.write('[INFO] saving unique words for person %s' % str(p['counter_id'])) saveOrAbortNew(model=VocabularySizeUniqueWords, person=Person.objects.get( id_parladata=int(p['counter_id'])), created_for=date_of, score=int(p['unique']), maxMP=maxMP, average=avg_score, maximum=max_score) #Spoken words all_words = sw.getSpokenWords() max_words, maxWordsMPid = sw.getMaxSpokenWords() avgSpokenWords = sw.getAvgSpokenWords() date_of = sw.getDate() maxMP = Person.objects.get(id_parladata=maxWordsMPid) self.stdout.write('[INFO] saving spoken words') for p in all_words: self.stdout.write('[INFO] saving spoken words for person %s' % str(p['counter_id'])) saveOrAbortNew(model=SpokenWords, created_for=date_of, person=Person.objects.get( id_parladata=int(p['counter_id'])), score=int(p['wordcount']), maxMP=maxMP, average=avgSpokenWords, maximum=max_words) self.stdout.write('[INFO] All MPs updated') return 0
def set_mismatch_of_pg(request, by_organization, date_=''): print 'Preparing date' if date_: f_date = datetime.strptime(date_, '%d.%m.%Y') else: f_date = datetime.now() #url = settings.API_URL_V2 + '/getVotesTableExtended/'+ str(by_organization) + '/' + date_ data = pd.DataFrame() for page in getBallotTable(organization=by_organization): temp = pd.DataFrame(page) data = data.append(temp, ignore_index=True) print 'Preparing pandas DataFrame' data['option_absent'] = 0 data['option_for'] = 0 data['option_against'] = 0 data['option_abstain'] = 0 data.loc[data['option'] == 'absent', 'option_absent'] = 1 data.loc[data['option'] == 'for', 'option_for'] = 1 data.loc[data['option'] == 'against', 'option_against'] = 1 data.loc[data['option'] == 'abstain', 'option_abstain'] = 1 data['voter_unit'] = 1 print 'Prepared pandas DataFrame, about to start analyzing things.' #for against absent abstain # all_votes = data.groupby('vote').sum() m_to_p = getVotersPairsWithOrg(organization_id=by_organization) mppgs = pd.DataFrame(m_to_p.items(), columns=['voter', 'voterparty']) #Get ballots of last members party keys = ['voter', 'voterparty'] i1 = mppgs.set_index(keys).index i2 = data.set_index(keys).index data2 = data[i2.isin(i1)] def getPartyBallot(row): """ using for set ballot of party: methodology: ignore not_present """ stats = {'for': row['option_for'], 'against': row['option_against'], 'abstain': row['option_abstain'], 'absent': row['option_absent']} if max(stats.values()) == 0: return None max_ids = [key for key, val in stats.iteritems() if val == max(stats.values())] #if len(max_ids) > 1: # return None #return max_ids[0] return ','.join(max_ids) partyBallots = data.groupby(['vote', 'voterparty']).sum().apply(lambda row: getPartyBallot(row), axis=1) partys = partyBallots.reset_index() partys = partys.rename(columns = {0:'partyoption'}) result = pd.merge(data2, partys, on=['vote', 'voterparty']) #remove absent option TURNED OFF # result = result[result.option != 'absent'] members_vote_count = result[['voter','voter_unit']].groupby('voter').count() def is_equal(row): if row['partyoption'] and row['option'] in row['partyoption']: return 1 else: return 2 equal_indexes = result[['option', 'partyoption']].apply(lambda x: is_equal(x), axis=1).reindex() result['ivan'] = result[['option', 'partyoption']].apply(lambda x: is_equal(x), axis=1).reindex() out = pd.concat([result, equal_indexes], axis=1).rename(columns = {0: 'equal_vote'}) out = out[out['equal_vote']==1] members_equal_count = out[['voter', 'equal_vote']].groupby('voter').sum() final = pd.concat([members_equal_count, members_vote_count], axis=1) final['percent'] = final.apply(lambda x: float(x['equal_vote'])/x['voter_unit']*100.0, axis=1) print 'saving' data = [] for member, row in final.iterrows(): print member value = 100 - row['percent'] person = Person.objects.get(id_parladata=member) party = person.static_data.latest('created_at').party party_classification = party.classification # TODO: make fix code under for people without party #if party_classification != settings.PS: # value = None data.append({'person': person, 'value': value}) maxMismatch = max(data, key=lambda x:x['value'] if x['value'] else 0) values = [i['value'] for i in data if i['value']] avg = float(sum(values))/len(values) for d in data: saveOrAbortNew(model=MismatchOfPG, person=d['person'], created_for=f_date, average=avg, maximum=maxMismatch['value'], maxMP=maxMismatch['person'], data=d['value']) return JsonResponse({'alliswell': True})
def handle(self, *args, **options): date_of = datetime.now().date() date_ = date_of.strftime(API_DATE_FORMAT) for org in getParentOrganizationsWithVoters(): self.stdout.write('getting organizations with voters') organization_ids = getOrganizationsWithVoters(date_=date_of) scores = {} for organization_id in organization_ids: self.stdout.write('Org id: %s' % str(organization_id)) # get word counts with solr counter = Counter( getCountList(self, int(organization_id), date_)) total = sum(counter.values()) scores_local = getScores( [problematicno, privzdignjeno, preprosto], counter, total) self.stdout.write('Outputting scores_local: %s' % str(scores_local)) scores[organization_id] = scores_local self.stdout.write('Outputting scores: %s' % str(scores)) average = { "problematicno": sum([score['problematicno'] for score in scores.values()]) / len(scores), "privzdignjeno": sum([score['privzdignjeno'] for score in scores.values()]) / len(scores), "preprosto": sum([score['preprosto'] for score in scores.values()]) / len(scores) } data = [] for org_id, score in scores.items(): data.append({ 'org': org_id, 'problematicno': score['problematicno'], 'privzdignjeno': score['privzdignjeno'], 'preprosto': score['preprosto'], 'problematicno_average': average['problematicno'], 'privzdignjeno_average': average['privzdignjeno'], 'preprosto_average': average['preprosto'] }) for score in data: self.stdout.write('About to save %s' % str(score)) status = saveOrAbortNew( StyleScores, organization=Organization.objects.get( id_parladata=int(score['org'])), created_for=date_of, problematicno=float(score['problematicno']), privzdignjeno=float(score['privzdignjeno']), preprosto=float(score['preprosto']), problematicno_average=float( score['problematicno_average']), privzdignjeno_average=float( score['privzdignjeno_average']), preprosto_average=float(score['preprosto_average'])) self.stdout.write('SaveOrAbort status: %s' % str(status)) return 0
def setBasicInfoOfPG(commander, pg, date): if date: date_of = datetime.strptime(date, API_DATE_FORMAT).date() else: date_of = datetime.now().date() org_data = getOrganizations(pg) contacts = getContactDetails(organization=pg) president = getPosts(date_of, organization=pg, role="president") deputy = getPosts(date_of, organization=pg, role="deputy") email = '' for contact in contacts: if contact['contact_type'] == 'EMAIL': email = contact['value'] facebook = [ link['url'] for link in getLinks(organization=pg, tags__name='fb') ] twitter = [ link['url'] for link in getLinks(organization=pg, tags__name='tw') ] numberOfSeats = dict(Counter( getVotersPairsWithOrg(date_=date_of).values()))[int(pg)] headOfPG = None viceOfPG = [] if president: commander.stdout.write('Defiing head of PG') headOfPG = Person.objects.get(id_parladata=int(president[0]['person'])) else: commander.stdout.write('No head of PG') headOfPG = None if deputy: for vice in deputy: if vice: viceOfPG.append(vice['person']) else: viceOfPG.append(None) else: viceOfPG.append(None) gov_sides = getCoalitionPGs(date_of, parent_org=session.organization.id_parladata) is_coalition = False for gov_side, orgs in gov_sides.items(): if Organization.objects.get(id_parladata=gov_side).is_coalition: is_coalition = pg.id_parladata in orgs org = Organization.objects.get(id_parladata=int(pg)) commander.stdout.write('Saving organization %s' % str(pg)) saveOrAbortNew(model=PGStatic, created_for=date_of, organization=org, headOfPG=headOfPG, viceOfPG=viceOfPG, numberOfSeats=numberOfSeats, allVoters=org_data['voters'], facebook=json.dumps(facebook) if facebook else None, twitter=json.dumps(twitter) if twitter else None, email=email, is_coalition=is_coalition) return
def handle(self, *args, **options): if options['date']: date_ = options['date'] date_of = datetime.strptime(date_, API_DATE_FORMAT).date() else: date_ = datetime.now().date().strftime(API_DATE_FORMAT) date_of = datetime.now().date() data = getQuestions() for org_id in getParentOrganizationsWithVoters(): pgs_on_date = getOrganizationsWithVoters(date_=date_of, organization_id=org_id) mps = getVotersIDs(date_=date_of, organization_id=org_id) mpStatic = {} for mp in mps: mpStatic[str(mp)] = getPersonData(str(mp), date_) pg_ids = pgs_on_date authors = [] for question in data: qDate = datetime.strptime(question['date'], '%Y-%m-%dT%X') qDate = qDate.strftime(API_DATE_FORMAT) for author in question['authors']: try: person_data = mpStatic[str(author)] except KeyError as e: print(str(question['authors'])) person_data = getPersonData(str(author), date_) mpStatic[str(author)] = person_data if person_data and person_data['party'] and person_data[ 'party']['id']: authors.append(person_data['party']['id']) else: print 'person nima mpstatic: ', author avg = len(authors) / float(len(pg_ids)) question_count = Counter(authors) max_value = 0 max_orgs = [] for maxi in question_count.most_common(90): if max_value == 0: max_value = maxi[1] if maxi[1] == max_value: max_orgs.append(maxi[0]) else: break is_saved = [] for pg_id in pg_ids: org = Organization.objects.get(id_parladata=pg_id) is_saved.append( saveOrAbortNew(model=NumberOfQuestions, created_for=date_of, organization=org, score=question_count[pg_id], average=avg, maximum=max_value, maxOrgs=max_orgs)) return 0
def handle(self, *args, **options): if options['date']: date_ = options['date'] date_of = datetime.strptime(API_DATE_FORMAT).date() else: date_of = datetime.now().date() date_ = date_of.strftime(API_DATE_FORMAT) self.stdout.write('Getting voters') for org in getParentOrganizationsWithVoters(): self.stdout.write('Starting style score for organization %s' % (org)) mps = getVotersIDs(organization_id=org, date_=date_of) scores = {} for person_id in mps: self.stdout.write('MP id: %s' % str(person_id)) # get word counts with solr counter = Counter(getCountList(self, int(person_id), date_)) total = sum(counter.values()) scores_local = getScores([problematicno, privzdignjeno, preprosto], counter, total) self.stdout.write('Outputting scores_local: %s' % str(scores_local)) scores[person_id] = scores_local self.stdout.write('Outputting scores: %s' % str(scores)) average = {"problematicno": sum([score['problematicno'] for score in scores.values()])/len(scores), "privzdignjeno": sum([score['privzdignjeno'] for score in scores.values()])/len(scores), "preprosto": sum([score['preprosto'] for score in scores.values()])/len(scores)} data = [] for person, score in scores.items(): data.append({'member': person, 'problematicno': score['problematicno'], 'privzdignjeno': score['privzdignjeno'], 'preprosto': score['preprosto'], 'problematicno_average': average['problematicno'], 'privzdignjeno_average': average['privzdignjeno'], 'preprosto_average': average['preprosto']}) for score in data: self.stdout.write('About to save %s' % str(score)) status = saveOrAbortNew(StyleScores, person=Person.objects.get( id_parladata=int(score["member"])), created_for=date_of, problematicno=float( score['problematicno']), privzdignjeno=float( score['privzdignjeno']), preprosto=float(score['preprosto']), problematicno_average=float( score['problematicno_average']), privzdignjeno_average=float( score['privzdignjeno_average']), preprosto_average=float( score['preprosto_average']) ) self.stdout.write('SaveOrAbort status: %s' % str(status)) return 0
def setEqualVotesPG(self): """ set cards how equal are voters to Partys """ results = self.equalVotersPGData for pg in results.keys(): org = Organization.objects.get(id_parladata=int(pg)) r = results[pg] r = r.reset_index().sort_values(pg, ascending=False) print(self.memsOfPGs) if int(pg) not in self.memsOfPGs.keys(): "if this PG hasn't members on this time" continue # TODO notify someone if this fails thisPG = r[r.voter.isin(self.memsOfPGs[int(pg)])] otherMembers = [ m for pg_key in self.memsOfPGs.keys() if pg_key != str(pg) for m in self.memsOfPGs[pg_key] ] print 'pg', pg, len(self.memsOfPGs[pg]), len(otherMembers) otherMems = r[r.voter.isin(otherMembers)] # most equal most_data = [] data = otherMems # for idx in range(5): for idx in range(len(data) - 1): member_id = int(data.iloc[idx]['voter']) if member_id in self.members: member = Person.objects.get(id_parladata=member_id) most_data.append({ 'member': member, 'score': data.iloc[idx][pg] }) most_data = list(reversed(most_data)) result = saveOrAbortNew(model=MostMatchingThem, created_for=self.date_of, organization=org, person1=most_data[0]['member'], votes1=most_data[0]['score'], person2=most_data[1]['member'], votes2=most_data[1]['score'], person3=most_data[2]['member'], votes3=most_data[2]['score'], person4=most_data[3]['member'], votes4=most_data[3]['score'], person5=most_data[4]['member'], votes5=most_data[4]['score']) # less equal less_data = [] data = otherMems #for idx in range(5): idx = 0 while len(less_data) < 5: member_id = int(data.iloc[idx]['voter']) if member_id in self.members: member = Person.objects.get(id_parladata=member_id) less_data.append({ 'member': member, 'score': data.iloc[idx][pg] }) idx += 1 result = saveOrAbortNew(model=LessMatchingThem, created_for=self.date_of, organization=org, person1=less_data[0]['member'], votes1=less_data[0]['score'], person2=less_data[1]['member'], votes2=less_data[1]['score'], person3=less_data[2]['member'], votes3=less_data[2]['score'], person4=less_data[3]['member'], votes4=less_data[3]['score'], person5=less_data[4]['member'], votes5=less_data[4]['score']) # deviation print thisPG dev_data = [] for idx in range(thisPG.count()['voter']): dev_data.append({ 'id': thisPG.iloc[idx]['voter'], 'ratio': thisPG.iloc[idx][pg] }) result = saveOrAbortNew(model=DeviationInOrganization, created_for=self.date_of, organization=org, data=dev_data)
def setMPStaticPL(commander, person_id, date_=None): if date_: date_of = date_ else: date_of = datetime.now() commander.stdout.write('Fetching data from %s/persons/%s with day %s' % (API_URL, str(person_id), date_of)) data = getPeople(id_=person_id) try: org_id = getVotersPairsWithOrg(date_=date_of)[int(person_id)] except Exception as e: commander.stdout.write( 'Person with ID %s has not correctly configured voter membership or he\'s not a MP' % str(person_id)) commander.stdout.write(str(e)) return organization = Organization.objects.get(id_parladata=org_id) person = Person.objects.get(id_parladata=int(person_id)) socials = {'fb': 'facebook', 'tw': 'twitter', 'linkedin': 'linkedin'} social_objs = {} for key, name in socials.items(): social_objs[name] = None for resp_data in getLinks(person=person_id, tags__name=key): if resp_data: social_objs[name] = resp_data['url'] if not data: commander.stderr.write('Didn\'t get data.') raise CommandError('No data returned.') if 'error' in data.keys(): commander.stderr.write('[API ERROR] %s' % data['error']) result = saveOrAbortNew( model=MPStaticPL, created_for=date_of, person=person, voters=data['voters'], points=data['points'], age=num_years(dateparse.parse_datetime(data['birth_date'])) if data['birth_date'] else None, birth_date=dateparse.parse_datetime(data['birth_date']) if data['birth_date'] else None, mandates=data['mandates'], party=organization, education=data['education'], education_level=data['education_level'], previous_occupation=data['previous_occupation'], name=data['name'], district=data['districts'], facebook=social_objs['facebook'], twitter=social_objs['twitter'], linkedin=social_objs['linkedin'], party_name=organization.name, acronym=organization.acronym, gov_id=data['gov_id'], gender='m' if data['gender'] == 'male' else 'f', working_bodies_functions=[]) commander.stdout.write('Set MP with id %s' % str(person_id))