def handle(self, *args, **options):
date_of = datetime.now().date()
date_ = date_of.strftime(API_DATE_FORMAT)
for org_id in getParentOrganizationsWithVoters():
sw = WordAnalysis(organization_id=org_id,
count_of='groups',
date_=date_)
#if not sw.isNewSpeech:
# return JsonResponse({'alliswell': True,
# 'msg': 'Na ta dan ni bilo govorov'})
# Vocabolary size
all_score = sw.getVocabularySize()
max_score, maxPGid = sw.getMaxVocabularySize()
avg_score = sw.getAvgVocabularySize()
date_of = sw.getDate()
maxPG = Organization.objects.get(id_parladata=maxPGid)
print '[INFO] saving vocabulary size'
for p in all_score:
self.stdout.write('Settings organisation %s' %
str(p['counter_id']))
org = Organization.objects.get(
id_parladata=int(p['counter_id']))
saveOrAbortNew(model=VocabularySize,
organization=org,
created_for=date_of,
score=int(p['coef']),
maxOrg=maxPG,
average=avg_score,
maximum=max_score)
self.stdout.write('DONE')
return 0
def setPGMismatch(commander, pg_id, date=None):
"""
Setter for analysis mismatch of parlament group
"""
if date:
date_of = datetime.strptime(date, API_DATE_FORMAT)
else:
date_of = datetime.now().date()
date = ''
org = Organization.objects.get(id_parladata=pg_id)
memsOfPGs = getOrganizationsWithVotersList(date_=date_of)
data = []
for member in memsOfPGs[int(pg_id)]:
try:
mismatch = getPersonCardModelNew(MismatchOfPG, int(member), date)
data.append({'id': member, 'ratio': mismatch.data})
except:
commander.stderr.write('No MismatchOfPG card for person %s' %
str(member))
pass
saveOrAbortNew(model=PGMismatch,
organization=org,
created_for=date_of,
data=data)
return
def handle(self, *args, **options):
date_ = ''
if options['date']:
date_of = datetime.strptime(options['date'],
API_DATE_FORMAT).date()
date_ = options['date']
else:
# dirty work around, TODO: fix findDatesFromLastCard for input without person_id
#date_of = findDatesFromLastCard(Presence, '11', datetime.now().strftime(API_DATE_FORMAT))[0]
date_of = datetime.now().date()
date_ = date_of.strftime(API_DATE_FORMAT)
data = getQuestions()
self.stdout.write('Get voters')
for org_id in getParentOrganizationsWithVoters():
mps_ids = getVotersIDs(date_=date_of, organization_id=org_id)
print(mps_ids)
authors = []
for question in data:
for author in question['authors']:
if author in mps_ids:
authors.append(author)
print(org_id, len(authors), float(len(mps_ids)))
try:
avg = len(authors) / float(len(mps_ids))
except ZeroDivisionError as error:
avg = 0
question_count = Counter(authors)
max_value = 0
max_persons = []
for maxi in question_count.most_common(90):
if max_value == 0:
max_value = maxi[1]
if maxi[1] == max_value:
max_persons.append(maxi[0])
else:
break
for person_id in mps_ids:
person = Person.objects.get(id_parladata=person_id)
saveOrAbortNew(model=NumberOfQuestions,
created_for=date_of,
person=person,
score=question_count[person_id],
average=avg,
maximum=max_value,
maxMPs=max_persons)
self.stdout.write('Done setting NumberOfQuestions to MP %s' %
str(person.id_parladata))
return 0
def setTfidfOfMP(commander, mp_id):
url = '%s/tfidf/person?id=%s' % (ISCI_URL, mp_id)
commander.stdout.write('About to fetch %s' % url)
r = requests.get(url)
commander.stdout.write('Saving speaker %s' % str(mp_id))
output = r.json()
date_of = datetime.now().date()
person = Person.objects.get(id_parladata=mp_id)
data = output.get('tfidf', [])
saveOrAbortNew(Tfidf,
person=person,
created_for=date_of,
is_visible=False,
data=data)
def setTfidfOfPG(commander, pg_id):
url = '%s/tfidf/party?id=%s' % (ISCI_URL, pg_id)
commander.stdout.write('About to fetch %s' % url)
r = requests.get(url)
commander.stdout.write('Saving PG %s' % str(pg_id))
output = r.json()
date_of = datetime.now().date()
organization = Organization.objects.get(id_parladata=pg_id)
data = output.get('tfidf', [])
saveOrAbortNew(Tfidf,
organization=organization,
created_for=date_of,
is_visible=False,
data=data)
def setTfidfOfSession(commander, session_id):
url = '%s/tfidf/session?id=%s' % (ISCI_URL, session_id)
commander.stdout.write('About to fetch %s' % url)
r = requests.get(url)
commander.stdout.write('Saving session %s' % str(session_id))
output = r.json()
date_of = datetime.now().date()
session = Session.objects.get(id_parladata=session_id)
data = output.get('tfidf', [])
saveOrAbortNew(Tfidf,
session=session,
created_for=date_of,
is_visible=False,
data=data)
def setPresenceOfPGs(self):
"""
set presence for all Partys
"""
table = self.presencePGs
averageSessions = table['sessions'].mean()
averageVotes = table['votes'].mean()
maxVotes = table['votes'].max()
maxVotesOrgIdx = table['votes'].idxmax()
maxSessions = table['sessions'].max()
maxSessionOrgIdx = table['sessions'].idxmax()
for pg in self.pgs:
try:
thisSessions = table[table.pg == pg].sessions[pg]
thisVotes = table[table.pg == pg].votes[pg]
except:
continue
thisOrg = Organization.objects.get(id_parladata=pg)
result = saveOrAbortNew(model=PercentOFAttendedSession,
created_for=self.date_of,
organization=thisOrg,
organization_value_sessions=thisSessions,
maxPG_sessions=[maxSessionOrgIdx],
average_sessions=averageSessions,
maximum_sessions=maxSessions,
organization_value_votes=thisVotes,
maxPG_votes=[maxVotesOrgIdx],
average_votes=averageVotes,
maximum_votes=maxVotes)
def setPercentOFAttendedSessionPG(pg_id, date_of, parenet_org):
date_ = date_of.strftime(API_DATE_FORMAT)
allSum = {}
data = {}
voters = getVotersPairsWithOrg(date_=date_of, organization_id=parenet_org)
membersOfPG = {}
for i, value in sorted(voters.iteritems()):
membersOfPG.setdefault(value, []).append(i)
data = getNumberOfAllMPAttendedSessions(date_of, getVotersIDs(date_=date_of, organization_id=parenet_org))
sessions = {pg: [] for pg in membersOfPG if membersOfPG[pg]}
votes = {pg: [] for pg in membersOfPG if membersOfPG[pg]}
for pg in membersOfPG:
if not membersOfPG[pg]:
continue
for member in membersOfPG[pg]:
if str(member) in data['sessions'].keys():
sessions[pg].append(data['sessions'][str(member)])
votes[pg].append(data['votes'][str(member)])
sessions[pg] = sum(sessions[pg])/len(sessions[pg])
votes[pg] = sum(votes[pg])/len(votes[pg])
thisMPSessions = sessions[pg_id]
maximumSessions = max(sessions.values())
maximumPGSessions = [pgId
for pgId
in sessions
if sessions[pgId] == maximumSessions][:5]
averageSessions = sum(data['sessions'].values()) / len(data['sessions'])
thisMPVotes = votes[pg_id]
maximumVotes = max(votes.values())
maximumPGVotes = [pgId
for pgId
in votes
if votes[pgId] == maximumVotes][:5]
averageVotes = sum(data['votes'].values()) / len(data['votes'])
org = Organization.objects.get(id_parladata=int(pg_id))
result = saveOrAbortNew(model=PercentOFAttendedSession,
created_for=date_of,
organization=org,
organization_value_sessions=thisMPSessions,
maxPG_sessions=maximumPGSessions,
average_sessions=averageSessions,
maximum_sessions=maximumSessions,
organization_value_votes=thisMPVotes,
maxPG_votes=maximumPGVotes,
average_votes=averageVotes,
maximum_votes=maximumVotes)
def setMPsOfPG(commander, pg_id, date_=None):
if date_:
date_of = datetime.strptime(date_, API_DATE_FORMAT).date()
commander.stdout.write('Setting for date %s' % str(date_of))
else:
date_of = datetime.now().date()
date_ = date_of.strftime(API_DATE_FORMAT)
commander.stdout.write('Setting for today (%s)' % str(date_of))
pairs = getVotersPairsWithOrg()
membersOfPG = {i: [] for i in set(pairs.values())}
for mem, org in pairs.items():
membersOfPG[org].append(mem)
org = Organization.objects.get(id_parladata=pg_id)
commander.stdout.write('Setting for organisation %s' % str(pg_id))
saveOrAbortNew(model=MPOfPg,
organization=org,
id_parladata=pg_id,
MPs=membersOfPG[pg_id],
created_for=date_of)
def setPresenceThroughTime(commander, person_id, date_=None):
if date_:
fdate = datetime.strptime(date_, '%d.%m.%Y').date()
else:
fdate = datetime.now().date()
person = Person.objects.filter(id_parladata=person_id)
if not person:
commander.stdout.write('Person with id %s doesnt exist' % str(pg))
return
data = getBallotsCounter(person[0], fdate)
data_for_save = []
commander.stdout.write('Iterating through months for person %s' %
str(person_id))
for month in data:
options = YES + NOT_PRESENT + AGAINST + ABSTAIN
stats = sum(
[month[option] for option in options if option in month.keys()])
not_member = month['total'] - stats
not_member = float(not_member) / month['total'] if not_member else 0
presence = float(stats - sum([
month[option] for option in NOT_PRESENT if option in month.keys()
])) / month['total'] if stats else 0
data_for_save.append({
'date_ts': month['date_ts'],
'presence': presence * 100,
'not_member': not_member * 100,
'vote_count': month['total']
})
saveOrAbortNew(model=PresenceThroughTime,
person=Person.objects.get(id_parladata=person_id),
created_for=fdate,
data=data_for_save)
commander.stdout.write('Set PresenceThroughTime for person id %s' %
str(person_id))
def setPresenceOfPG(commander, session_id):
""" Stores presence of PGs on specific session
"""
PGs = getOrganizationsWithVoters()
commander.stdout.write(
'About to get ballots for session')
votes = getBallotsForSession(session_id)
counters_in = Counter([vote['voterparty']
for vote in votes if vote['option'] not in NOT_PRESENT])
counters_out = Counter([vote['voterparty']
for vote in votes if vote['option'] in NOT_PRESENT])
pgs = list(set(counters_in.keys() + counters_out.keys()))
results = {}
for pg in pgs:
if not pg in PGs:
continue
try:
results[pg] = counters_in[pg] * 100 / \
(counters_in[pg] + counters_out[pg])
except:
if pg in counters_in.keys():
results[pg] = 100
elif pg in counters_out.keys():
results[pg] = 0
else:
commander.stderr.write(
'Something is wrong with PG %s' % str(pg))
session = Session.objects.get(id_parladata=session_id)
saveOrAbortNew(model=PresenceOfPG,
created_for=session.start_time,
presence=[results],
session=session)
commander.stdout.write(
'Successfully saved presence of PGs for session %s' % str(session_id))
def setMembershipsOfMember(commander, person_id, date_=None):
if date_:
date_of = datetime.strptime(date_, API_DATE_FORMAT)
else:
date_of = datetime.now().date()
organizations = {org['id']: org for org in getOrganizations()}
memberships = getMembershipsOfMember(person_id=person_id, date_=date_of)
data = defaultdict(list)
person = Person.objects.get(id_parladata=int(person_id))
for mem in memberships:
if mem['role'] != 'voter':
continue
organization = organizations[mem['organization']]
org_links = getLinks(organization=organization['id'])
if org_links:
org_link = org_links[0]['url']
else:
org_link = None
data[organization['classification']].append({
'url':
org_link,
'org_type':
organization['classification'],
'org_id':
organization['id'],
'name':
organization['_name']
})
saveOrAbortNew(MembershipsOfMember,
created_for=date_of,
person=person,
data=dict(data))
commander.stdout.write('Set MembershipsOfMember for person id %s' %
str(person_id))
def setPGPresenceThroughTime(commander, pg, date):
if date:
date_of = datetime.strptime(date, API_DATE_FORMAT).date()
else:
date_of = datetime.now().date()
org = Organization.objects.filter(id_parladata=pg)
if not org:
commander.stdout.write('Organization with id %s doesnt exist' %
str(pg))
return
else:
org = org[0]
data = getBallotsCounter(org, date_of)
data_for_save = []
for month in data:
options = YES + NOT_PRESENT + AGAINST + ABSTAIN
stats = sum(
[month[option] for option in options if option in month.keys()])
presence = float(stats - sum([
month[option] for option in NOT_PRESENT if option in month.keys()
])) / stats if stats else 0
data_for_save.append({
'date_ts': month['date_ts'],
'presence': presence * 100,
})
org = Organization.objects.get(id_parladata=pg)
commander.stdout.write('Saving presence for organization %s' % str(pg))
saveOrAbortNew(model=PresenceThroughTime,
organization=org,
created_for=date_of,
data=data_for_save)
return
def setPercentOFAttendedSession(commander, members, date_of=datetime.now().date()):
data = getNumberOfAllMPAttendedSessions(date_of, members)
for person_id in members:
if not data["votes"].values():
commander.stdout.write('Member with id: %s has not votes' % str(person_id))
return
if person_id in data["sessions"].keys():
thisMP = data["sessions"][person_id]
else:
# ta member se ni obstajal
commander.stdout.write('Member with id %s didn\'t exist' % str(person_id))
return
org_sessions_values = [value for key, value in data["sessions"].items() if int(key) in members]
maximum = max(org_sessions_values)
maximumMP = [pId for pId in data["sessions"]
if data["sessions"][pId] == maximum and int(pId) in members]
average = sum(org_sessions_values) / len(org_sessions_values)
if person_id in data["votes"].keys():
thisMPVotes = data["votes"][person_id]
else:
thisMPVotes = 0
org_votes_values = [value for key, value in data["votes"].items() if int(key) in members]
maximumVotes = max(org_votes_values)
maximumMPVotes = [pId for pId in data["votes"]
if data["votes"][pId] == maximumVotes and int(pId) in members]
averageVotes = sum(org_votes_values) / len(org_votes_values)
person = Person.objects.get(id_parladata=int(person_id))
result = saveOrAbortNew(model=Presence,
created_for=date_of,
person=person,
person_value_sessions=thisMP,
maxMP_sessions=maximumMP,
average_sessions=average,
maximum_sessions=maximum,
person_value_votes=thisMPVotes,
maxMP_votes=maximumMPVotes,
average_votes=averageVotes,
maximum_votes=maximumVotes)
commander.stdout.write('Set Presence for member with id %s' % str(person_id))
def setPresenceMPs(self):
"""
set presence for all Members
"""
votes = self.presenceMP_V
sessions = self.presenceMP_S
actualVotes = votes[votes.voter.isin(self.members)]
avgVote = actualVotes[['attendance']].mean().attendance
maxVote = actualVotes.max()
max_vote_value = actualVotes['attendance'].max()
max_vote_voters = actualVotes[actualVotes.attendance ==
max_vote_value]['voter'].tolist()
actualSession = sessions[sessions.voter.isin(self.members)]
avgSession = actualSession[['attendance']].mean().attendance
max_session_value = actualSession['attendance'].max()
max_ses_voters = actualSession[actualSession.attendance ==
max_session_value]['voter'].tolist()
for mp in self.members:
try:
person = Person.objects.get(id_parladata=int(mp))
thisVotes = votes[votes.voter == mp].reset_index().at[
0, 'attendance']
tempS = sessions[sessions.voter == mp].reset_index()
thisSession = tempS.at[0, 'attendance']
print mp
result = saveOrAbortNew(model=Presence,
created_for=self.date_of,
person=person,
person_value_sessions=thisSession,
maxMP_sessions=max_ses_voters,
average_sessions=avgSession,
maximum_sessions=max_session_value,
person_value_votes=thisVotes,
maxMP_votes=max_vote_voters,
average_votes=avgVote,
maximum_votes=max_vote_value)
except:
print mp, 'fail set presence'
def setEqualVoters(self):
"""
set cards how equal are voters
"""
for mp in self.members:
print '.:' + str(mp) + ':.'
person = Person.objects.get(id_parladata=int(mp))
try:
r = self.equalVotersData[mp]
except:
print mp, 'fail set equal voters'
continue
mps_data = r.reset_index().sort_values(mp, ascending=False)
# most equal
most_data = []
data = mps_data
for idx in range(len(data) - 1):
member_id = int(data.iloc[idx]['voter'])
if member_id in self.members:
member = Person.objects.get(id_parladata=member_id)
most_data.append({
'member': member,
'score': data.iloc[idx][mp]
})
most_data = list(reversed(most_data))
result = saveOrAbortNew(model=EqualVoters,
created_for=self.date_of,
person=person,
person1=most_data[0]['member'],
votes1=most_data[0]['score'],
person2=most_data[1]['member'],
votes2=most_data[1]['score'],
person3=most_data[2]['member'],
votes3=most_data[2]['score'],
person4=most_data[3]['member'],
votes4=most_data[3]['score'],
person5=most_data[4]['member'],
votes5=most_data[4]['score'])
# less equal
less_data = []
data = mps_data
#for idx in range(5):
idx = 0
while len(less_data) < 5:
member_id = int(data.iloc[idx]['voter'])
if member_id in self.members:
member = Person.objects.get(id_parladata=member_id)
less_data.append({
'member': member,
'score': data.iloc[idx][mp]
})
idx += 1
result = saveOrAbortNew(model=LessEqualVoters,
created_for=self.date_of,
person=person,
person1=less_data[0]['member'],
votes1=less_data[0]['score'],
person2=less_data[1]['member'],
votes2=less_data[1]['score'],
person3=less_data[2]['member'],
votes3=less_data[2]['score'],
person4=less_data[3]['member'],
votes4=less_data[3]['score'],
person5=less_data[4]['member'],
votes5=less_data[4]['score'])
def setMotionOfSession(commander, session_id):
"""Stores all motions with detiled data of specific sesison.
"""
if commander:
commander.stdout.write('Beginning setMotionOfSession for session %s' %
str(session_id))
commander.stdout.write('Getting for motion for session %s' %
(str(session_id)))
votes = parladata_api.getVotesForSession(session_id)
session = Session.objects.get(id_parladata=session_id)
laws = []
for vote in votes:
if commander:
commander.stdout.write('Handling vote %s' % str(vote['id']))
if not vote['name']:
commander.stdout.write('Skipping vote because it has no name: %s' %
str(vote['id']))
motion = parladata_api.getMotion(vote['motion'])
if vote['results']:
votes_for = vote['results']['for']
votes_against = vote['results']['against']
votes_abstain = vote['results']['abstain']
votes_absent = vote['results']['absent']
if vote['counter']:
# this is for votes without ballots
counter = json.loads(vote['counter'])
opts_set = set(counter.keys())
if opts_set.intersection(YES):
votes_for = counter['for']
if opts_set.intersection(AGAINST):
votes_against = counter['against']
if opts_set.intersection(ABSTAIN):
votes_abstain = counter['abstain']
# hardcoded croations number of member
votes_absent = 151 - sum([int(v) for v in counter.values()])
result = motion['result']
documents = [{
'url': link['url'],
'name': link['url']
} for link in motion['links']]
# TODO: replace try with: "if vote['epa']"
try:
law = Legislation.objects.get(epa=motion['epa'])
laws.append(law)
except:
law = None
classification = getMotionClassification(vote['name'])
vote_obj = Vote.objects.filter(id_parladata=vote['id'])
agendaItems = list(
AgendaItem.objects.filter(id_parladata__in=motion['agenda_item']))
if vote_obj:
if commander:
commander.stdout.write('Updating vote %s' % str(vote['id']))
commander.stdout.write('Updating data %s' % str(vote))
prev_result = vote_obj[0].result
vote_obj.update(
created_for=session.start_time,
start_time=vote['start_time'],
session=session,
motion=vote['name'],
tags=vote['tags'],
votes_for=votes_for,
against=votes_against,
abstain=votes_abstain,
not_present=votes_absent,
result=result,
id_parladata=vote['id'],
document_url=documents,
epa=motion['epa'],
law=law,
classification=classification,
)
vote_obj = vote_obj[0]
vote_obj.agenda_item.add(*agendaItems)
if prev_result != vote_obj.result:
if commander:
commander.stdout.write(
'Running finish_legislation_by_final_vote(vote[0])')
finish_legislation_by_final_vote(vote_obj)
else:
if commander:
commander.stdout.write('Saving new vote %s' % str(vote['id']))
result = saveOrAbortNew(
model=Vote,
created_for=session.start_time,
start_time=vote['start_time'],
session=session,
motion=vote['name'],
tags=vote['tags'],
votes_for=votes_for,
against=votes_against,
abstain=votes_abstain,
not_present=votes_absent,
result=result,
id_parladata=vote['id'],
document_url=documents,
epa=motion['epa'],
law=law,
classification=classification,
)
vote_obj = Vote.objects.get(id_parladata=vote['id'])
commander.stdout.write(
'Running finish_legislation_by_final_vote(vote[0])')
finish_legislation_by_final_vote(vote_obj)
owners = getOwnersOfAmendment(vote_obj)
if owners['orgs']:
a_orgs = list(
Organization.objects.filter(id_parladata__in=owners['orgs']))
else:
a_orgs = []
if owners['people']:
a_people = Person.objects.filter(id_parladata__in=owners['people'])
else:
a_people = []
if a_orgs:
vote_obj.amendment_of.clear()
for org in a_orgs:
AmendmentOfOrg(vote=vote_obj, organization=org).save()
vote_obj.amendment_of_person.add(*a_people)
# set motion analize
if commander:
commander.stdout.write('Running setMotionAnalize for %s' %
str(session_id))
setMotionAnalize(session_id)
# TODO figure out what to do with this
# if laws:
# recacheLegislationsOnSession(session_id)
if commander:
commander.stdout.write('Running deleteRendersOfSessionVotes for %s' %
str(session_id))
deleteRendersOfSession([session_id])
return 0
def handle(self, *args, **options):
if options['date']:
date_of = datetime.strptime(options['date'], API_DATE_FORMAT)
else:
date_of = datetime.now().date()
date_ = date_of.strftime(API_DATE_FORMAT)
for org_id in getParentOrganizationsWithVoters():
sw = WordAnalysis(organization_id=org_id,
count_of='members',
date_=date_)
#if not sw.isNewSpeech:
# return JsonResponse({'alliswell': False})
#Vocabolary size
all_score = sw.getVocabularySize()
max_score, maxMPid = sw.getMaxVocabularySize()
avg_score = sw.getAvgVocabularySize()
date_of = sw.getDate()
maxMP = Person.objects.get(id_parladata=maxMPid)
self.stdout.write('[INFO] saving vocabulary size')
for p in all_score:
self.stdout.write(
'[INFO] saving vocabulary size for person %s' %
str(p['counter_id']))
saveOrAbortNew(model=VocabularySize,
person=Person.objects.get(
id_parladata=int(p['counter_id'])),
created_for=date_of,
score=int(p['coef']),
maxMP=maxMP,
average=avg_score,
maximum=max_score)
#Unique words
all_score = sw.getUniqueWords()
max_score, maxMPid = sw.getMaxUniqueWords()
avg_score = sw.getAvgUniqueWords()
date_of = sw.getDate()
maxMP = Person.objects.get(id_parladata=maxMPid)
self.stdout.write('[INFO] saving unique words')
for p in all_score:
self.stdout.write('[INFO] saving unique words for person %s' %
str(p['counter_id']))
saveOrAbortNew(model=VocabularySizeUniqueWords,
person=Person.objects.get(
id_parladata=int(p['counter_id'])),
created_for=date_of,
score=int(p['unique']),
maxMP=maxMP,
average=avg_score,
maximum=max_score)
#Spoken words
all_words = sw.getSpokenWords()
max_words, maxWordsMPid = sw.getMaxSpokenWords()
avgSpokenWords = sw.getAvgSpokenWords()
date_of = sw.getDate()
maxMP = Person.objects.get(id_parladata=maxWordsMPid)
self.stdout.write('[INFO] saving spoken words')
for p in all_words:
self.stdout.write('[INFO] saving spoken words for person %s' %
str(p['counter_id']))
saveOrAbortNew(model=SpokenWords,
created_for=date_of,
person=Person.objects.get(
id_parladata=int(p['counter_id'])),
score=int(p['wordcount']),
maxMP=maxMP,
average=avgSpokenWords,
maximum=max_words)
self.stdout.write('[INFO] All MPs updated')
return 0
def set_mismatch_of_pg(request, by_organization, date_=''):
print 'Preparing date'
if date_:
f_date = datetime.strptime(date_, '%d.%m.%Y')
else:
f_date = datetime.now()
#url = settings.API_URL_V2 + '/getVotesTableExtended/'+ str(by_organization) + '/' + date_
data = pd.DataFrame()
for page in getBallotTable(organization=by_organization):
temp = pd.DataFrame(page)
data = data.append(temp, ignore_index=True)
print 'Preparing pandas DataFrame'
data['option_absent'] = 0
data['option_for'] = 0
data['option_against'] = 0
data['option_abstain'] = 0
data.loc[data['option'] == 'absent', 'option_absent'] = 1
data.loc[data['option'] == 'for', 'option_for'] = 1
data.loc[data['option'] == 'against', 'option_against'] = 1
data.loc[data['option'] == 'abstain', 'option_abstain'] = 1
data['voter_unit'] = 1
print 'Prepared pandas DataFrame, about to start analyzing things.'
#for against absent abstain
# all_votes = data.groupby('vote').sum()
m_to_p = getVotersPairsWithOrg(organization_id=by_organization)
mppgs = pd.DataFrame(m_to_p.items(), columns=['voter', 'voterparty'])
#Get ballots of last members party
keys = ['voter', 'voterparty']
i1 = mppgs.set_index(keys).index
i2 = data.set_index(keys).index
data2 = data[i2.isin(i1)]
def getPartyBallot(row):
"""
using for set ballot of party:
methodology: ignore not_present
"""
stats = {'for': row['option_for'],
'against': row['option_against'],
'abstain': row['option_abstain'],
'absent': row['option_absent']}
if max(stats.values()) == 0:
return None
max_ids = [key for key, val in stats.iteritems() if val == max(stats.values())]
#if len(max_ids) > 1:
# return None
#return max_ids[0]
return ','.join(max_ids)
partyBallots = data.groupby(['vote',
'voterparty']).sum().apply(lambda row: getPartyBallot(row), axis=1)
partys = partyBallots.reset_index()
partys = partys.rename(columns = {0:'partyoption'})
result = pd.merge(data2, partys, on=['vote', 'voterparty'])
#remove absent option TURNED OFF
# result = result[result.option != 'absent']
members_vote_count = result[['voter','voter_unit']].groupby('voter').count()
def is_equal(row):
if row['partyoption'] and row['option'] in row['partyoption']:
return 1
else:
return 2
equal_indexes = result[['option', 'partyoption']].apply(lambda x: is_equal(x), axis=1).reindex()
result['ivan'] = result[['option', 'partyoption']].apply(lambda x: is_equal(x), axis=1).reindex()
out = pd.concat([result, equal_indexes], axis=1).rename(columns = {0: 'equal_vote'})
out = out[out['equal_vote']==1]
members_equal_count = out[['voter', 'equal_vote']].groupby('voter').sum()
final = pd.concat([members_equal_count, members_vote_count], axis=1)
final['percent'] = final.apply(lambda x: float(x['equal_vote'])/x['voter_unit']*100.0, axis=1)
print 'saving'
data = []
for member, row in final.iterrows():
print member
value = 100 - row['percent']
person = Person.objects.get(id_parladata=member)
party = person.static_data.latest('created_at').party
party_classification = party.classification
# TODO: make fix code under for people without party
#if party_classification != settings.PS:
# value = None
data.append({'person': person,
'value': value})
maxMismatch = max(data, key=lambda x:x['value'] if x['value'] else 0)
values = [i['value'] for i in data if i['value']]
avg = float(sum(values))/len(values)
for d in data:
saveOrAbortNew(model=MismatchOfPG,
person=d['person'],
created_for=f_date,
average=avg,
maximum=maxMismatch['value'],
maxMP=maxMismatch['person'],
data=d['value'])
return JsonResponse({'alliswell': True})
def handle(self, *args, **options):
date_of = datetime.now().date()
date_ = date_of.strftime(API_DATE_FORMAT)
for org in getParentOrganizationsWithVoters():
self.stdout.write('getting organizations with voters')
organization_ids = getOrganizationsWithVoters(date_=date_of)
scores = {}
for organization_id in organization_ids:
self.stdout.write('Org id: %s' % str(organization_id))
# get word counts with solr
counter = Counter(
getCountList(self, int(organization_id), date_))
total = sum(counter.values())
scores_local = getScores(
[problematicno, privzdignjeno, preprosto], counter, total)
self.stdout.write('Outputting scores_local: %s' %
str(scores_local))
scores[organization_id] = scores_local
self.stdout.write('Outputting scores: %s' % str(scores))
average = {
"problematicno":
sum([score['problematicno']
for score in scores.values()]) / len(scores),
"privzdignjeno":
sum([score['privzdignjeno']
for score in scores.values()]) / len(scores),
"preprosto":
sum([score['preprosto']
for score in scores.values()]) / len(scores)
}
data = []
for org_id, score in scores.items():
data.append({
'org': org_id,
'problematicno': score['problematicno'],
'privzdignjeno': score['privzdignjeno'],
'preprosto': score['preprosto'],
'problematicno_average': average['problematicno'],
'privzdignjeno_average': average['privzdignjeno'],
'preprosto_average': average['preprosto']
})
for score in data:
self.stdout.write('About to save %s' % str(score))
status = saveOrAbortNew(
StyleScores,
organization=Organization.objects.get(
id_parladata=int(score['org'])),
created_for=date_of,
problematicno=float(score['problematicno']),
privzdignjeno=float(score['privzdignjeno']),
preprosto=float(score['preprosto']),
problematicno_average=float(
score['problematicno_average']),
privzdignjeno_average=float(
score['privzdignjeno_average']),
preprosto_average=float(score['preprosto_average']))
self.stdout.write('SaveOrAbort status: %s' % str(status))
return 0
def setBasicInfoOfPG(commander, pg, date):
if date:
date_of = datetime.strptime(date, API_DATE_FORMAT).date()
else:
date_of = datetime.now().date()
org_data = getOrganizations(pg)
contacts = getContactDetails(organization=pg)
president = getPosts(date_of, organization=pg, role="president")
deputy = getPosts(date_of, organization=pg, role="deputy")
email = ''
for contact in contacts:
if contact['contact_type'] == 'EMAIL':
email = contact['value']
facebook = [
link['url'] for link in getLinks(organization=pg, tags__name='fb')
]
twitter = [
link['url'] for link in getLinks(organization=pg, tags__name='tw')
]
numberOfSeats = dict(Counter(
getVotersPairsWithOrg(date_=date_of).values()))[int(pg)]
headOfPG = None
viceOfPG = []
if president:
commander.stdout.write('Defiing head of PG')
headOfPG = Person.objects.get(id_parladata=int(president[0]['person']))
else:
commander.stdout.write('No head of PG')
headOfPG = None
if deputy:
for vice in deputy:
if vice:
viceOfPG.append(vice['person'])
else:
viceOfPG.append(None)
else:
viceOfPG.append(None)
gov_sides = getCoalitionPGs(date_of,
parent_org=session.organization.id_parladata)
is_coalition = False
for gov_side, orgs in gov_sides.items():
if Organization.objects.get(id_parladata=gov_side).is_coalition:
is_coalition = pg.id_parladata in orgs
org = Organization.objects.get(id_parladata=int(pg))
commander.stdout.write('Saving organization %s' % str(pg))
saveOrAbortNew(model=PGStatic,
created_for=date_of,
organization=org,
headOfPG=headOfPG,
viceOfPG=viceOfPG,
numberOfSeats=numberOfSeats,
allVoters=org_data['voters'],
facebook=json.dumps(facebook) if facebook else None,
twitter=json.dumps(twitter) if twitter else None,
email=email,
is_coalition=is_coalition)
return
def handle(self, *args, **options):
if options['date']:
date_ = options['date']
date_of = datetime.strptime(date_, API_DATE_FORMAT).date()
else:
date_ = datetime.now().date().strftime(API_DATE_FORMAT)
date_of = datetime.now().date()
data = getQuestions()
for org_id in getParentOrganizationsWithVoters():
pgs_on_date = getOrganizationsWithVoters(date_=date_of,
organization_id=org_id)
mps = getVotersIDs(date_=date_of, organization_id=org_id)
mpStatic = {}
for mp in mps:
mpStatic[str(mp)] = getPersonData(str(mp), date_)
pg_ids = pgs_on_date
authors = []
for question in data:
qDate = datetime.strptime(question['date'], '%Y-%m-%dT%X')
qDate = qDate.strftime(API_DATE_FORMAT)
for author in question['authors']:
try:
person_data = mpStatic[str(author)]
except KeyError as e:
print(str(question['authors']))
person_data = getPersonData(str(author), date_)
mpStatic[str(author)] = person_data
if person_data and person_data['party'] and person_data[
'party']['id']:
authors.append(person_data['party']['id'])
else:
print 'person nima mpstatic: ', author
avg = len(authors) / float(len(pg_ids))
question_count = Counter(authors)
max_value = 0
max_orgs = []
for maxi in question_count.most_common(90):
if max_value == 0:
max_value = maxi[1]
if maxi[1] == max_value:
max_orgs.append(maxi[0])
else:
break
is_saved = []
for pg_id in pg_ids:
org = Organization.objects.get(id_parladata=pg_id)
is_saved.append(
saveOrAbortNew(model=NumberOfQuestions,
created_for=date_of,
organization=org,
score=question_count[pg_id],
average=avg,
maximum=max_value,
maxOrgs=max_orgs))
return 0
def handle(self, *args, **options):
if options['date']:
date_ = options['date']
date_of = datetime.strptime(API_DATE_FORMAT).date()
else:
date_of = datetime.now().date()
date_ = date_of.strftime(API_DATE_FORMAT)
self.stdout.write('Getting voters')
for org in getParentOrganizationsWithVoters():
self.stdout.write('Starting style score for organization %s' % (org))
mps = getVotersIDs(organization_id=org, date_=date_of)
scores = {}
for person_id in mps:
self.stdout.write('MP id: %s' % str(person_id))
# get word counts with solr
counter = Counter(getCountList(self, int(person_id), date_))
total = sum(counter.values())
scores_local = getScores([problematicno, privzdignjeno, preprosto],
counter,
total)
self.stdout.write('Outputting scores_local: %s' %
str(scores_local))
scores[person_id] = scores_local
self.stdout.write('Outputting scores: %s' % str(scores))
average = {"problematicno": sum([score['problematicno']
for score
in scores.values()])/len(scores),
"privzdignjeno": sum([score['privzdignjeno']
for score
in scores.values()])/len(scores),
"preprosto": sum([score['preprosto']
for score
in scores.values()])/len(scores)}
data = []
for person, score in scores.items():
data.append({'member': person,
'problematicno': score['problematicno'],
'privzdignjeno': score['privzdignjeno'],
'preprosto': score['preprosto'],
'problematicno_average': average['problematicno'],
'privzdignjeno_average': average['privzdignjeno'],
'preprosto_average': average['preprosto']})
for score in data:
self.stdout.write('About to save %s' % str(score))
status = saveOrAbortNew(StyleScores,
person=Person.objects.get(
id_parladata=int(score["member"])),
created_for=date_of,
problematicno=float(
score['problematicno']),
privzdignjeno=float(
score['privzdignjeno']),
preprosto=float(score['preprosto']),
problematicno_average=float(
score['problematicno_average']),
privzdignjeno_average=float(
score['privzdignjeno_average']),
preprosto_average=float(
score['preprosto_average'])
)
self.stdout.write('SaveOrAbort status: %s' % str(status))
return 0
def setEqualVotesPG(self):
"""
set cards how equal are voters to Partys
"""
results = self.equalVotersPGData
for pg in results.keys():
org = Organization.objects.get(id_parladata=int(pg))
r = results[pg]
r = r.reset_index().sort_values(pg, ascending=False)
print(self.memsOfPGs)
if int(pg) not in self.memsOfPGs.keys():
"if this PG hasn't members on this time"
continue
# TODO notify someone if this fails
thisPG = r[r.voter.isin(self.memsOfPGs[int(pg)])]
otherMembers = [
m for pg_key in self.memsOfPGs.keys() if pg_key != str(pg)
for m in self.memsOfPGs[pg_key]
]
print 'pg', pg, len(self.memsOfPGs[pg]), len(otherMembers)
otherMems = r[r.voter.isin(otherMembers)]
# most equal
most_data = []
data = otherMems
# for idx in range(5):
for idx in range(len(data) - 1):
member_id = int(data.iloc[idx]['voter'])
if member_id in self.members:
member = Person.objects.get(id_parladata=member_id)
most_data.append({
'member': member,
'score': data.iloc[idx][pg]
})
most_data = list(reversed(most_data))
result = saveOrAbortNew(model=MostMatchingThem,
created_for=self.date_of,
organization=org,
person1=most_data[0]['member'],
votes1=most_data[0]['score'],
person2=most_data[1]['member'],
votes2=most_data[1]['score'],
person3=most_data[2]['member'],
votes3=most_data[2]['score'],
person4=most_data[3]['member'],
votes4=most_data[3]['score'],
person5=most_data[4]['member'],
votes5=most_data[4]['score'])
# less equal
less_data = []
data = otherMems
#for idx in range(5):
idx = 0
while len(less_data) < 5:
member_id = int(data.iloc[idx]['voter'])
if member_id in self.members:
member = Person.objects.get(id_parladata=member_id)
less_data.append({
'member': member,
'score': data.iloc[idx][pg]
})
idx += 1
result = saveOrAbortNew(model=LessMatchingThem,
created_for=self.date_of,
organization=org,
person1=less_data[0]['member'],
votes1=less_data[0]['score'],
person2=less_data[1]['member'],
votes2=less_data[1]['score'],
person3=less_data[2]['member'],
votes3=less_data[2]['score'],
person4=less_data[3]['member'],
votes4=less_data[3]['score'],
person5=less_data[4]['member'],
votes5=less_data[4]['score'])
# deviation
print thisPG
dev_data = []
for idx in range(thisPG.count()['voter']):
dev_data.append({
'id': thisPG.iloc[idx]['voter'],
'ratio': thisPG.iloc[idx][pg]
})
result = saveOrAbortNew(model=DeviationInOrganization,
created_for=self.date_of,
organization=org,
data=dev_data)
def setMPStaticPL(commander, person_id, date_=None):
if date_:
date_of = date_
else:
date_of = datetime.now()
commander.stdout.write('Fetching data from %s/persons/%s with day %s' %
(API_URL, str(person_id), date_of))
data = getPeople(id_=person_id)
try:
org_id = getVotersPairsWithOrg(date_=date_of)[int(person_id)]
except Exception as e:
commander.stdout.write(
'Person with ID %s has not correctly configured voter membership or he\'s not a MP'
% str(person_id))
commander.stdout.write(str(e))
return
organization = Organization.objects.get(id_parladata=org_id)
person = Person.objects.get(id_parladata=int(person_id))
socials = {'fb': 'facebook', 'tw': 'twitter', 'linkedin': 'linkedin'}
social_objs = {}
for key, name in socials.items():
social_objs[name] = None
for resp_data in getLinks(person=person_id, tags__name=key):
if resp_data:
social_objs[name] = resp_data['url']
if not data:
commander.stderr.write('Didn\'t get data.')
raise CommandError('No data returned.')
if 'error' in data.keys():
commander.stderr.write('[API ERROR] %s' % data['error'])
result = saveOrAbortNew(
model=MPStaticPL,
created_for=date_of,
person=person,
voters=data['voters'],
points=data['points'],
age=num_years(dateparse.parse_datetime(data['birth_date']))
if data['birth_date'] else None,
birth_date=dateparse.parse_datetime(data['birth_date'])
if data['birth_date'] else None,
mandates=data['mandates'],
party=organization,
education=data['education'],
education_level=data['education_level'],
previous_occupation=data['previous_occupation'],
name=data['name'],
district=data['districts'],
facebook=social_objs['facebook'],
twitter=social_objs['twitter'],
linkedin=social_objs['linkedin'],
party_name=organization.name,
acronym=organization.acronym,
gov_id=data['gov_id'],
gender='m' if data['gender'] == 'male' else 'f',
working_bodies_functions=[])
commander.stdout.write('Set MP with id %s' % str(person_id))