def prepareData(self):
if self.debug:
self.data = pd.read_pickle('backup_baze.pkl')
else:
self.data = pd.DataFrame()
for page in getBallotTable(organization=self.organization_id):
temp = pd.DataFrame(page)
self.data = self.data.append(temp, ignore_index=True)
self.members = getVotersIDs(organization_id=self.organization_id,
date_=self.date_of)
self.memsOfPGs = getOrganizationsWithVotersList(
organization_id=self.organization_id, date_=self.date_of)
self.pgs = getOrganizationsWithVoters(
organization_id=self.organization_id, date_=self.date_of)
def toLogic(row):
"""
voter option to logic value
"""
return VOTE_MAP[row['option']]
# prepere helper columns (logic, option_ni, voter_unit)
self.data['logic'] = self.data.apply(lambda row: toLogic(row), axis=1)
self.data['option_x'] = 0
self.data.loc[self.data['option'] == 'absent', 'option_x'] = 1
self.data['voter_unit'] = 1
def __init__(self, organization_id, count_of='members', date_=None):
self.date_ = ''
self.api_url = None
self.date_of = None
self.members = None
self.membersOfPGsRanges = None
self.membersOfPGs = None
self.count_of = 'members'
self.text = {}
self.unique_words = []
self.all_words = []
self.swizec_coef = []
self.average_words = None
if date_:
self.date_of = datetime.strptime(date_, API_DATE_FORMAT).date()
self.date_ = date_
else:
self.date_of = datetime.now().date()
self.date_ = ''
self.count_of = count_of
# get members for this day
self.members = getVotersIDs(organization_id=organization_id,
date_=self.date_of)
self.organizations = getOrganizationsWithVoters(
organization_id=organization_id, date_=self.date_of)
self.prepereSpeeches()
self.wordCounter()
def getAllStaticData(request, force_render=False):
sc = SlackClient(slack_token)
date_of = datetime.now().date()
date_ = date_of.strftime(API_DATE_FORMAT)
c_data = cache.get('all_statics')
if c_data and not force_render:
out = c_data
else:
sc.api_call("chat.postMessage",
channel="#parlalize_notif",
text='StaticDataDebug start: ' + str(c_data)[:100] + ' ' +
str(force_render))
date_ = datetime.now().strftime(API_DATE_FORMAT)
print("asdasdasd")
out = {
'persons': {},
'partys': {},
'wbs': {},
'sessions': {},
'ministrs': {}
}
for person in Person.objects.all():
personData = getPersonData(person.id_parladata, date_)
out['persons'][person.id_parladata] = personData
print("asdasdasd")
parliamentary_group = Organization.objects.filter(
id_parladata__in=getOrganizationsWithVoters())
for party in parliamentary_group:
out['partys'][party.id_parladata] = party.getOrganizationData()
sessions = Session.objects.all()
for session in sessions:
out['sessions'][session.id_parladata] = session.getSessionData()
ministrs = MinisterStatic.objects.all()
for ministr in ministrs:
out['ministrs'][ministr.id] = ministr.getJsonData()
orgs = Organization.objects.filter(classification__in=settings.WBS)
out['wbs'] = [{
'id': org.id_parladata,
'name': org.name
} for org in orgs]
cache.set('all_statics', out, ALL_STATIC_CACHE_AGE)
sc.api_call("chat.postMessage",
channel="#parlalize_notif",
text='StaticDataDebug end: ' + str(out)[:100] + ' ' +
str(force_render))
return JsonResponse(out)
def handle(self, *args, **options):
if options['pgs']:
pgs = options['pgs']
else:
if options['date']:
date_ = datetime.strptime(options['date'], API_DATE_FORMAT)
else:
date_ = datetime.now()
pgs = getOrganizationsWithVoters(date_=date_)
for pg in pgs:
setPGMismatch(self, pg, options['date'])
return 0
def handle(self, *args, **options):
if options['pgs']:
pgs = options['pgs']
else:
if options['date']:
date_ = options['date']
date_of = datetime.strptime(date_, API_DATE_FORMAT)
else:
date_of = datetime.now()
pgs = getOrganizationsWithVoters(date_=date_of)
for pg in pgs:
setBasicInfoOfPG(self, pg, options['date'])
return 0
def handle(self, *args, **options):
date_of = datetime.now().date()
date_ = date_of.strftime(API_DATE_FORMAT)
organization_ids = []
if options['organization_ids']:
organization_ids = options['organization_ids']
else:
self.stdout.write('getting organizations with voters')
organization_ids = getOrganizationsWithVoters(date_=date_of)
for organization_id in organization_ids:
setTfidfOfPG(self, organization_id)
return 0
def handle(self, *args, **options):
date_of = datetime.now().date()
date_ = date_of.strftime(API_DATE_FORMAT)
pgs = []
if options['pgs']:
pgs = options['pgs']
else:
pgs = getOrganizationsWithVoters(date_=date_of)
for pg in pgs:
self.stdout.write('About to set MPS of %s' % str(pg))
setMPsOfPG(self, pg)
return 0
def handle(self, *args, **options):
pg_ids = []
if options['pg_ids']:
pg_ids = options['pg_ids']
else:
date_of = datetime.now().date()
date_ = date_of.strftime(API_DATE_FORMAT)
pg_ids = getOrganizationsWithVoters(date_=date_of)
# get static data
self.stdout.write('Getting all static data')
static_data = json.loads(getAllStaticData(None).content)
for pg_id in pg_ids:
self.stdout.write('About to begin with PG %s' % str(pg_id))
pg = Organization.objects.filter(id_parladata=pg_id)
if not pg:
self.stdout.write('Organization with id %s does not exist' %
str(pg_id))
continue
else:
pg = pg[0]
output = [{
'term':
'VIII',
'type':
'pgmegastring',
'id':
'pgms_' + str(pg.id_parladata),
'party_id':
pg.id_parladata,
'party_json':
json.dumps(static_data['partys'][str(pg.id_parladata)]),
'content':
getOrgMegastring(pg),
}]
commit_to_solr(self, output)
return 0
def setPresenceOfPG(commander, session_id):
""" Stores presence of PGs on specific session
"""
PGs = getOrganizationsWithVoters()
commander.stdout.write(
'About to get ballots for session')
votes = getBallotsForSession(session_id)
counters_in = Counter([vote['voterparty']
for vote in votes if vote['option'] not in NOT_PRESENT])
counters_out = Counter([vote['voterparty']
for vote in votes if vote['option'] in NOT_PRESENT])
pgs = list(set(counters_in.keys() + counters_out.keys()))
results = {}
for pg in pgs:
if not pg in PGs:
continue
try:
results[pg] = counters_in[pg] * 100 / \
(counters_in[pg] + counters_out[pg])
except:
if pg in counters_in.keys():
results[pg] = 100
elif pg in counters_out.keys():
results[pg] = 0
else:
commander.stderr.write(
'Something is wrong with PG %s' % str(pg))
session = Session.objects.get(id_parladata=session_id)
saveOrAbortNew(model=PresenceOfPG,
created_for=session.start_time,
presence=[results],
session=session)
commander.stdout.write(
'Successfully saved presence of PGs for session %s' % str(session_id))
def getOwnersOfAmendment(vote):
orgs_ids = []
people_ids = []
if settings.COUNTRY == 'SI':
if 'Amandma' in vote.motion:
acronyms = re.findall('\; \s*(\w+)|\[\s*(\w+)', vote.motion)
acronyms = [
pg[0] + ',' if pg[0] else pg[1] + ',' for pg in acronyms
]
if acronyms:
query = reduce(operator.or_, (Q(name_parser__icontains=item)
for item in acronyms))
orgs = Organization.objects.filter(query)
s_time = vote.start_time
#orgs = orgs.filter(Q(founding_date__lte=s_time) |
# Q(founding_date=None),
# Q(dissolution_date__gte=s_time) |
# Q(dissolution_date=None))
org_ids = list(orgs.values_list('id_parladata', flat=True))
else:
org_ids = []
else:
org_ids = []
return {'orgs': org_ids, 'people': []}
elif settings.COUNTRY == 'HR':
amendment_words = ['AMANDMANI', 'AMANDMAN']
links = vote.document_url if vote.document_url else []
org_ids = parladata_api.getOrganizationsWithVoters()
orgs = Organization.objects.filter(id_parladata__in=org_ids)
acronyms = {}
for org in orgs:
acronyms[' '.join(org.acronym.split(', '))] = org.id
vlada_id = Organization.objects.get(name='Vlada').id
acronyms['Vlada VladaRH'] = vlada_id
for link in links:
tokens = link.name.replace(" ", '_').replace("-", '_').split('_')
print(link.name)
if 'AMANDMAN' in link.name:
for acronym, i in acronyms.items():
# find orgs
for splited_acr in acronym.split(' '):
if splited_acr in link.name:
orgs_ids.append(i)
break
num_ids = [hasNumbersOrPdfOrEndWord(token) for token in tokens]
if True in num_ids:
tokens = tokens[:num_ids.index(True)]
has_amendment = [token in amendment_words for token in tokens]
if True in has_amendment:
tokens = tokens[has_amendment.index(True) + 1:]
# find proposers
#if tokens[0].lower() == 'vlada' or tokens[0].lower() == 'vladarh':
# vlada
elif tokens[0].lower() == 'klub':
tokens = tokens[1:]
n_tokens = len(tokens)
for i in range(n_tokens):
d_tokens = [[tokens[i]]]
if i + 1 < n_tokens:
d_tokens.append([tokens[i], tokens[i + 1]])
for d_token in d_tokens:
n_tokens = len(tokens)
for i in range(n_tokens):
d_tokens = [[tokens[i]]]
if i + 1 < n_tokens:
d_tokens.append([tokens[i], tokens[i + 1]])
for d_token in d_tokens:
person = Person.objects.filter(
name_parser__icontains=' '.join(d_token))
if person.count() == 1:
people_ids.append(person[0].id)
break
if person.count() > 0:
names = person.values('id', 'name_parser')
for name in names:
if re.search(
"\\b" + ' '.join(d_token) +
"\\b", name['name_parser']):
people_ids.append(name['id'])
break
print acronyms
return {'orgs': orgs_ids, 'people': list(set(people_ids))}
def handle(self, *args, **options):
date_of = datetime.now().date()
date_ = date_of.strftime(API_DATE_FORMAT)
for org in getParentOrganizationsWithVoters():
self.stdout.write('getting organizations with voters')
organization_ids = getOrganizationsWithVoters(date_=date_of)
scores = {}
for organization_id in organization_ids:
self.stdout.write('Org id: %s' % str(organization_id))
# get word counts with solr
counter = Counter(
getCountList(self, int(organization_id), date_))
total = sum(counter.values())
scores_local = getScores(
[problematicno, privzdignjeno, preprosto], counter, total)
self.stdout.write('Outputting scores_local: %s' %
str(scores_local))
scores[organization_id] = scores_local
self.stdout.write('Outputting scores: %s' % str(scores))
average = {
"problematicno":
sum([score['problematicno']
for score in scores.values()]) / len(scores),
"privzdignjeno":
sum([score['privzdignjeno']
for score in scores.values()]) / len(scores),
"preprosto":
sum([score['preprosto']
for score in scores.values()]) / len(scores)
}
data = []
for org_id, score in scores.items():
data.append({
'org': org_id,
'problematicno': score['problematicno'],
'privzdignjeno': score['privzdignjeno'],
'preprosto': score['preprosto'],
'problematicno_average': average['problematicno'],
'privzdignjeno_average': average['privzdignjeno'],
'preprosto_average': average['preprosto']
})
for score in data:
self.stdout.write('About to save %s' % str(score))
status = saveOrAbortNew(
StyleScores,
organization=Organization.objects.get(
id_parladata=int(score['org'])),
created_for=date_of,
problematicno=float(score['problematicno']),
privzdignjeno=float(score['privzdignjeno']),
preprosto=float(score['preprosto']),
problematicno_average=float(
score['problematicno_average']),
privzdignjeno_average=float(
score['privzdignjeno_average']),
preprosto_average=float(score['preprosto_average']))
self.stdout.write('SaveOrAbort status: %s' % str(status))
return 0
def handle(self, *args, **options):
if options['date']:
date_ = options['date']
date_of = datetime.strptime(date_, API_DATE_FORMAT).date()
else:
date_ = datetime.now().date().strftime(API_DATE_FORMAT)
date_of = datetime.now().date()
data = getQuestions()
for org_id in getParentOrganizationsWithVoters():
pgs_on_date = getOrganizationsWithVoters(date_=date_of,
organization_id=org_id)
mps = getVotersIDs(date_=date_of, organization_id=org_id)
mpStatic = {}
for mp in mps:
mpStatic[str(mp)] = getPersonData(str(mp), date_)
pg_ids = pgs_on_date
authors = []
for question in data:
qDate = datetime.strptime(question['date'], '%Y-%m-%dT%X')
qDate = qDate.strftime(API_DATE_FORMAT)
for author in question['authors']:
try:
person_data = mpStatic[str(author)]
except KeyError as e:
print(str(question['authors']))
person_data = getPersonData(str(author), date_)
mpStatic[str(author)] = person_data
if person_data and person_data['party'] and person_data[
'party']['id']:
authors.append(person_data['party']['id'])
else:
print 'person nima mpstatic: ', author
avg = len(authors) / float(len(pg_ids))
question_count = Counter(authors)
max_value = 0
max_orgs = []
for maxi in question_count.most_common(90):
if max_value == 0:
max_value = maxi[1]
if maxi[1] == max_value:
max_orgs.append(maxi[0])
else:
break
is_saved = []
for pg_id in pg_ids:
org = Organization.objects.get(id_parladata=pg_id)
is_saved.append(
saveOrAbortNew(model=NumberOfQuestions,
created_for=date_of,
organization=org,
score=question_count[pg_id],
average=avg,
maximum=max_value,
maxOrgs=max_orgs))
return 0
def setMotionAnalize(session_id):
"""
request argument is here just because runner put 2 arguments in setter
setMotionAnalyze
setIntraDisunion
"""
session = get_object_or_404(Session, id_parladata=session_id)
data = pd.DataFrame()
for page in getBallotTable(session=session_id):
temp = pd.DataFrame(page)
data = data.append(temp, ignore_index=True)
if data.empty:
return
gov_sides = getCoalitionPGs(session.start_time, parent_org=session.organization.id_parladata)
paries_ids = getOrganizationsWithVoters(organization_id=session.organization.id_parladata)
for key, value in gov_sides.items():
cur = Organization.objects.get(id_parladata=key)
if cur.classification == 'coalition':
coalition = value
coalition_org = cur
else:
opozition_org = cur
# if coalition exist
calc_sides = bool(len(set(coalition+paries_ids)) != len(coalition+paries_ids))
data['option_absent'] = 0
data['option_for'] = 0
data['option_against'] = 0
data['option_abstain'] = 0
data.loc[data['option'] == 'absent', 'option_absent'] = 1
data.loc[data['option'] == 'for', 'option_for'] = 1
data.loc[data['option'] == 'against', 'option_against'] = 1
data.loc[data['option'] == 'abstain', 'option_abstain'] = 1
data['voter_unit'] = 1
data['is_coalition'] = 0
data.loc[data['voterparty'].isin(coalition), 'is_coalition'] = 1
#for against ni abstain
all_votes = data.groupby('vote').sum()
all_votes['max_option_percent'] = all_votes.apply(lambda row: getPercent(row['option_for'], row['option_against'], row['option_abstain'], row['option_absent']), axis=1)
m_against = data[data.option_against == 1].groupby(['vote']).apply(lambda x: x["voter"])
m_for = data[data.option_for == 1].groupby(['vote']).apply(lambda x: x["voter"])
m_absent = data[data.option_absent == 1].groupby(['vote']).apply(lambda x: x["voter"])
m_abstain = data[data.option_abstain == 1].groupby(['vote']).apply(lambda x: x["voter"])
pg_against = data[data.option_against == 1].groupby(['vote']).apply(lambda x: x["voterparty"])
pg_for = data[data.option_for == 1].groupby(['vote']).apply(lambda x: x["voterparty"])
pg_absent = data[data.option_absent == 1].groupby(['vote']).apply(lambda x: x["voterparty"])
pg_abstain = data[data.option_abstain == 1].groupby(['vote']).apply(lambda x: x["voterparty"])
all_votes['m_against'] = all_votes.apply(lambda row: getMPsList(row, m_against), axis=1)
all_votes['m_for'] = all_votes.apply(lambda row: getMPsList(row, m_for), axis=1)
all_votes['m_absent'] = all_votes.apply(lambda row: getMPsList(row, m_absent), axis=1)
all_votes['m_abstain'] = all_votes.apply(lambda row: getMPsList(row, m_abstain), axis=1)
all_votes['pg_against'] = all_votes.apply(lambda row: getPGsList(row, pg_against), axis=1)
all_votes['pg_for'] = all_votes.apply(lambda row: getPGsList(row, pg_for), axis=1)
all_votes['pg_absent'] = all_votes.apply(lambda row: getPGsList(row, pg_absent), axis=1)
all_votes['pg_abstain'] = all_votes.apply(lambda row: getPGsList(row, pg_abstain), axis=1)
try:
all_votes['coal'] = data[data.is_coalition == 1].groupby(['vote']).sum().apply(lambda row: getOptions(row, 'coal'), axis=1)
except:
all_votes['coal'] = ""
all_votes['oppo'] = data[data.is_coalition == 0].groupby(['vote']).sum().apply(lambda row: getOptions(row, 'oppo'), axis=1)
parties = data.groupby(['vote',
'voterparty']).sum().apply(lambda row: getOptions(row,
'ps'), axis=1)
partyBallots = data.groupby(['vote',
'voterparty']).sum().apply(lambda row: getPartyBallot(row), axis=1)
partyIntryDisunion = data.groupby(['vote', 'voterparty']).sum().apply(lambda row: getIntraDisunion(row), axis=1)
# TODO: create save-ing for coalInter, oppoInter
coalInterCalc = data[data.is_coalition == 1].groupby(['vote']).sum().apply(lambda row: getIntraDisunion(row), axis=1)
oppoInterCalc = data[data.is_coalition == 0].groupby(['vote']).sum().apply(lambda row: getIntraDisunion(row), axis=1)
allInter = data.groupby(['vote']).sum().apply(lambda row: getIntraDisunion(row), axis=1)
for vote_id in all_votes.index.values:
vote = Vote.objects.get(id_parladata=vote_id)
vote_a = Vote_analysis.objects.filter(vote__id_parladata=vote_id)
party_data = {}
has_outliers = False
for party in parties[vote_id].keys():
# save just parlimetary groups
if party in paries_ids:
party_data[str(party)] = parties[vote_id][party]
if json.loads(party_data[str(party)])['outliers']:
has_outliers = True
# update/add IntraDisunion
intra = IntraDisunion.objects.filter(organization__id_parladata=party,
vote=vote)
if intra:
intra.update(maximum=partyIntryDisunion[vote_id][party])
else:
org = Organization.objects.get(id_parladata=party)
IntraDisunion(organization=org,
vote=vote,
maximum=partyIntryDisunion[vote_id][party]
).save()
# save org Ballot
options = json.loads(partyBallots[vote_id][party])
for option in options:
# if ballot doesn't exist then create it
if not Ballot.objects.filter(org_voter__id_parladata=party,
vote__id_parladata=vote_id,
option=option):
org = Organization.objects.get(id_parladata=party)
Ballot(vote=vote,
org_voter=org,
option=option,
start_time=vote.start_time,
session=vote.session).save()
if calc_sides:
opoIntra = IntraDisunion.objects.filter(organization=opozition_org,
vote=vote)
coalIntra = IntraDisunion.objects.filter(organization=coalition_org,
vote=vote)
if opoIntra:
opoIntra.update(maximum=oppoInterCalc[vote_id])
else:
IntraDisunion(organization=opozition_org,
vote=vote,
maximum=oppoInterCalc[vote_id]
).save()
if coalInterCalc.empty:
IntraDisunion(organization=coalition_org,
vote=vote,
maximum=0
).save()
else:
if coalIntra:
coalIntra.update(maximum=coalInterCalc[vote_id])
else:
IntraDisunion(organization=coalition_org,
vote=vote,
maximum=coalInterCalc[vote_id]
).save()
vote.has_outlier_voters = has_outliers
vote.intra_disunion = allInter[vote_id]
vote.save()
print all_votes.loc[vote_id, 'pg_for']
if vote_a:
vote_a.update(votes_for=all_votes.loc[vote_id, 'option_for'],
against=all_votes.loc[vote_id, 'option_against'],
abstain=all_votes.loc[vote_id, 'option_abstain'],
not_present=all_votes.loc[vote_id, 'option_absent'],
pgs_data=party_data,
mp_yes=all_votes.loc[vote_id, 'm_for'],
mp_no=all_votes.loc[vote_id, 'm_against'],
mp_np=all_votes.loc[vote_id, 'm_absent'],
mp_kvor=all_votes.loc[vote_id, 'm_abstain'],
coal_opts=all_votes.loc[vote_id, 'coal'] if calc_sides else None,
oppo_opts=all_votes.loc[vote_id, 'oppo'] if calc_sides else None)
else:
Vote_analysis(session=session,
vote=vote,
created_for=vote.start_time,
votes_for=all_votes.loc[vote_id, 'option_for'],
against=all_votes.loc[vote_id, 'option_against'],
abstain=all_votes.loc[vote_id, 'option_abstain'],
not_present=all_votes.loc[vote_id, 'option_absent'],
pgs_data=party_data,
mp_yes=all_votes.loc[vote_id, 'm_for'],
mp_no=all_votes.loc[vote_id, 'm_against'],
mp_np=all_votes.loc[vote_id, 'm_absent'],
mp_kvor=all_votes.loc[vote_id, 'm_abstain'],
coal_opts=all_votes.loc[vote_id, 'coal'] if calc_sides else None,
oppo_opts=all_votes.loc[vote_id, 'oppo'] if calc_sides else None).save()
def handle(self, *args, **options):
date_of = datetime.now().date()
for parent_id in getParentOrganizationsWithVoters():
mps = getVotersIDs(date_=date_of, organization_id=parent_id)
for org_id in getOrganizationsWithVoters(organization_id=parent_id, date_=date_of):
setPercentOFAttendedSessionPG(org_id, date_of=date_of, parenet_org=parent_id)