def __init__(self, organization_id, count_of='members', date_=None):
self.date_ = ''
self.api_url = None
self.date_of = None
self.members = None
self.membersOfPGsRanges = None
self.membersOfPGs = None
self.count_of = 'members'
self.text = {}
self.unique_words = []
self.all_words = []
self.swizec_coef = []
self.average_words = None
if date_:
self.date_of = datetime.strptime(date_, API_DATE_FORMAT).date()
self.date_ = date_
else:
self.date_of = datetime.now().date()
self.date_ = ''
self.count_of = count_of
# get members for this day
self.members = getVotersIDs(organization_id=organization_id,
date_=self.date_of)
self.organizations = getOrganizationsWithVoters(
organization_id=organization_id, date_=self.date_of)
self.prepereSpeeches()
self.wordCounter()
def prepareData(self):
if self.debug:
self.data = pd.read_pickle('backup_baze.pkl')
else:
self.data = pd.DataFrame()
for page in getBallotTable(organization=self.organization_id):
temp = pd.DataFrame(page)
self.data = self.data.append(temp, ignore_index=True)
self.members = getVotersIDs(organization_id=self.organization_id,
date_=self.date_of)
self.memsOfPGs = getOrganizationsWithVotersList(
organization_id=self.organization_id, date_=self.date_of)
self.pgs = getOrganizationsWithVoters(
organization_id=self.organization_id, date_=self.date_of)
def toLogic(row):
"""
voter option to logic value
"""
return VOTE_MAP[row['option']]
# prepere helper columns (logic, option_ni, voter_unit)
self.data['logic'] = self.data.apply(lambda row: toLogic(row), axis=1)
self.data['option_x'] = 0
self.data.loc[self.data['option'] == 'absent', 'option_x'] = 1
self.data['voter_unit'] = 1
def data_to_csv():
models = [(Presence, 'prisotnost na glasovanjih sej DZ'),
(NumberOfQuestions, 'st. poslanskih vprasanj in pobud'),
(MismatchOfPG, 'neujemanje s poslansko skupino'),
(SpokenWords, 'st. izgovorjenih besed'),
(AverageNumberOfSpeechesPerSession, 'stevilo govorov na sejo'),
(VocabularySize, 'raznolikost besedisca'),
(MPStaticPL, 'st_mandatov')]
mps = getVotersIDs()
data = []
for mp in mps:
tmp = {'member_id': mp}
for model, column in models:
print mp, model
model_data = getPersonCardModelNew(model, mp)
if model == Presence:
tmp[column] = model_data.person_value_votes
if model == NumberOfQuestions or model == SpokenWords or model == VocabularySize:
tmp[column] = model_data.score
if model == MismatchOfPG:
tmp[column] = model_data.data
if model == NumberOfSpeechesPerSession:
tmp[column] = model_data.person_value
if model == MPStaticPL:
tmp[column] = model_data.mandates
tmp['izobrazba'] = model_data.education_level
tmp['ime'] = model_data.person.name
tmp['PS'] = model_data.party.acronym
data.append(tmp)
listToCSV(data, 'toski_export.csv')
def handle(self, *args, **options):
date_of = datetime.now().date()
date_ = date_of.strftime(API_DATE_FORMAT)
for org_id in getParentOrganizationsWithVoters():
members = getVotersIDs(organization_id=org_id)
setPercentOFAttendedSession(self, members, date_of)
return 0
def setPercentOFAttendedSessionPG(pg_id, date_of, parenet_org):
date_ = date_of.strftime(API_DATE_FORMAT)
allSum = {}
data = {}
voters = getVotersPairsWithOrg(date_=date_of, organization_id=parenet_org)
membersOfPG = {}
for i, value in sorted(voters.iteritems()):
membersOfPG.setdefault(value, []).append(i)
data = getNumberOfAllMPAttendedSessions(date_of, getVotersIDs(date_=date_of, organization_id=parenet_org))
sessions = {pg: [] for pg in membersOfPG if membersOfPG[pg]}
votes = {pg: [] for pg in membersOfPG if membersOfPG[pg]}
for pg in membersOfPG:
if not membersOfPG[pg]:
continue
for member in membersOfPG[pg]:
if str(member) in data['sessions'].keys():
sessions[pg].append(data['sessions'][str(member)])
votes[pg].append(data['votes'][str(member)])
sessions[pg] = sum(sessions[pg])/len(sessions[pg])
votes[pg] = sum(votes[pg])/len(votes[pg])
thisMPSessions = sessions[pg_id]
maximumSessions = max(sessions.values())
maximumPGSessions = [pgId
for pgId
in sessions
if sessions[pgId] == maximumSessions][:5]
averageSessions = sum(data['sessions'].values()) / len(data['sessions'])
thisMPVotes = votes[pg_id]
maximumVotes = max(votes.values())
maximumPGVotes = [pgId
for pgId
in votes
if votes[pgId] == maximumVotes][:5]
averageVotes = sum(data['votes'].values()) / len(data['votes'])
org = Organization.objects.get(id_parladata=int(pg_id))
result = saveOrAbortNew(model=PercentOFAttendedSession,
created_for=date_of,
organization=org,
organization_value_sessions=thisMPSessions,
maxPG_sessions=maximumPGSessions,
average_sessions=averageSessions,
maximum_sessions=maximumSessions,
organization_value_votes=thisMPVotes,
maxPG_votes=maximumPGVotes,
average_votes=averageVotes,
maximum_votes=maximumVotes)
def handle(self, *args, **options):
date_ = ''
if options['date']:
date_of = datetime.strptime(options['date'],
API_DATE_FORMAT).date()
date_ = options['date']
else:
# dirty work around, TODO: fix findDatesFromLastCard for input without person_id
#date_of = findDatesFromLastCard(Presence, '11', datetime.now().strftime(API_DATE_FORMAT))[0]
date_of = datetime.now().date()
date_ = date_of.strftime(API_DATE_FORMAT)
data = getQuestions()
self.stdout.write('Get voters')
for org_id in getParentOrganizationsWithVoters():
mps_ids = getVotersIDs(date_=date_of, organization_id=org_id)
print(mps_ids)
authors = []
for question in data:
for author in question['authors']:
if author in mps_ids:
authors.append(author)
print(org_id, len(authors), float(len(mps_ids)))
try:
avg = len(authors) / float(len(mps_ids))
except ZeroDivisionError as error:
avg = 0
question_count = Counter(authors)
max_value = 0
max_persons = []
for maxi in question_count.most_common(90):
if max_value == 0:
max_value = maxi[1]
if maxi[1] == max_value:
max_persons.append(maxi[0])
else:
break
for person_id in mps_ids:
person = Person.objects.get(id_parladata=person_id)
saveOrAbortNew(model=NumberOfQuestions,
created_for=date_of,
person=person,
score=question_count[person_id],
average=avg,
maximum=max_value,
maxMPs=max_persons)
self.stdout.write('Done setting NumberOfQuestions to MP %s' %
str(person.id_parladata))
return 0
def handle(self, *args, **options):
speaker_ids = []
if options['speaker_ids']:
speaker_ids = options['speaker_ids']
else:
self.stdout.write('getVotersIDs')
speaker_ids = mps_ids = getVotersIDs()
for speaker_id in speaker_ids:
setTfidfOfMP(self, speaker_id)
return 0
def handle(self, *args, **options):
self.stdout.write('Fetching VotersIDs')
mps_ids = getVotersIDs()
for person in Person.objects.all():
self.stdout.write('Updating person %s' % str(person.id_parladata))
if person.actived == 'Yes':
if person.id_parladata not in mps_ids:
person.actived = 'No'
person.save()
else:
if person.id_parladata in mps_ids:
person.actived = 'Yes'
person.save()
return 0
def handle(self, *args, **options):
if options['date']:
date_of = datetime.strptime(options['date'],
API_DATE_FORMAT).date()
date_ = options['date']
else:
# dirty work around, TODO: fix findDatesFromLastCard for input without person_id
#date_of = findDatesFromLastCard(Presence, '11', datetime.now().strftime(API_DATE_FORMAT))[0]
date_of = datetime.now().date()
date_ = date_of.strftime(API_DATE_FORMAT)
self.stdout.write('Getting voters')
for mp in getVotersIDs(date_=date_of):
self.stdout.write('Running setMembershipsOfMember on %s' % str(mp))
setMembershipsOfMember(self, str(mp), date_)
return 0
def getSpeechesAnalyses(self, organization_id):
with open('members_speech_score.csv', 'wb') as csvfile:
csvwriter = csv.writer(csvfile,
delimiter=',',
quotechar='|',
quoting=csv.QUOTE_MINIMAL)
date_of = datetime.now().date()
mps = getVotersIDs(date_=date_of, organization_id=organization_id)
# NumberOfSpeechesPerSession
mp_scores = {}
for mp in mps:
mp_no_of_speeches = len(
getSpeechContentOfPerson(person_id, fdate=date_of))
mp_no_of_sessions = Activity.objects.filter(
person__id_parladata=mp,
speech__isnull=False).distinct("session").count()
if mp_no_of_sessions > 0:
mp_scores[mp] = mp_no_of_speeches / mp_no_of_sessions
else:
mp_scores[mp] = 0
print 'NumberOfSpeechesPerSession done'
# VocabularySize
vocabulary_sizes = {}
mp_results = {}
for mp in mps:
text = ' '.join(getSpeechContentOfPerson(mp, fdate=date_of))
vocabulary_sizes[mp] = len(countWords(text, Counter()))
mp_results[mp] = numberOfWords(text)
print 'VocabularySize done'
print 'spoken words done'
for mp in mps:
csvwriter.writerow(
[mp, mp_results[mp], vocabulary_sizes[mp], mp_scores[mp]])
def handle(self, *args, **options):
speaker_ids = []
if options['speaker_ids']:
speaker_ids = options['speaker_ids']
else:
self.stdout.write('Getting voters')
speaker_ids = getVotersIDs()
# get static data
self.stdout.write('Getting all static data')
static_data = json.loads(getAllStaticData(None).content)
for speaker_id in speaker_ids:
self.stdout.write('About to begin with speaker %s' %
str(speaker_id))
speaker = Person.objects.filter(id_parladata=speaker_id)
if not speaker:
self.stdout.write('Speaker with id %s does not exist' %
str(speaker_id))
continue
else:
speaker = speaker[0]
output = [{
'term':
'VIII',
'type':
'pmegastring',
'id':
'pms_' + str(speaker.id_parladata),
'person_id':
speaker.id_parladata,
'person_json':
json.dumps(static_data['persons'][str(speaker.id_parladata)]),
'content':
getSpeakerMegastring(speaker),
}]
commit_to_solr(self, output)
return 0
def handle(self, *args, **options):
factory = RequestFactory()
request_with_key = factory.get('?key=' + SETTER_KEY)
fast = False
date_ = None
sc = SlackClient(slack_token)
start_time = time()
yesterday = (datetime.now() - timedelta(days=1)).date()
yesterday = datetime.combine(yesterday, datetime.min.time())
new_redna_seja = []
lockFile = open('parser.lock', 'w+')
lockFile.write('LOCKED')
lockFile.close()
sc.api_call("chat.postMessage",
channel="#parlalize_notif",
text='Start fast update at: ' + str(datetime.now()))
dates = []
lastBallotTime = self.getModelStartTime(Ballot)
lastVoteTime = self.getModelStartTime(Vote)
lastSpeechTime = self.getModelStartTime(Speech)
lastQustionTime = self.getModelStartTime(Question)
lastLegislationTime = self.getModelStartTime(Legislation)
if date_:
dates = [date_ + '_00:00' for i in range(5)]
else:
# get dates of last update
dates.append(self.getModelStartTime(Person))
dates.append(self.getModelStartTime(Session))
dates.append(datetime.now() - timedelta(days=1)) #lastSpeechTime)
dates.append(lastBallotTime)
dates.append(lastQustionTime)
#lastLegislationTime=datetime.now()-timedelta(days=10)
dates.append(lastLegislationTime)
# prepare url
url = API_URL + '/getAllChangesAfter/'
for sDate in dates:
url += sDate.strftime(API_DATE_FORMAT + '_%H:%M') + '/'
self.stdout.write(url)
data = tryHard(url[:-1]).json()
self.stdout.write('Speeches: ' + str(len(data['speeches'])))
self.stdout.write('Sessions: ' + str(len(data['sessions'])))
self.stdout.write('Persons: ' + str(len(data['persons'])))
self.stdout.write('Questions: ' + str(len(data['questions'])))
self.stdout.write('Legislation: ' + str(len(data['laws'])))
text = ('Received data: \n'
'Speeches: ' + str(len(data['speeches'])) + '\n'
'Sessions: ' + str(len(data['sessions'])) + '\n'
'Persons: ' + str(len(data['persons'])) + '\n'
'Questions: ' + str(len(data['questions'])) + '\n'
'Legislation: ' + str(len(data['laws'])) + '\n')
sc.api_call("chat.postMessage", channel="#parlalize_notif", text=text)
sdate = datetime.now().strftime(API_DATE_FORMAT)
# Persons
mps_ids = getVotersIDs()
for mp in data['persons']:
if Person.objects.filter(id_parladata=mp['id']):
person = Person.objects.get(id_parladata=mp['id'])
person.name = mp['name']
person.pg = mp['membership']
person.id_parladata = int(mp['id'])
person.image = mp['image']
person.actived = True if int(mp['id']) in mps_ids else False
person.gov_id = mp['gov_id']
person.save()
else:
actived = True if int(mp['id']) in mps_ids else False
person = Person(name=mp['name'],
pg=mp['membership'],
id_parladata=int(mp['id']),
image=mp['image'],
actived=actived,
gov_id=mp['gov_id'])
person.save()
session_ids = list(Session.objects.all().values_list('id_parladata',
flat=True))
# sessions
for sessions in data['sessions']:
orgs = Organization.objects.filter(
id_parladata__in=sessions['organizations_id'])
if not orgs:
orgs = Organization.objects.filter(
id_parladata=sessions['organization_id'])
if sessions['id'] not in session_ids:
result = Session(name=sessions['name'],
gov_id=sessions['gov_id'],
start_time=sessions['start_time'],
end_time=sessions['end_time'],
classification=sessions['classification'],
id_parladata=sessions['id'],
organization=orgs[0],
in_review=sessions['is_in_review'])
result.save()
orgs = list(orgs)
result.organizations.add(*orgs)
if sessions['id'] == DZ:
if 'redna seja' in sessions['name'].lower():
# call method for create new list of members
#new_redna_seja.append(sessions)
pass
else:
if not Session.objects.filter(
name=sessions['name'],
gov_id=sessions['gov_id'],
start_time=sessions['start_time'],
end_time=sessions['end_time'],
classification=sessions['classification'],
id_parladata=sessions['id'],
organization=orgs[0],
in_review=sessions['is_in_review']):
# save changes
session = Session.objects.get(id_parladata=sessions['id'])
session.name = sessions['name']
session.gov_id = sessions['gov_id']
session.start_time = sessions['start_time']
session.end_time = sessions['end_time']
session.classification = sessions['classification']
session.in_review = sessions['is_in_review']
session.save()
orgs = list(orgs)
session.organizations.add(*orgs)
# update Legislation
for epa, laws in groupby(data['laws'], lambda item: item['epa']):
last_obj = None
sessions = []
is_ended = False
for law in laws:
sessions.append(law['session'])
law['date'] = datetime.strptime(law['date'], '%Y-%m-%dT%X')
if not is_ended:
if law['procedure_ended']:
is_ended = True
if last_obj:
if law['date'] > last_obj['date']:
last_obj = law
else:
last_obj = law
result = Legislation.objects.filter(epa=epa)
# dont update Legislatin procedure_ended back to False
if result:
result = result[0]
if result.procedure_ended:
is_ended = True
self.stdout.write('update')
result.text = last_obj['text']
result.mdt = last_obj['mdt']
result.proposer_text = last_obj['proposer_text']
result.procedure_phase = last_obj['procedure_phase']
result.procedure = last_obj['procedure']
result.type_of_law = last_obj['type_of_law']
result.id_parladata = last_obj['id']
result.date = last_obj['date']
result.procedure_ended = is_ended
result.classification = last_obj['classification']
result.save()
else:
self.stdout.write('adding')
result = Legislation(
text=last_obj['text'],
epa=last_obj['epa'],
mdt=last_obj['mdt'],
proposer_text=last_obj['proposer_text'],
procedure_phase=last_obj['procedure_phase'],
procedure=last_obj['procedure'],
type_of_law=last_obj['type_of_law'],
id_parladata=last_obj['id'],
date=last_obj['date'],
procedure_ended=is_ended,
classification=last_obj['classification'],
result=LEGISLATION_STATUS[0][0])
result.save()
sessions = list(set(sessions))
sessions = list(Session.objects.filter(id_parladata__in=sessions))
result.sessions.add(*sessions)
#self.stdout.write(epa)
# update speeches
existingIDs = list(Speech.objects.all().values_list('id_parladata',
flat=True))
sc.api_call('chat.postMessage',
channel='#parlalize_notif',
text='Start update speeches at: ' + str(datetime.now()))
for dic in data['speeches']:
if int(dic['id']) not in existingIDs:
self.stdout.write('adding speech')
person = Person.objects.get(id_parladata=int(dic['speaker']))
speech = Speech(organization=Organization.objects.get(
id_parladata=int(dic['party'])),
content=dic['content'],
agenda_item_order=dic['agenda_item_order'],
order=dic['order'],
session=Session.objects.get(
id_parladata=int(dic['session'])),
start_time=dic['start_time'],
end_time=dic['end_time'],
valid_from=dic['valid_from'],
valid_to=dic['valid_to'],
id_parladata=dic['id'])
speech.save()
speech.person.add(person)
else:
self.stdout.write('update speech')
person = Person.objects.get(id_parladata=int(dic['speaker']))
speech = Speech.objects.filter(id_parladata=dic["id"])
speech.update(content=dic['content'],
valid_from=dic['valid_from'],
valid_to=dic['valid_to'])
# update Votes
sc.api_call("chat.postMessage",
channel="#parlalize_notif",
text='Start update votes at: ' + str(datetime.now()))
for session_id in data['sessions_of_updated_votes']:
self.stdout.write('set motion of session ' + str(session_id))
c = Command()
setMotionOfSession(self, str(session_id))
# update ballots
sc.api_call("chat.postMessage",
channel="#parlalize_notif",
text='Start update ballots at: ' + str(datetime.now()))
existingISs = Ballot.objects.all().values_list('id_parladata',
flat=True)
for dic in data['ballots']:
if int(dic['id']) not in existingISs:
self.stdout.write('adding ballot ' + str(dic['vote']))
vote = Vote.objects.get(id_parladata=dic['vote'])
person = Person.objects.get(id_parladata=int(dic['voter']))
ballots = Ballot(option=dic['option'],
vote=vote,
start_time=vote.start_time,
end_time=None,
id_parladata=dic['id'],
voter_party=Organization.objects.get(
id_parladata=dic['voterparty']))
ballots.save()
ballots.person.add(person)
# update questions
sc.api_call("chat.postMessage",
channel="#parlalize_notif",
text='Start update Questions at: ' + str(datetime.now()))
existingISs = list(Question.objects.all().values_list('id_parladata',
flat=True))
for dic in data['questions']:
if int(dic['id']) not in existingISs:
self.stdout.write('adding question')
if dic['session_id']:
session = Session.objects.get(
id_parladata=int(dic['session_id']))
else:
session = None
link = dic['link'] if dic['link'] else None
person = []
for i in dic['author_id']:
person.append(Person.objects.get(id_parladata=int(i)))
if dic['recipient_id']:
rec_p = list(
Person.objects.filter(
id_parladata__in=dic['recipient_id']))
else:
rec_p = []
if dic['recipient_org_id']:
rec_org = list(
Organization.objects.filter(
id_parladata__in=dic['recipient_org_id']))
else:
rec_org = []
author_org = []
for i in dic['author_org_id']:
author_org.append(Organization.objects.get(id_parladata=i))
rec_posts = []
for post in dic['recipient_posts']:
static = MinisterStatic.objects.filter(
person__id_parladata=post['membership__person_id'],
ministry__id_parladata=post['organization_id']
).order_by('-created_for')
if static:
rec_posts.append(static[0])
question = Question(
session=session,
start_time=dic['date'],
id_parladata=dic['id'],
recipient_text=dic['recipient_text'],
title=dic['title'],
content_link=link,
)
question.save()
question.person.add(*person)
question.author_orgs.add(*author_org)
question.recipient_persons.add(*rec_p)
question.recipient_organizations.add(*rec_org)
question.recipient_persons_static.add(*rec_posts)
sc.api_call("chat.postMessage",
channel="#parlalize_notif",
text='Start update distircts and tags at: ' +
str(datetime.now()))
t_delta = time() - start_time
text = ('End fast update (' + str(t_delta) + ' s) and start'
'update sessions cards at: ' + str(datetime.now()) + '')
sc.api_call("chat.postMessage", channel="#parlalize_notif", text=text)
self.stdout.write('sessions')
s_update = []
# sessions = Session.objects.filter(updated_at__gte=datetime.now().date)
# s_update += list(sessions.values_list('id_parladata', flat=True))
votes = Vote.objects.filter(updated_at__gt=lastVoteTime)
s_update += list(votes.values_list('session__id_parladata', flat=True))
ballots = Ballot.objects.filter(updated_at__gt=lastBallotTime)
s_update += list(
ballots.values_list('vote__session__id_parladata', flat=True))
p_update = list(ballots.values_list("person__id_parladata", flat=True))
#if s_update:
# runSettersSessions(sessions_ids=list(set(s_update)))
t_delta = time() - start_time
text = ('End creating cards (' + str(t_delta) + ' s) and start'
'creating recache: ' + str(datetime.now()) + '')
sc.api_call("chat.postMessage", channel="#parlalize_notif", text=text)
lockFile = open('parser.lock', 'w+')
lockFile.write('UNLOCKED')
lockFile.close()
# recache
# add sesessions of updated speeches to recache
speeches = Speech.objects.filter(updated_at__gt=lastSpeechTime)
s_update += list(
speeches.values_list("session__id_parladata", flat=True))
s_p_update = list(
speeches.values_list("person__id_parladata", flat=True))
date_ = (datetime.now() + timedelta(days=1)).strftime(API_DATE_FORMAT)
if s_update:
getSessionsList(None, date_, force_render=True)
self.stdout.write(str(s_update))
if s_update:
self.stdout.write('recache')
speech_the_order()
deleteRendersOfSession(list(set(s_update)),
update_votes_details=True,
update_speeches=True)
refetch()
p_update += list(
speeches.values_list("person__id_parladata", flat=True))
questions = Question.objects.filter(updated_at__gt=lastQustionTime)
q_update = list(
questions.values_list("person__id_parladata", flat=True))
p_update += q_update
# nightly update
if not fast:
# read draft legislations
importDraftLegislationsFromFeed()
# update last activites
deleteRendersOfIDs(list(set(p_update)), 'p', 'zadnje-aktivnosti')
deleteRendersOfIDs(list(set(q_update)), 'p',
'poslanska-vprasanja-in-pobude')
deleteRendersOfIDs(list(set(s_p_update)), 'p',
'povezave-do-govorov')
delete_renders(group='pg', method='poslanska-vprasanja-in-pobude')
delete_renders(group='pg', method='vsi-govori-poslanske-skupine')
t_delta = time() - start_time
text = ('End fastUpdate everything (' + str(t_delta) + ' s): '
'' + str(datetime.now()) + '')
sc.api_call("chat.postMessage", channel="#parlalize_notif", text=text)
def getData(date_of, org_id):
"""
group balots for each person and calculate SVD
"""
# getting all the necessary data
allballots = Ballot.objects.filter(vote__start_time__lte=date_of).values(
"person__id_parladata", "option", "vote")
print(allballots, org_id, date_of)
# sort people's ids
people_ids = sorted(getVotersIDs(organization_id=org_id, date_=date_of))
print('peplz', people_ids)
people_without_ballots = []
# group ballots by people
people_ballots = []
for voter in people_ids:
people_ballots.append([
ballot for ballot in allballots
if ballot['person__id_parladata'] == voter
])
if people_ballots[-1] == []:
people_without_ballots.append(voter)
lengths = [len(person) for person in people_ballots]
print(lengths)
# get ids of all votes
all_vote_ids = []
for person in people_ballots:
for ballot in person:
all_vote_ids.append(ballot['vote'])
all_vote_ids = set(all_vote_ids)
# pad votes and write in "ni obstajal" for people who didn't exist yet
print people_ballots
for person in people_ballots:
if len(person) < max(lengths):
for vote_id in all_vote_ids:
if len(person) == 0:
if vote_id not in [ballot['vote'] for ballot in person]:
person.append({
'vote':
vote_id,
'person__id_parladata':
people_without_ballots[0],
'option':
'ni obstajal',
'id':
-1
})
people_without_ballots.remove(
people_without_ballots[0])
else:
if vote_id not in [ballot['vote'] for ballot in person]:
person.append({
'vote':
vote_id,
'person__id_parladata':
person[0]['person__id_parladata'],
'option':
'ni obstajal',
'id':
-1
})
# sort ballots
people_ballots_sorted = sorted(
[sorted(person, key=lambda k: k['vote']) for person in people_ballots],
key=lambda j: j[0]['person__id_parladata'])
people_ballots_sorted_list = [person for person in people_ballots_sorted]
hijene = []
# assign numerical values to ballots
for i, person in enumerate(people_ballots_sorted_list):
# print person
for j, ballot in enumerate(person):
people_ballots_sorted_list[i][j] = assignValueToOption(
ballot['option'])
if (len(people_ballots_sorted_list[i]) -
people_ballots_sorted_list[i].count(4)) < 5:
hijene.append(i)
print "brisem", i
print people_ids
hijene.sort()
for i in reversed(hijene):
print "delete ", people_ids[i], i
del people_ballots_sorted_list[i]
del people_ids[i]
# transform numerical ballot values to numpy array
thearray = np.array(people_ballots_sorted_list)
print len(people_ballots_sorted_list)
# generate similarity matrix by adding +1 each time two people's ballots match
similarities = makeSimilarities(people_ballots_sorted_list)
u, s, vT = np.linalg.svd(similarities)
return enrichData(vT[1, :], vT[0, :], people_ids, date_of)
def handle(self, *args, **options):
if options['date']:
date_ = options['date']
date_of = datetime.strptime(API_DATE_FORMAT).date()
else:
date_of = datetime.now().date()
date_ = date_of.strftime(API_DATE_FORMAT)
self.stdout.write('Getting voters')
for org in getParentOrganizationsWithVoters():
self.stdout.write('Starting style score for organization %s' % (org))
mps = getVotersIDs(organization_id=org, date_=date_of)
scores = {}
for person_id in mps:
self.stdout.write('MP id: %s' % str(person_id))
# get word counts with solr
counter = Counter(getCountList(self, int(person_id), date_))
total = sum(counter.values())
scores_local = getScores([problematicno, privzdignjeno, preprosto],
counter,
total)
self.stdout.write('Outputting scores_local: %s' %
str(scores_local))
scores[person_id] = scores_local
self.stdout.write('Outputting scores: %s' % str(scores))
average = {"problematicno": sum([score['problematicno']
for score
in scores.values()])/len(scores),
"privzdignjeno": sum([score['privzdignjeno']
for score
in scores.values()])/len(scores),
"preprosto": sum([score['preprosto']
for score
in scores.values()])/len(scores)}
data = []
for person, score in scores.items():
data.append({'member': person,
'problematicno': score['problematicno'],
'privzdignjeno': score['privzdignjeno'],
'preprosto': score['preprosto'],
'problematicno_average': average['problematicno'],
'privzdignjeno_average': average['privzdignjeno'],
'preprosto_average': average['preprosto']})
for score in data:
self.stdout.write('About to save %s' % str(score))
status = saveOrAbortNew(StyleScores,
person=Person.objects.get(
id_parladata=int(score["member"])),
created_for=date_of,
problematicno=float(
score['problematicno']),
privzdignjeno=float(
score['privzdignjeno']),
preprosto=float(score['preprosto']),
problematicno_average=float(
score['problematicno_average']),
privzdignjeno_average=float(
score['privzdignjeno_average']),
preprosto_average=float(
score['preprosto_average'])
)
self.stdout.write('SaveOrAbort status: %s' % str(status))
return 0
def handle(self, *args, **options):
if options['date']:
date_ = options['date']
date_of = datetime.strptime(date_, API_DATE_FORMAT).date()
else:
date_ = datetime.now().date().strftime(API_DATE_FORMAT)
date_of = datetime.now().date()
data = getQuestions()
for org_id in getParentOrganizationsWithVoters():
pgs_on_date = getOrganizationsWithVoters(date_=date_of,
organization_id=org_id)
mps = getVotersIDs(date_=date_of, organization_id=org_id)
mpStatic = {}
for mp in mps:
mpStatic[str(mp)] = getPersonData(str(mp), date_)
pg_ids = pgs_on_date
authors = []
for question in data:
qDate = datetime.strptime(question['date'], '%Y-%m-%dT%X')
qDate = qDate.strftime(API_DATE_FORMAT)
for author in question['authors']:
try:
person_data = mpStatic[str(author)]
except KeyError as e:
print(str(question['authors']))
person_data = getPersonData(str(author), date_)
mpStatic[str(author)] = person_data
if person_data and person_data['party'] and person_data[
'party']['id']:
authors.append(person_data['party']['id'])
else:
print 'person nima mpstatic: ', author
avg = len(authors) / float(len(pg_ids))
question_count = Counter(authors)
max_value = 0
max_orgs = []
for maxi in question_count.most_common(90):
if max_value == 0:
max_value = maxi[1]
if maxi[1] == max_value:
max_orgs.append(maxi[0])
else:
break
is_saved = []
for pg_id in pg_ids:
org = Organization.objects.get(id_parladata=pg_id)
is_saved.append(
saveOrAbortNew(model=NumberOfQuestions,
created_for=date_of,
organization=org,
score=question_count[pg_id],
average=avg,
maximum=max_value,
maxOrgs=max_orgs))
return 0
def handle(self, *args, **options):
date_of = datetime.now().date()
for parent_id in getParentOrganizationsWithVoters():
mps = getVotersIDs(date_=date_of, organization_id=parent_id)
for org_id in getOrganizationsWithVoters(organization_id=parent_id, date_=date_of):
setPercentOFAttendedSessionPG(org_id, date_of=date_of, parenet_org=parent_id)
