def __init__(self, organization_id, count_of='members', date_=None): self.date_ = '' self.api_url = None self.date_of = None self.members = None self.membersOfPGsRanges = None self.membersOfPGs = None self.count_of = 'members' self.text = {} self.unique_words = [] self.all_words = [] self.swizec_coef = [] self.average_words = None if date_: self.date_of = datetime.strptime(date_, API_DATE_FORMAT).date() self.date_ = date_ else: self.date_of = datetime.now().date() self.date_ = '' self.count_of = count_of # get members for this day self.members = getVotersIDs(organization_id=organization_id, date_=self.date_of) self.organizations = getOrganizationsWithVoters( organization_id=organization_id, date_=self.date_of) self.prepereSpeeches() self.wordCounter()
def prepareData(self): if self.debug: self.data = pd.read_pickle('backup_baze.pkl') else: self.data = pd.DataFrame() for page in getBallotTable(organization=self.organization_id): temp = pd.DataFrame(page) self.data = self.data.append(temp, ignore_index=True) self.members = getVotersIDs(organization_id=self.organization_id, date_=self.date_of) self.memsOfPGs = getOrganizationsWithVotersList( organization_id=self.organization_id, date_=self.date_of) self.pgs = getOrganizationsWithVoters( organization_id=self.organization_id, date_=self.date_of) def toLogic(row): """ voter option to logic value """ return VOTE_MAP[row['option']] # prepere helper columns (logic, option_ni, voter_unit) self.data['logic'] = self.data.apply(lambda row: toLogic(row), axis=1) self.data['option_x'] = 0 self.data.loc[self.data['option'] == 'absent', 'option_x'] = 1 self.data['voter_unit'] = 1
def data_to_csv(): models = [(Presence, 'prisotnost na glasovanjih sej DZ'), (NumberOfQuestions, 'st. poslanskih vprasanj in pobud'), (MismatchOfPG, 'neujemanje s poslansko skupino'), (SpokenWords, 'st. izgovorjenih besed'), (AverageNumberOfSpeechesPerSession, 'stevilo govorov na sejo'), (VocabularySize, 'raznolikost besedisca'), (MPStaticPL, 'st_mandatov')] mps = getVotersIDs() data = [] for mp in mps: tmp = {'member_id': mp} for model, column in models: print mp, model model_data = getPersonCardModelNew(model, mp) if model == Presence: tmp[column] = model_data.person_value_votes if model == NumberOfQuestions or model == SpokenWords or model == VocabularySize: tmp[column] = model_data.score if model == MismatchOfPG: tmp[column] = model_data.data if model == NumberOfSpeechesPerSession: tmp[column] = model_data.person_value if model == MPStaticPL: tmp[column] = model_data.mandates tmp['izobrazba'] = model_data.education_level tmp['ime'] = model_data.person.name tmp['PS'] = model_data.party.acronym data.append(tmp) listToCSV(data, 'toski_export.csv')
def handle(self, *args, **options): date_of = datetime.now().date() date_ = date_of.strftime(API_DATE_FORMAT) for org_id in getParentOrganizationsWithVoters(): members = getVotersIDs(organization_id=org_id) setPercentOFAttendedSession(self, members, date_of) return 0
def setPercentOFAttendedSessionPG(pg_id, date_of, parenet_org): date_ = date_of.strftime(API_DATE_FORMAT) allSum = {} data = {} voters = getVotersPairsWithOrg(date_=date_of, organization_id=parenet_org) membersOfPG = {} for i, value in sorted(voters.iteritems()): membersOfPG.setdefault(value, []).append(i) data = getNumberOfAllMPAttendedSessions(date_of, getVotersIDs(date_=date_of, organization_id=parenet_org)) sessions = {pg: [] for pg in membersOfPG if membersOfPG[pg]} votes = {pg: [] for pg in membersOfPG if membersOfPG[pg]} for pg in membersOfPG: if not membersOfPG[pg]: continue for member in membersOfPG[pg]: if str(member) in data['sessions'].keys(): sessions[pg].append(data['sessions'][str(member)]) votes[pg].append(data['votes'][str(member)]) sessions[pg] = sum(sessions[pg])/len(sessions[pg]) votes[pg] = sum(votes[pg])/len(votes[pg]) thisMPSessions = sessions[pg_id] maximumSessions = max(sessions.values()) maximumPGSessions = [pgId for pgId in sessions if sessions[pgId] == maximumSessions][:5] averageSessions = sum(data['sessions'].values()) / len(data['sessions']) thisMPVotes = votes[pg_id] maximumVotes = max(votes.values()) maximumPGVotes = [pgId for pgId in votes if votes[pgId] == maximumVotes][:5] averageVotes = sum(data['votes'].values()) / len(data['votes']) org = Organization.objects.get(id_parladata=int(pg_id)) result = saveOrAbortNew(model=PercentOFAttendedSession, created_for=date_of, organization=org, organization_value_sessions=thisMPSessions, maxPG_sessions=maximumPGSessions, average_sessions=averageSessions, maximum_sessions=maximumSessions, organization_value_votes=thisMPVotes, maxPG_votes=maximumPGVotes, average_votes=averageVotes, maximum_votes=maximumVotes)
def handle(self, *args, **options): date_ = '' if options['date']: date_of = datetime.strptime(options['date'], API_DATE_FORMAT).date() date_ = options['date'] else: # dirty work around, TODO: fix findDatesFromLastCard for input without person_id #date_of = findDatesFromLastCard(Presence, '11', datetime.now().strftime(API_DATE_FORMAT))[0] date_of = datetime.now().date() date_ = date_of.strftime(API_DATE_FORMAT) data = getQuestions() self.stdout.write('Get voters') for org_id in getParentOrganizationsWithVoters(): mps_ids = getVotersIDs(date_=date_of, organization_id=org_id) print(mps_ids) authors = [] for question in data: for author in question['authors']: if author in mps_ids: authors.append(author) print(org_id, len(authors), float(len(mps_ids))) try: avg = len(authors) / float(len(mps_ids)) except ZeroDivisionError as error: avg = 0 question_count = Counter(authors) max_value = 0 max_persons = [] for maxi in question_count.most_common(90): if max_value == 0: max_value = maxi[1] if maxi[1] == max_value: max_persons.append(maxi[0]) else: break for person_id in mps_ids: person = Person.objects.get(id_parladata=person_id) saveOrAbortNew(model=NumberOfQuestions, created_for=date_of, person=person, score=question_count[person_id], average=avg, maximum=max_value, maxMPs=max_persons) self.stdout.write('Done setting NumberOfQuestions to MP %s' % str(person.id_parladata)) return 0
def handle(self, *args, **options): speaker_ids = [] if options['speaker_ids']: speaker_ids = options['speaker_ids'] else: self.stdout.write('getVotersIDs') speaker_ids = mps_ids = getVotersIDs() for speaker_id in speaker_ids: setTfidfOfMP(self, speaker_id) return 0
def handle(self, *args, **options): self.stdout.write('Fetching VotersIDs') mps_ids = getVotersIDs() for person in Person.objects.all(): self.stdout.write('Updating person %s' % str(person.id_parladata)) if person.actived == 'Yes': if person.id_parladata not in mps_ids: person.actived = 'No' person.save() else: if person.id_parladata in mps_ids: person.actived = 'Yes' person.save() return 0
def handle(self, *args, **options): if options['date']: date_of = datetime.strptime(options['date'], API_DATE_FORMAT).date() date_ = options['date'] else: # dirty work around, TODO: fix findDatesFromLastCard for input without person_id #date_of = findDatesFromLastCard(Presence, '11', datetime.now().strftime(API_DATE_FORMAT))[0] date_of = datetime.now().date() date_ = date_of.strftime(API_DATE_FORMAT) self.stdout.write('Getting voters') for mp in getVotersIDs(date_=date_of): self.stdout.write('Running setMembershipsOfMember on %s' % str(mp)) setMembershipsOfMember(self, str(mp), date_) return 0
def getSpeechesAnalyses(self, organization_id): with open('members_speech_score.csv', 'wb') as csvfile: csvwriter = csv.writer(csvfile, delimiter=',', quotechar='|', quoting=csv.QUOTE_MINIMAL) date_of = datetime.now().date() mps = getVotersIDs(date_=date_of, organization_id=organization_id) # NumberOfSpeechesPerSession mp_scores = {} for mp in mps: mp_no_of_speeches = len( getSpeechContentOfPerson(person_id, fdate=date_of)) mp_no_of_sessions = Activity.objects.filter( person__id_parladata=mp, speech__isnull=False).distinct("session").count() if mp_no_of_sessions > 0: mp_scores[mp] = mp_no_of_speeches / mp_no_of_sessions else: mp_scores[mp] = 0 print 'NumberOfSpeechesPerSession done' # VocabularySize vocabulary_sizes = {} mp_results = {} for mp in mps: text = ' '.join(getSpeechContentOfPerson(mp, fdate=date_of)) vocabulary_sizes[mp] = len(countWords(text, Counter())) mp_results[mp] = numberOfWords(text) print 'VocabularySize done' print 'spoken words done' for mp in mps: csvwriter.writerow( [mp, mp_results[mp], vocabulary_sizes[mp], mp_scores[mp]])
def handle(self, *args, **options): speaker_ids = [] if options['speaker_ids']: speaker_ids = options['speaker_ids'] else: self.stdout.write('Getting voters') speaker_ids = getVotersIDs() # get static data self.stdout.write('Getting all static data') static_data = json.loads(getAllStaticData(None).content) for speaker_id in speaker_ids: self.stdout.write('About to begin with speaker %s' % str(speaker_id)) speaker = Person.objects.filter(id_parladata=speaker_id) if not speaker: self.stdout.write('Speaker with id %s does not exist' % str(speaker_id)) continue else: speaker = speaker[0] output = [{ 'term': 'VIII', 'type': 'pmegastring', 'id': 'pms_' + str(speaker.id_parladata), 'person_id': speaker.id_parladata, 'person_json': json.dumps(static_data['persons'][str(speaker.id_parladata)]), 'content': getSpeakerMegastring(speaker), }] commit_to_solr(self, output) return 0
def handle(self, *args, **options): factory = RequestFactory() request_with_key = factory.get('?key=' + SETTER_KEY) fast = False date_ = None sc = SlackClient(slack_token) start_time = time() yesterday = (datetime.now() - timedelta(days=1)).date() yesterday = datetime.combine(yesterday, datetime.min.time()) new_redna_seja = [] lockFile = open('parser.lock', 'w+') lockFile.write('LOCKED') lockFile.close() sc.api_call("chat.postMessage", channel="#parlalize_notif", text='Start fast update at: ' + str(datetime.now())) dates = [] lastBallotTime = self.getModelStartTime(Ballot) lastVoteTime = self.getModelStartTime(Vote) lastSpeechTime = self.getModelStartTime(Speech) lastQustionTime = self.getModelStartTime(Question) lastLegislationTime = self.getModelStartTime(Legislation) if date_: dates = [date_ + '_00:00' for i in range(5)] else: # get dates of last update dates.append(self.getModelStartTime(Person)) dates.append(self.getModelStartTime(Session)) dates.append(datetime.now() - timedelta(days=1)) #lastSpeechTime) dates.append(lastBallotTime) dates.append(lastQustionTime) #lastLegislationTime=datetime.now()-timedelta(days=10) dates.append(lastLegislationTime) # prepare url url = API_URL + '/getAllChangesAfter/' for sDate in dates: url += sDate.strftime(API_DATE_FORMAT + '_%H:%M') + '/' self.stdout.write(url) data = tryHard(url[:-1]).json() self.stdout.write('Speeches: ' + str(len(data['speeches']))) self.stdout.write('Sessions: ' + str(len(data['sessions']))) self.stdout.write('Persons: ' + str(len(data['persons']))) self.stdout.write('Questions: ' + str(len(data['questions']))) self.stdout.write('Legislation: ' + str(len(data['laws']))) text = ('Received data: \n' 'Speeches: ' + str(len(data['speeches'])) + '\n' 'Sessions: ' + str(len(data['sessions'])) + '\n' 'Persons: ' + str(len(data['persons'])) + '\n' 'Questions: ' + str(len(data['questions'])) + '\n' 'Legislation: ' + str(len(data['laws'])) + '\n') sc.api_call("chat.postMessage", channel="#parlalize_notif", text=text) sdate = datetime.now().strftime(API_DATE_FORMAT) # Persons mps_ids = getVotersIDs() for mp in data['persons']: if Person.objects.filter(id_parladata=mp['id']): person = Person.objects.get(id_parladata=mp['id']) person.name = mp['name'] person.pg = mp['membership'] person.id_parladata = int(mp['id']) person.image = mp['image'] person.actived = True if int(mp['id']) in mps_ids else False person.gov_id = mp['gov_id'] person.save() else: actived = True if int(mp['id']) in mps_ids else False person = Person(name=mp['name'], pg=mp['membership'], id_parladata=int(mp['id']), image=mp['image'], actived=actived, gov_id=mp['gov_id']) person.save() session_ids = list(Session.objects.all().values_list('id_parladata', flat=True)) # sessions for sessions in data['sessions']: orgs = Organization.objects.filter( id_parladata__in=sessions['organizations_id']) if not orgs: orgs = Organization.objects.filter( id_parladata=sessions['organization_id']) if sessions['id'] not in session_ids: result = Session(name=sessions['name'], gov_id=sessions['gov_id'], start_time=sessions['start_time'], end_time=sessions['end_time'], classification=sessions['classification'], id_parladata=sessions['id'], organization=orgs[0], in_review=sessions['is_in_review']) result.save() orgs = list(orgs) result.organizations.add(*orgs) if sessions['id'] == DZ: if 'redna seja' in sessions['name'].lower(): # call method for create new list of members #new_redna_seja.append(sessions) pass else: if not Session.objects.filter( name=sessions['name'], gov_id=sessions['gov_id'], start_time=sessions['start_time'], end_time=sessions['end_time'], classification=sessions['classification'], id_parladata=sessions['id'], organization=orgs[0], in_review=sessions['is_in_review']): # save changes session = Session.objects.get(id_parladata=sessions['id']) session.name = sessions['name'] session.gov_id = sessions['gov_id'] session.start_time = sessions['start_time'] session.end_time = sessions['end_time'] session.classification = sessions['classification'] session.in_review = sessions['is_in_review'] session.save() orgs = list(orgs) session.organizations.add(*orgs) # update Legislation for epa, laws in groupby(data['laws'], lambda item: item['epa']): last_obj = None sessions = [] is_ended = False for law in laws: sessions.append(law['session']) law['date'] = datetime.strptime(law['date'], '%Y-%m-%dT%X') if not is_ended: if law['procedure_ended']: is_ended = True if last_obj: if law['date'] > last_obj['date']: last_obj = law else: last_obj = law result = Legislation.objects.filter(epa=epa) # dont update Legislatin procedure_ended back to False if result: result = result[0] if result.procedure_ended: is_ended = True self.stdout.write('update') result.text = last_obj['text'] result.mdt = last_obj['mdt'] result.proposer_text = last_obj['proposer_text'] result.procedure_phase = last_obj['procedure_phase'] result.procedure = last_obj['procedure'] result.type_of_law = last_obj['type_of_law'] result.id_parladata = last_obj['id'] result.date = last_obj['date'] result.procedure_ended = is_ended result.classification = last_obj['classification'] result.save() else: self.stdout.write('adding') result = Legislation( text=last_obj['text'], epa=last_obj['epa'], mdt=last_obj['mdt'], proposer_text=last_obj['proposer_text'], procedure_phase=last_obj['procedure_phase'], procedure=last_obj['procedure'], type_of_law=last_obj['type_of_law'], id_parladata=last_obj['id'], date=last_obj['date'], procedure_ended=is_ended, classification=last_obj['classification'], result=LEGISLATION_STATUS[0][0]) result.save() sessions = list(set(sessions)) sessions = list(Session.objects.filter(id_parladata__in=sessions)) result.sessions.add(*sessions) #self.stdout.write(epa) # update speeches existingIDs = list(Speech.objects.all().values_list('id_parladata', flat=True)) sc.api_call('chat.postMessage', channel='#parlalize_notif', text='Start update speeches at: ' + str(datetime.now())) for dic in data['speeches']: if int(dic['id']) not in existingIDs: self.stdout.write('adding speech') person = Person.objects.get(id_parladata=int(dic['speaker'])) speech = Speech(organization=Organization.objects.get( id_parladata=int(dic['party'])), content=dic['content'], agenda_item_order=dic['agenda_item_order'], order=dic['order'], session=Session.objects.get( id_parladata=int(dic['session'])), start_time=dic['start_time'], end_time=dic['end_time'], valid_from=dic['valid_from'], valid_to=dic['valid_to'], id_parladata=dic['id']) speech.save() speech.person.add(person) else: self.stdout.write('update speech') person = Person.objects.get(id_parladata=int(dic['speaker'])) speech = Speech.objects.filter(id_parladata=dic["id"]) speech.update(content=dic['content'], valid_from=dic['valid_from'], valid_to=dic['valid_to']) # update Votes sc.api_call("chat.postMessage", channel="#parlalize_notif", text='Start update votes at: ' + str(datetime.now())) for session_id in data['sessions_of_updated_votes']: self.stdout.write('set motion of session ' + str(session_id)) c = Command() setMotionOfSession(self, str(session_id)) # update ballots sc.api_call("chat.postMessage", channel="#parlalize_notif", text='Start update ballots at: ' + str(datetime.now())) existingISs = Ballot.objects.all().values_list('id_parladata', flat=True) for dic in data['ballots']: if int(dic['id']) not in existingISs: self.stdout.write('adding ballot ' + str(dic['vote'])) vote = Vote.objects.get(id_parladata=dic['vote']) person = Person.objects.get(id_parladata=int(dic['voter'])) ballots = Ballot(option=dic['option'], vote=vote, start_time=vote.start_time, end_time=None, id_parladata=dic['id'], voter_party=Organization.objects.get( id_parladata=dic['voterparty'])) ballots.save() ballots.person.add(person) # update questions sc.api_call("chat.postMessage", channel="#parlalize_notif", text='Start update Questions at: ' + str(datetime.now())) existingISs = list(Question.objects.all().values_list('id_parladata', flat=True)) for dic in data['questions']: if int(dic['id']) not in existingISs: self.stdout.write('adding question') if dic['session_id']: session = Session.objects.get( id_parladata=int(dic['session_id'])) else: session = None link = dic['link'] if dic['link'] else None person = [] for i in dic['author_id']: person.append(Person.objects.get(id_parladata=int(i))) if dic['recipient_id']: rec_p = list( Person.objects.filter( id_parladata__in=dic['recipient_id'])) else: rec_p = [] if dic['recipient_org_id']: rec_org = list( Organization.objects.filter( id_parladata__in=dic['recipient_org_id'])) else: rec_org = [] author_org = [] for i in dic['author_org_id']: author_org.append(Organization.objects.get(id_parladata=i)) rec_posts = [] for post in dic['recipient_posts']: static = MinisterStatic.objects.filter( person__id_parladata=post['membership__person_id'], ministry__id_parladata=post['organization_id'] ).order_by('-created_for') if static: rec_posts.append(static[0]) question = Question( session=session, start_time=dic['date'], id_parladata=dic['id'], recipient_text=dic['recipient_text'], title=dic['title'], content_link=link, ) question.save() question.person.add(*person) question.author_orgs.add(*author_org) question.recipient_persons.add(*rec_p) question.recipient_organizations.add(*rec_org) question.recipient_persons_static.add(*rec_posts) sc.api_call("chat.postMessage", channel="#parlalize_notif", text='Start update distircts and tags at: ' + str(datetime.now())) t_delta = time() - start_time text = ('End fast update (' + str(t_delta) + ' s) and start' 'update sessions cards at: ' + str(datetime.now()) + '') sc.api_call("chat.postMessage", channel="#parlalize_notif", text=text) self.stdout.write('sessions') s_update = [] # sessions = Session.objects.filter(updated_at__gte=datetime.now().date) # s_update += list(sessions.values_list('id_parladata', flat=True)) votes = Vote.objects.filter(updated_at__gt=lastVoteTime) s_update += list(votes.values_list('session__id_parladata', flat=True)) ballots = Ballot.objects.filter(updated_at__gt=lastBallotTime) s_update += list( ballots.values_list('vote__session__id_parladata', flat=True)) p_update = list(ballots.values_list("person__id_parladata", flat=True)) #if s_update: # runSettersSessions(sessions_ids=list(set(s_update))) t_delta = time() - start_time text = ('End creating cards (' + str(t_delta) + ' s) and start' 'creating recache: ' + str(datetime.now()) + '') sc.api_call("chat.postMessage", channel="#parlalize_notif", text=text) lockFile = open('parser.lock', 'w+') lockFile.write('UNLOCKED') lockFile.close() # recache # add sesessions of updated speeches to recache speeches = Speech.objects.filter(updated_at__gt=lastSpeechTime) s_update += list( speeches.values_list("session__id_parladata", flat=True)) s_p_update = list( speeches.values_list("person__id_parladata", flat=True)) date_ = (datetime.now() + timedelta(days=1)).strftime(API_DATE_FORMAT) if s_update: getSessionsList(None, date_, force_render=True) self.stdout.write(str(s_update)) if s_update: self.stdout.write('recache') speech_the_order() deleteRendersOfSession(list(set(s_update)), update_votes_details=True, update_speeches=True) refetch() p_update += list( speeches.values_list("person__id_parladata", flat=True)) questions = Question.objects.filter(updated_at__gt=lastQustionTime) q_update = list( questions.values_list("person__id_parladata", flat=True)) p_update += q_update # nightly update if not fast: # read draft legislations importDraftLegislationsFromFeed() # update last activites deleteRendersOfIDs(list(set(p_update)), 'p', 'zadnje-aktivnosti') deleteRendersOfIDs(list(set(q_update)), 'p', 'poslanska-vprasanja-in-pobude') deleteRendersOfIDs(list(set(s_p_update)), 'p', 'povezave-do-govorov') delete_renders(group='pg', method='poslanska-vprasanja-in-pobude') delete_renders(group='pg', method='vsi-govori-poslanske-skupine') t_delta = time() - start_time text = ('End fastUpdate everything (' + str(t_delta) + ' s): ' '' + str(datetime.now()) + '') sc.api_call("chat.postMessage", channel="#parlalize_notif", text=text)
def getData(date_of, org_id): """ group balots for each person and calculate SVD """ # getting all the necessary data allballots = Ballot.objects.filter(vote__start_time__lte=date_of).values( "person__id_parladata", "option", "vote") print(allballots, org_id, date_of) # sort people's ids people_ids = sorted(getVotersIDs(organization_id=org_id, date_=date_of)) print('peplz', people_ids) people_without_ballots = [] # group ballots by people people_ballots = [] for voter in people_ids: people_ballots.append([ ballot for ballot in allballots if ballot['person__id_parladata'] == voter ]) if people_ballots[-1] == []: people_without_ballots.append(voter) lengths = [len(person) for person in people_ballots] print(lengths) # get ids of all votes all_vote_ids = [] for person in people_ballots: for ballot in person: all_vote_ids.append(ballot['vote']) all_vote_ids = set(all_vote_ids) # pad votes and write in "ni obstajal" for people who didn't exist yet print people_ballots for person in people_ballots: if len(person) < max(lengths): for vote_id in all_vote_ids: if len(person) == 0: if vote_id not in [ballot['vote'] for ballot in person]: person.append({ 'vote': vote_id, 'person__id_parladata': people_without_ballots[0], 'option': 'ni obstajal', 'id': -1 }) people_without_ballots.remove( people_without_ballots[0]) else: if vote_id not in [ballot['vote'] for ballot in person]: person.append({ 'vote': vote_id, 'person__id_parladata': person[0]['person__id_parladata'], 'option': 'ni obstajal', 'id': -1 }) # sort ballots people_ballots_sorted = sorted( [sorted(person, key=lambda k: k['vote']) for person in people_ballots], key=lambda j: j[0]['person__id_parladata']) people_ballots_sorted_list = [person for person in people_ballots_sorted] hijene = [] # assign numerical values to ballots for i, person in enumerate(people_ballots_sorted_list): # print person for j, ballot in enumerate(person): people_ballots_sorted_list[i][j] = assignValueToOption( ballot['option']) if (len(people_ballots_sorted_list[i]) - people_ballots_sorted_list[i].count(4)) < 5: hijene.append(i) print "brisem", i print people_ids hijene.sort() for i in reversed(hijene): print "delete ", people_ids[i], i del people_ballots_sorted_list[i] del people_ids[i] # transform numerical ballot values to numpy array thearray = np.array(people_ballots_sorted_list) print len(people_ballots_sorted_list) # generate similarity matrix by adding +1 each time two people's ballots match similarities = makeSimilarities(people_ballots_sorted_list) u, s, vT = np.linalg.svd(similarities) return enrichData(vT[1, :], vT[0, :], people_ids, date_of)
def handle(self, *args, **options): if options['date']: date_ = options['date'] date_of = datetime.strptime(API_DATE_FORMAT).date() else: date_of = datetime.now().date() date_ = date_of.strftime(API_DATE_FORMAT) self.stdout.write('Getting voters') for org in getParentOrganizationsWithVoters(): self.stdout.write('Starting style score for organization %s' % (org)) mps = getVotersIDs(organization_id=org, date_=date_of) scores = {} for person_id in mps: self.stdout.write('MP id: %s' % str(person_id)) # get word counts with solr counter = Counter(getCountList(self, int(person_id), date_)) total = sum(counter.values()) scores_local = getScores([problematicno, privzdignjeno, preprosto], counter, total) self.stdout.write('Outputting scores_local: %s' % str(scores_local)) scores[person_id] = scores_local self.stdout.write('Outputting scores: %s' % str(scores)) average = {"problematicno": sum([score['problematicno'] for score in scores.values()])/len(scores), "privzdignjeno": sum([score['privzdignjeno'] for score in scores.values()])/len(scores), "preprosto": sum([score['preprosto'] for score in scores.values()])/len(scores)} data = [] for person, score in scores.items(): data.append({'member': person, 'problematicno': score['problematicno'], 'privzdignjeno': score['privzdignjeno'], 'preprosto': score['preprosto'], 'problematicno_average': average['problematicno'], 'privzdignjeno_average': average['privzdignjeno'], 'preprosto_average': average['preprosto']}) for score in data: self.stdout.write('About to save %s' % str(score)) status = saveOrAbortNew(StyleScores, person=Person.objects.get( id_parladata=int(score["member"])), created_for=date_of, problematicno=float( score['problematicno']), privzdignjeno=float( score['privzdignjeno']), preprosto=float(score['preprosto']), problematicno_average=float( score['problematicno_average']), privzdignjeno_average=float( score['privzdignjeno_average']), preprosto_average=float( score['preprosto_average']) ) self.stdout.write('SaveOrAbort status: %s' % str(status)) return 0
def handle(self, *args, **options): if options['date']: date_ = options['date'] date_of = datetime.strptime(date_, API_DATE_FORMAT).date() else: date_ = datetime.now().date().strftime(API_DATE_FORMAT) date_of = datetime.now().date() data = getQuestions() for org_id in getParentOrganizationsWithVoters(): pgs_on_date = getOrganizationsWithVoters(date_=date_of, organization_id=org_id) mps = getVotersIDs(date_=date_of, organization_id=org_id) mpStatic = {} for mp in mps: mpStatic[str(mp)] = getPersonData(str(mp), date_) pg_ids = pgs_on_date authors = [] for question in data: qDate = datetime.strptime(question['date'], '%Y-%m-%dT%X') qDate = qDate.strftime(API_DATE_FORMAT) for author in question['authors']: try: person_data = mpStatic[str(author)] except KeyError as e: print(str(question['authors'])) person_data = getPersonData(str(author), date_) mpStatic[str(author)] = person_data if person_data and person_data['party'] and person_data[ 'party']['id']: authors.append(person_data['party']['id']) else: print 'person nima mpstatic: ', author avg = len(authors) / float(len(pg_ids)) question_count = Counter(authors) max_value = 0 max_orgs = [] for maxi in question_count.most_common(90): if max_value == 0: max_value = maxi[1] if maxi[1] == max_value: max_orgs.append(maxi[0]) else: break is_saved = [] for pg_id in pg_ids: org = Organization.objects.get(id_parladata=pg_id) is_saved.append( saveOrAbortNew(model=NumberOfQuestions, created_for=date_of, organization=org, score=question_count[pg_id], average=avg, maximum=max_value, maxOrgs=max_orgs)) return 0
def handle(self, *args, **options): date_of = datetime.now().date() for parent_id in getParentOrganizationsWithVoters(): mps = getVotersIDs(date_=date_of, organization_id=parent_id) for org_id in getOrganizationsWithVoters(organization_id=parent_id, date_=date_of): setPercentOFAttendedSessionPG(org_id, date_of=date_of, parenet_org=parent_id)