def _handle_protocol(self, cm, request): if not cm.protocol_text: # don't override existing protocols cm.protocol_text = request.POST.get('protocol_text') cm.save() cm.create_protocol_parts() mks, mk_names = get_all_mk_names() cm.find_attending_members(mks, mk_names)
def find_attending_members(self, mks=None, mk_names=None): if mks is None and mk_names is None: mks, mk_names = get_all_mk_names() try: r = re.search( "חברי הו?ועדה(.*?)(\n[^\n]*(ייעוץ|יועץ|רישום|רש(מים|מות|מו|מ|מת|ם|מה)|קצר(נים|ניות|ן|נית))[\s|:])".decode( 'utf8'), self.protocol_text, re.DOTALL).group(1) s = r.split('\n') for (i, name) in enumerate(mk_names): if not mks[i].party_at(self.date): # not a member at time of # this meeting? continue # then don't search for this MK. for s0 in s: if s0.find(name) >= 0: self.mks_attended.add(mks[i]) except Exception: exceptionType, exceptionValue, exceptionTraceback = sys.exc_info() logger.debug("%s%s", ''.join(traceback.format_exception(exceptionType, exceptionValue, exceptionTraceback) ), '\nCommitteeMeeting.id=' + str(self.id)) logger.debug('meeting %d now has %d attending members' % ( self.id, self.mks_attended.count()))
def _handle_add_protocol(self, cm, request): if not cm.protocol_text: # don't override existing protocols cm.protocol_text = request.POST.get('protocol_text') cm.save() cm.create_protocol_parts() mks, mk_names = get_all_mk_names() cm.find_attending_members(mks, mk_names)
def redownload_and_reparse_protocol(self, request, qs): mks, mk_names = get_all_mk_names() for meeting in qs: meeting.reparse_protocol(mks=mks, mk_names=mk_names) self.message_user( request, "successfully redownloaded & reparsed %s meetings" % qs.count())
def create_plenum_protocol_parts(meeting, mks=None, mk_names=None): global _mks global _mk_names if mks is None or mk_names is None: (mks, mk_names) = get_all_mk_names() (_mks, _mk_names) = (mks, mk_names) global _parts _parts = [] txt = meeting.protocol_text.encode('utf-8') tree = ElementTree.fromstring(txt) titles = None for para in tree.iter('para'): (isBold, txt) = _plenum_parseParaElement(para) t = _plenum_parseParaText(txt, isBold) titles = _plenum_parsePara(txt, t, titles) for title in titles: titleHeader = title['t'].strip() titleBody = [] #_savePart(meeting,'',t,'title') for child in title['c']: if child['s'] == 1: # it's a speaker, save the aggregated title texts if len(titleHeader) > 0 or len(titleBody) > 0: _savePart(meeting, titleHeader, '\n\n'.join(titleBody), 'title') titleHeader = '' titleBody = [] speakerHeader = child['t'].strip() speakerText = [] for schild in child['c']: t = schild['t'].strip() if len(t) > 0: speakerText.append(t) _savePart(meeting, speakerHeader, '\n\n'.join(speakerText), 'speaker') else: t = child['t'].strip() if len(t) > 0: titleBody.append(t) if len(titleHeader) > 0 or len(titleBody) > 0: _savePart(meeting, titleHeader, '\n\n'.join(titleBody), 'title') if len(_parts) > 0: # find duplicates gotDuplicate = False otherMeetings = committees.models.CommitteeMeeting.objects.filter( date_string=meeting.date_string).exclude(id=meeting.id) if len(otherMeetings) > 0: for otherMeeting in otherMeetings: if otherMeeting.parts.count() == len(_parts): meeting.delete() gotDuplicate = True if gotDuplicate: logger.debug('got a duplicate meeting - deleting my meeting') else: committees.models.ProtocolPart.objects.bulk_create(_parts) logger.debug('wrote ' + str(len(_parts)) + ' protocol parts') for mk in _mks_attended: meeting.mks_attended.add(mk)
def reparse_protocol(self, request, qs): mks, mk_names = get_all_mk_names() for meeting in qs: logger.debug('reparsing meeting %s' % meeting.pk) meeting.reparse_protocol(redownload=False, mks=mks, mk_names=mk_names) self.message_user(request, "successfully reparsed %s meetings" % qs.count())
def create_plenum_protocol_parts(meeting,mks=None,mk_names=None): global _mks global _mk_names if mks is None or mk_names is None: (mks,mk_names)=get_all_mk_names() (_mks,_mk_names)=(mks,mk_names) global _parts _parts=[] txt=meeting.protocol_text.encode('utf-8') tree=ElementTree.fromstring(txt) titles=None for para in tree.iter('para'): (isBold,txt)=_plenum_parseParaElement(para) t=_plenum_parseParaText(txt,isBold) titles=_plenum_parsePara(txt,t,titles) for title in titles: titleHeader=title['t'].strip() titleBody=[] #_savePart(meeting,'',t,'title') for child in title['c']: if child['s']==1: # it's a speaker, save the aggregated title texts if len(titleHeader)>0 or len(titleBody)>0: _savePart(meeting,titleHeader,'\n\n'.join(titleBody),'title') titleHeader='' titleBody=[] speakerHeader=child['t'].strip() speakerText=[] for schild in child['c']: t=schild['t'].strip() if len(t)>0: speakerText.append(t) _savePart(meeting,speakerHeader,'\n\n'.join(speakerText),'speaker') else: t=child['t'].strip() if len(t)>0: titleBody.append(t) if len(titleHeader)>0 or len(titleBody)>0: _savePart(meeting,titleHeader,'\n\n'.join(titleBody),'title') if len(_parts)>0: # find duplicates gotDuplicate=False otherMeetings=committees.models.CommitteeMeeting.objects.filter(date_string=meeting.date_string).exclude(id=meeting.id) if len(otherMeetings)>0: for otherMeeting in otherMeetings: if otherMeeting.parts.count()==len(_parts): meeting.delete() gotDuplicate=True if gotDuplicate: logger.debug('got a duplicate meeting - deleting my meeting') else: committees.models.ProtocolPart.objects.bulk_create(_parts) logger.debug('wrote '+str(len(_parts))+' protocol parts') for mk in _mks_attended: meeting.mks_attended.add(mk)
def find_attending_members(self, mks=None, mk_names=None): logger.debug('find_attending_members') if mks is None and mk_names is None: logger.debug('get_all_mk_names') mks, mk_names = get_all_mk_names() with KnessetDataCommitteeMeetingProtocol.get_from_text(self.protocol_text) as protocol: attended_mk_names = protocol.find_attending_members(mk_names) for name in attended_mk_names: i = mk_names.index(name) if not mks[i].party_at(self.date): # not a member at time of this meeting? continue # then don't search for this MK. self.mks_attended.add(mks[i]) logger.debug('meeting %d now has %d attending members' % ( self.id, self.mks_attended.count()))
def find_attending_members(self, mks=None, mk_names=None): logger.debug('find_attending_members') if mks is None and mk_names is None: logger.debug('get_all_mk_names') mks, mk_names = get_all_mk_names() try: with KnessetDataCommitteeMeetingProtocol.get_from_text(self.protocol_text) as protocol: attended_mk_names = protocol.find_attending_members(mk_names) for name in attended_mk_names: i = mk_names.index(name) if not mks[i].party_at(self.date): # not a member at time of this meeting? continue # then don't search for this MK. self.mks_attended.add(mks[i]) except Exception: exceptionType, exceptionValue, exceptionTraceback = sys.exc_info() logger.debug("%s%s", ''.join(traceback.format_exception(exceptionType, exceptionValue, exceptionTraceback) ), '\nCommitteeMeeting.id=' + str(self.id)) logger.debug('meeting %d now has %d attending members' % ( self.id, self.mks_attended.count()))
def find_attending_members(self, mks=None, mk_names=None): if mks is None and mk_names is None: mks, mk_names = get_all_mk_names() try: r = re.search( "חברי הו?ועדה(.*?)(\n[^\n]*(ייעוץ|יועץ|רישום|רש(מים|מות|מו|מ|מת|ם|מה)|קצר(נים|ניות|ן|נית))[\s|:])" .decode('utf8'), self.protocol_text, re.DOTALL).group(1) s = r.split('\n') for (i, name) in enumerate(mk_names): if not mks[i].party_at(self.date): # not a member at time of # this meeting? continue # then don't search for this MK. for s0 in s: if s0.find(name) >= 0: self.mks_attended.add(mks[i]) except Exception: exceptionType, exceptionValue, exceptionTraceback = sys.exc_info() logger.debug( "%s%s", ''.join( traceback.format_exception(exceptionType, exceptionValue, exceptionTraceback)), '\nCommitteeMeeting.id=' + str(self.id)) logger.debug('meeting %d now has %d attending members' % (self.id, self.mks_attended.count()))
def __init__(self): super(Command, self).__init__() self.mks, self.mk_names = get_all_mk_names()
def redownload_and_reparse_protocol(self, request, qs): mks, mk_names = get_all_mk_names() for meeting in qs: meeting.reparse_protocol(mks=mks, mk_names=mk_names) self.message_user(request, "successfully redownloaded & reparsed %s meetings" % qs.count())
def _reparse_protocol(meeting): mks, mk_names = get_all_mk_names() meeting.reparse_protocol(mks=mks, mk_names=mk_names)
def post(self, request, **kwargs): cm = get_object_or_404(CommitteeMeeting, pk=kwargs['pk']) bill = None request = self.request user_input_type = request.POST.get('user_input_type') if user_input_type == 'bill': bill_id = request.POST.get('bill_id') if bill_id.isdigit(): bill = get_object_or_404(Bill, pk=bill_id) else: # not a number, maybe its p/1234 m = re.findall('\d+',bill_id) if len(m)!=1: raise ValueError("didn't find exactly 1 number in bill_id=%s" % bill_id) pp = PrivateProposal.objects.get(proposal_id=m[0]) bill = pp.bill if bill.stage in ['1','2','-2','3']: # this bill is in early stage, so cm must be one of the first meetings bill.first_committee_meetings.add(cm) else: # this bill is in later stages v = bill.first_vote # look for first vote if v and v.time.date() < cm.date: # and check if the cm is after it, bill.second_committee_meetings.add(cm) # if so, this is a second committee meeting else: # otherwise, assume its first cms. bill.first_committee_meetings.add(cm) bill.update_stage() action.send(request.user, verb='added-bill-to-cm', description=cm, target=bill, timestamp=datetime.datetime.now()) if user_input_type == 'mk': mk_names = Member.objects.values_list('name', flat=True) mk_name = difflib.get_close_matches(request.POST.get('mk_name'), mk_names)[0] mk = Member.objects.get(name=mk_name) cm.mks_attended.add(mk) cm.save() # just to signal, so the attended Action gets created. action.send(request.user, verb='added-mk-to-cm', description=cm, target=mk, timestamp=datetime.datetime.now()) if user_input_type == 'remove-mk': mk_names = Member.objects.values_list('name', flat=True) mk_name = difflib.get_close_matches(request.POST.get('mk_name'), mk_names)[0] mk = Member.objects.get(name=mk_name) cm.mks_attended.remove(mk) cm.save() # just to signal, so the attended Action gets created. action.send(request.user, verb='removed-mk-to-cm', description=cm, target=mk, timestamp=datetime.datetime.now()) if user_input_type == "protocol": if not cm.protocol_text: # don't override existing protocols cm.protocol_text = request.POST.get('protocol_text') cm.save() cm.create_protocol_parts() mks, mk_names = get_all_mk_names() cm.find_attending_members(mks, mk_names) return HttpResponseRedirect(".")
def post(self, request, **kwargs): cm = get_object_or_404(CommitteeMeeting, pk=kwargs['pk']) request = self.request user_input_type = request.POST.get('user_input_type') if user_input_type == 'bill': bill_id = request.POST.get('bill_id') if bill_id.isdigit(): bill = get_object_or_404(Bill, pk=bill_id) else: # not a number, maybe its p/1234 m = re.findall('\d+', bill_id) if len(m) != 1: raise ValueError("didn't find exactly 1 number in bill_id=%s" % bill_id) pp = PrivateProposal.objects.get(proposal_id=m[0]) bill = pp.bill if bill.stage in ['1', '2', '-2', '3']: # this bill is in early stage, so cm must be one of the first meetings bill.first_committee_meetings.add(cm) else: # this bill is in later stages v = bill.first_vote # look for first vote if v and v.time.date() < cm.date: # and check if the cm is after it, bill.second_committee_meetings.add( cm) # if so, this is a second committee meeting else: # otherwise, assume its first cms. bill.first_committee_meetings.add(cm) bill.update_stage() action.send(request.user, verb='added-bill-to-cm', description=cm, target=bill, timestamp=datetime.datetime.now()) if user_input_type == 'mk': mk_names = Member.objects.values_list('name', flat=True) mk_name = difflib.get_close_matches(request.POST.get('mk_name'), mk_names)[0] mk = Member.objects.get(name=mk_name) cm.mks_attended.add(mk) cm.save() # just to signal, so the attended Action gets created. action.send(request.user, verb='added-mk-to-cm', description=cm, target=mk, timestamp=datetime.datetime.now()) if user_input_type == 'remove-mk': mk_names = Member.objects.values_list('name', flat=True) mk_name = difflib.get_close_matches(request.POST.get('mk_name'), mk_names)[0] mk = Member.objects.get(name=mk_name) cm.mks_attended.remove(mk) cm.save() # just to signal, so the attended Action gets created. action.send(request.user, verb='removed-mk-to-cm', description=cm, target=mk, timestamp=datetime.datetime.now()) if user_input_type == 'add-lobbyist': l = Lobbyist.objects.get(person__name=request.POST.get( 'lobbyist_name')) cm.lobbyists_mentioned.add(l) if user_input_type == 'remove-lobbyist': l = Lobbyist.objects.get(person__name=request.POST.get( 'lobbyist_name')) cm.lobbyists_mentioned.remove(l) if user_input_type == "protocol": if not cm.protocol_text: # don't override existing protocols cm.protocol_text = request.POST.get('protocol_text') cm.save() cm.create_protocol_parts() mks, mk_names = get_all_mk_names() cm.find_attending_members(mks, mk_names) return HttpResponseRedirect(".")