def find_meeting(self, start_date=None, end_date=None): """ Find meetings within a given time frame and add them to the meeting queue. """ meeting_find_url = ( self.config['scraper']['allris']['meeting_find_url'] % (self.config['scraper']['base_url'], start_date.strftime("%d.%m.%Y"), end_date.strftime("%d.%m.%Y"))) logging.info("Getting meeting overview from %s", meeting_find_url) parser = etree.XMLParser(recover=True) h = HTMLParser.HTMLParser() r = self.get_url(meeting_find_url) if not r: return xml = r.text.encode('ascii', 'xmlcharrefreplace').replace('</a>', '') xml = re.sub(r'<a href="([^"]*)" target="_blank" ?>', r'\1', xml) root = etree.fromstring(xml, parser=parser) for item in root: if item.tag == 'list': root = item break for item in root.iterchildren(): raw_meeting = {} for e in item.iterchildren(): if e.text: raw_meeting[e.tag] = h.unescape(e.text) else: raw_meeting[e.tag] = '' meeting = Meeting(originalId=int(raw_meeting['silfdnr'])) meeting.start = self.parse_date(raw_meeting['sisbvcs']) meeting.end = self.parse_date(raw_meeting['sisevcs']) meeting.name = raw_meeting['siname'] meeting.originalUrl = ( "%sto010.asp?SILFDNR=%s&options=4" % (self.config['scraper']['base_url'], raw_meeting['silfdnr'])) meeting.name = raw_meeting['sitext'] meeting.organization_name = raw_meeting['grname'] # meeting.description = raw_meeting['sitext'] # WHAT TO DO WITH THIS self.db.save_meeting(meeting) self.meeting_queue.add(meeting.originalId)
def find_meeting(self, start_date=None, end_date=None): """ Find meetings within a given time frame and add them to the meeting queue. """ meeting_find_url = (self.config['scraper']['allris']['meeting_find_url'] % (self.config['scraper']['base_url'], start_date.strftime("%d.%m.%Y"), end_date.strftime("%d.%m.%Y"))) logging.info("Getting meeting overview from %s", meeting_find_url) parser = etree.XMLParser(recover=True) h = HTMLParser.HTMLParser() r = self.get_url(meeting_find_url) if not r: return xml = r.text.encode('ascii', 'xmlcharrefreplace').replace('</a>', '') xml = re.sub(r'<a href="([^"]*)" target="_blank" ?>', r'\1', xml) root = etree.fromstring(xml, parser=parser) for item in root: if item.tag == 'list': root = item break for item in root.iterchildren(): raw_meeting = {} for e in item.iterchildren(): if e.text: raw_meeting[e.tag] = h.unescape(e.text) else: raw_meeting[e.tag] = '' meeting = Meeting(originalId=int(raw_meeting['silfdnr'])) meeting.start = self.parse_date(raw_meeting['sisbvcs']) meeting.end = self.parse_date(raw_meeting['sisevcs']) meeting.name = raw_meeting['siname'] meeting.originalUrl = ("%sto010.asp?SILFDNR=%s&options=4" % (self.config['scraper']['base_url'], raw_meeting['silfdnr'])) meeting.name = raw_meeting['sitext'] meeting.organization_name = raw_meeting['grname'] # meeting.description = raw_meeting['sitext'] # WHAT TO DO WITH THIS self.db.save_meeting(meeting) self.meeting_queue.add(meeting.originalId)
def get_meeting(self, meeting_url=None, meeting_id=None): """ Load meeting details for the given detail page URL or numeric ID """ # Read either meeting_id or meeting_url from the opposite if meeting_id is not None: meeting_url = self.urls['MEETING_DETAIL_PRINT_PATTERN'] % (self.config["scraper"]["base_url"], meeting_id) elif meeting_url is not None: parsed = parse.search(self.urls['MEETING_DETAIL_PARSE_PATTERN'], meeting_url) meeting_id = parsed['meeting_id'] logging.info("Getting meeting (session) %d from %s", meeting_id, meeting_url) meeting = Meeting(originalId=meeting_id) time.sleep(self.config['scraper']['wait_time']) response = self.get_url(meeting_url) if not response: return # forms for later document download mechanize_forms = mechanize.ParseResponse(response, backwards_compat=False) # seek(0) is necessary to reset response pointer. response.seek(0) html = response.read() html = html.replace(' ', ' ') parser = etree.HTMLParser() dom = etree.parse(StringIO(html), parser) # check for page errors try: page_title = dom.xpath('//h1')[0].text if 'Fehlermeldung' in page_title: logging.info("Page %s cannot be accessed due to server error", meeting_url) return if 'Berechtigungsfehler' in page_title: logging.info("Page %s cannot be accessed due to permissions", meeting_url) return except: pass try: error_h3 = dom.xpath('//h3[@class="smc_h3"]')[0].text.strip() if 'Keine Daten gefunden' in error_h3: logging.info("Page %s does not contain any agenda items", meeting_url) return if 'Fehlercode: 1104' in error_h3: logging.info("Page %s cannot be accessed due to permissions", meeting_url) return except: pass meeting.originalUrl = meeting_url # Session title try: meeting.name = dom.xpath(self.xpath['MEETING_DETAIL_TITLE'])[0].text except: logging.critical('Cannot find session title element using XPath MEETING_DETAIL_TITLE') raise TemplateError('Cannot find session title element using XPath MEETING_DETAIL_TITLE') # Committe link #try: # links = dom.xpath(self.xpath['MEETING_DETAIL_COMMITTEE_LINK']) # for link in links: # href = link.get('href') # parsed = parse.search(self.urls['COMMITTEE_DETAIL_PARSE_PATTERN'], href) # if parsed is not None: # meeting.committees = [Commitee(numeric_id=int(parsed['committee_id']))] # if hasattr(self, 'committee_queue'): # self.committee_queue.add(int(parsed['committee_id'])) #except: # logging.critical('Cannot find link to committee detail page using MEETING_DETAIL_COMMITTEE_LINK_XPATH') # raise TemplateError('Cannot find link to committee detail page using MEETING_DETAIL_COMMITTEE_LINK_XPATH') # Meeting identifier, date, address etc tds = dom.xpath(self.xpath['MEETING_DETAIL_IDENTIFIER_TD']) if len(tds) == 0: logging.critical('Cannot find table fields using MEETING_DETAIL_IDENTIFIER_TD_XPATH at session ' + meeting_url) raise TemplateError('Cannot find table fields using MEETING_DETAIL_IDENTIFIER_TD_XPATH at session ' + meeting_url) else: for n in range(0, len(tds)): try: tdcontent = tds[n].text.strip() nextcontent = tds[n + 1].text.strip() except: continue if tdcontent == 'Sitzung:': meeting.shortName = nextcontent # We don't need this any more because it's scraped in committee detail page(?) #elif tdcontent == 'Gremium:': # meeting.committee_name = nextcontent elif tdcontent == 'Datum:': start = nextcontent end = nextcontent if tds[n + 2].text == 'Zeit:': if tds[n + 3].text is not None: times = tds[n + 3].text.replace(' Uhr', '').split('-') start = start + ' ' + times[0] if len(times) > 1: end = end + ' ' + times[1] else: end = start meeting.start = start meeting.end = end elif tdcontent == 'Raum:': meeting.address = " ".join(tds[n + 1].xpath('./text()')) elif tdcontent == 'Bezeichnung:': meeting.description = nextcontent # sense? #if not hasattr(meeting, 'originalId'): # logging.critical('Cannot find session identifier using XPath MEETING_DETAIL_IDENTIFIER_TD') # raise TemplateError('Cannot find session identifier using XPath MEETING_DETAIL_IDENTIFIER_TD') # Agendaitems found_files = [] rows = dom.xpath(self.xpath['MEETING_DETAIL_AGENDAITEM_ROWS']) if len(rows) == 0: no_agendaitem_check = dom.xpath('//h5') no_agendaitem = False for item in no_agendaitem_check: if item.text.strip() == 'Keine Daten gefunden.': no_agendaitem = True if no_agendaitem: logging.warn("Meeting without agendaitems found in %s" % meeting_url) else: logging.critical('Cannot find agenda using XPath MEETING_DETAIL_AGENDAITEM_ROWS at meeting %s' % meeting_url) raise TemplateError('Cannot find agenda using XPath MEETING_DETAIL_AGENDAITEM_ROWS at %s' % meeting_url) meeting.agendaitem = [] else: agendaitems = [] agendaitem_id = None public = True agendaitem = None for row in rows: row_id = row.get('id') row_classes = row.get('class').split(' ') fields = row.xpath('td') number = fields[0].xpath('./text()') if len(number) > 0: number = number[0] else: # when theres a updated notice theres an additional spam number = fields[0].xpath('.//span/text()') if len(number) > 0: number = number[0] if number == []: number = None if row_id is not None: # Agendaitem main row # first: save agendaitem from before if agendaitem: agendaitems.append(agendaitem) # create new agendaitem agendaitem = AgendaItem(originalId=int(row_id.rsplit('_', 1)[1])) if number is not None: agendaitem.number = number # in some ris this is a link, sometimes not. test both. if len(fields[1].xpath('./a/text()')): agendaitem.name = "; ".join(fields[1].xpath('./a/text()')) elif len(fields[1].xpath('./text()')): agendaitem.name = "; ".join(fields[1].xpath('./text()')) # ignore no agendaitem information if agendaitem.name == 'keine Tagesordnungspunkte': agendaitem = None continue agendaitem.public = public # paper links links = row.xpath(self.xpath['MEETING_DETAIL_AGENDAITEM_ROWS_PAPER_LINK']) consultations = [] papers = [] for link in links: href = link.get('href') if href is None: continue # links to papers parsed = parse.search(self.urls['PAPER_DETAIL_PARSE_PATTERN'], href) if parsed is not None: consultation = Consultation(originalId=unicode(agendaitem.originalId) + unicode(parsed['paper_id'])) consultation.paper = Paper(originalId=int(parsed['paper_id'])) #, name=link.text #TODO: da stehen ab und an Zusatzinfos drin. Wohin damit? consultations.append(consultation) # Add paper to paper queue if hasattr(self, 'paper_queue'): self.paper_queue.add(int(parsed['paper_id'])) if len(consultations) == 1: agendaitem.consultation = consultations[0] elif len(consultations) > 1: logging.warn('Multible Papers found in an Agendaitem at %s' % meeting_url) """ Note: we don't scrape agendaitem-related documents for now, based on the assumption that they are all found via paper detail pages. All we do here is get a list of document IDs in found_files """ # find links links = row.xpath('.//a[contains(@href,"getfile.")]') for link in links: if not link.xpath('.//img'): file_link = self.config['scraper']['base_url'] + link.get('href') file_id = file_link.split('id=')[1].split('&')[0] found_files.append(file_id) # find forms forms = row.xpath('.//form') for form in forms: for hidden_field in form.xpath('input'): if hidden_field.get('name') != 'DT': continue file_id = hidden_field.get('value') found_files.append(file_id) # Alternative für smc_tophz wegen Version 4.3.5 bi (Layout 3) elif ('smc_tophz' in row_classes) or (row.get('valign') == 'top' and row.get('debug') == '3'): # additional (optional row for agendaitem) label = fields[1].text value = fields[2].text if label is not None and value is not None: label = label.strip() value = value.strip() if label in ['Ergebnis:', 'Beschluss:', 'Beratungsergebnis:']: new_result_string = '' for result_string in self.config['scraper']['result_strings']: if result_string[0] == value: new_result_string = result_string[1] break if not new_result_string: new_result_string = self.db.save_result_string(value) logging.warn("String '%s' not found in configured RESULT_STRINGS", value) agendaitem.result = new_result_string elif label in ['Bemerkung:', 'Abstimmung:']: agendaitem.result_details = value # What's this? #elif label == 'Abstimmung:': # agendaitems[agendaitem_id]['voting'] = value else: logging.critical("Agendaitem info label '%s' is unknown", label) raise ValueError('Agendaitem info label "%s" is unknown' % label) elif 'smcrowh' in row_classes: # Subheading (public / nonpublic part) if fields[0].text is not None and "Nicht öffentlich" in fields[0].text.encode('utf-8'): public = False meeting.agendaItem = agendaitems # meeting-related documents containers = dom.xpath(self.xpath['MEETING_DETAIL_FILES']) for container in containers: classes = container.get('class') if classes is None: continue classes = classes.split(' ') if self.xpath['MEETING_DETAIL_FILES_CONTAINER_CLASSNAME'] not in classes: continue invitations = [] resultsProtocol = None verbatimProtocol = None auxiliaryFile = [] rows = container.xpath('.//tr') for row in rows: if not row.xpath('.//form'): links = row.xpath('.//a') for link in links: # ignore additional pdf icon links if not link.xpath('.//img'): name = ' '.join(link.xpath('./text()')).strip() file_link = self.config['scraper']['base_url'] + link.get('href') file_id = file_link.split('id=')[1].split('&')[0] if file_id in found_files: continue file = File( originalId=file_id, name=name, originalUrl=file_link, originalDownloadPossible = True) file = self.get_file(file=file, link=file_link) if 'Einladung' in name: invitations.append(file) elif 'Niederschrift' in name: if resultsProtocol: logging.warn('Two resultsProtocols found at %s' % meeting_url) else: resultsProtocol = file else: auxiliaryFile.append(file) found_files.append(file_id) else: forms = row.xpath('.//form') for form in forms: name = " ".join(row.xpath('./td/text()')).strip() for hidden_field in form.xpath('input'): if hidden_field.get('name') != 'DT': continue file_id = hidden_field.get('value') # make sure to add only those which aren't agendaitem-related if file_id not in found_files: file = File( originalId=file_id, name=name, originalDownloadPossible = False ) # Traversing the whole mechanize response to submit this form for mform in mechanize_forms: for control in mform.controls: if control.name == 'DT' and control.value == file_id: file = self.get_file(file, mform) if 'Einladung' in name: invitations.append(file) elif 'Niederschrift' in name: if resultsProtocol: logging.warn('Two resultsProtocols found at %s' % meeting_url) else: resultsProtocol = file else: auxiliaryFile.append(file) found_files.append(file_id) if len(invitations): meeting.invitation = invitations if resultsProtocol: meeting.resultsProtocol = resultsProtocol if verbatimProtocol: meeting.verbatimProtocol = verbatimProtocol if len(auxiliaryFile): meeting.auxiliaryFile = auxiliaryFile oid = self.db.save_meeting(meeting) logging.info("Meeting %d stored with _id %s", meeting_id, oid)
def get_meeting(self, meeting_url=None, meeting_id=None): """ Load meeting details (e.g. agendaitems) for the given detail page URL or numeric ID """ meeting_url = ("%sto010.asp?selfaction=ws&template=xyz&SILFDNR=%s" % (self.config['scraper']['base_url'], meeting_id)) logging.info("Getting meeting %d from %s", meeting_id, meeting_url) r = self.get_url(meeting_url) if not r: return # If r.history has an item we have a problem if len(r.history): if r.history[0].status_code == 302: logging.info("Meeting %d from %s seems to be private", meeting_id, meeting_id) else: logging.error( "Strange redirect %d from %s with status code %s", meeting_id, meeting_url, r.history[0].status_code) return h = HTMLParser.HTMLParser() xml = str(r.text.encode('ascii', 'xmlcharrefreplace')) parser = etree.XMLParser(recover=True) root = etree.fromstring(xml, parser=parser) meeting = Meeting(originalId=meeting_id) # special area special = {} for item in root[0].iterchildren(): special[item.tag] = item.text # Woher kriegen wir das Datum? Nur über die Übersicht? #if 'sisb' in special: #if 'sise' in special: if 'saname' in special: meeting.type = special['saname'] # head area head = {} for item in root[1].iterchildren(): if item.text: head[item.tag] = h.unescape(item.text) else: head[item.text] = '' if 'sitext' in head: meeting.name = head['sitext'] if 'raname' in head: meeting.room = head['raname'] if 'raort' in head: meeting.address = head['raort'] agendaitems = [] for item in root[2].iterchildren(): elem = {} for e in item.iterchildren(): elem[e.tag] = e.text section = [elem['tofnum'], elem['tofunum'], elem['tofuunum']] section = [x for x in section if x != "0"] elem['section'] = ".".join(section) agendaitem = AgendaItem() agendaitem.originalId = int(elem['tolfdnr']) agendaitem.public = (elem['toostLang'] == u'öffentlich') #agendaitem.name = elem['totext1'] # get agenda detail page # TODO: Own Queue time.sleep(self.config['scraper']['wait_time']) agendaitem_url = ( '%sto020.asp?selfaction=ws&template=xyz&TOLFDNR=%s' % (self.config['scraper']['base_url'], agendaitem.originalId)) logging.info("Getting agendaitem %d from %s", agendaitem.originalId, agendaitem_url) agendaitem_r = self.get_url(agendaitem_url) if not agendaitem_r: return if len(agendaitem_r.history): logging.info("Agenda item %d from %s seems to be private", meeting_id, meeting_url) else: agendaitem_xml = agendaitem_r.text.encode( 'ascii', 'xmlcharrefreplace') # TODO: mixup of agendaitem_parser / parser below? agendaitem_parser = etree.XMLParser(recover=True) agendaitem_root = etree.fromstring(agendaitem_xml, parser=parser) add_agenda_item = {} for add_item in agendaitem_root[0].iterchildren(): if add_item.tag == "rtfWP" and len(add_item) > 0: try: agendaitem.resolution_text = h.unescape( etree.tostring(add_item[0][1][0])) except: logging.warn( "Unable to parse resolution text at " "%s", agendaitem_url) else: if add_item.text: add_agenda_item[add_item.tag] = h.unescape( add_item.text) if 'toptext' in add_agenda_item: agendaitem.name = add_agenda_item['toptext'] # there are papers with id = 0. we don't need them. if int(elem['volfdnr']): consult_id = (unicode(agendaitem.originalId) + unicode(int(elem['volfdnr']))) consultation = Consultation(originalId=consult_id) paper_id = int(elem['volfdnr']) if 'voname' in add_agenda_item: consultation.paper = Paper( originalId=paper_id, name=add_agenda_item['voname']) else: consultation.paper = Paper(originalId=paper_id) agendaitem.consultation = [consultation] if 'vobetr' in add_agenda_item: if add_agenda_item['vobetr'] != agendaitem.name: logging.warn( "different values for name: %s and %s", agendaitem.name, add_agenda_item['vobetr']) if hasattr(self, 'paper_queue'): self.paper_queue.add(int(elem['volfdnr'])) if 'totyp' in add_agenda_item: agendaitem.result = add_agenda_item['totyp'] agendaitems.append(agendaitem) meeting.agendaItem = agendaitems oid = self.db.save_meeting(meeting) logging.info("Meeting %d stored with _id %s", meeting_id, oid)
def get_meeting(self, meeting_url=None, meeting_id=None): """ Load meeting details (e.g. agendaitems) for the given detail page URL or numeric ID """ meeting_url = ("%sto010.asp?selfaction=ws&template=xyz&SILFDNR=%s" % (self.config['scraper']['base_url'], meeting_id)) logging.info("Getting meeting %d from %s", meeting_id, meeting_url) r = self.get_url(meeting_url) if not r: return # If r.history has an item we have a problem if len(r.history): if r.history[0].status_code == 302: logging.info("Meeting %d from %s seems to be private", meeting_id, meeting_id) else: logging.error("Strange redirect %d from %s with status code %s", meeting_id, meeting_url, r.history[0].status_code) return h = HTMLParser.HTMLParser() xml = str(r.text.encode('ascii', 'xmlcharrefreplace')) parser = etree.XMLParser(recover=True) root = etree.fromstring(xml, parser=parser) meeting = Meeting(originalId=meeting_id) # special area special = {} for item in root[0].iterchildren(): special[item.tag] = item.text # Woher kriegen wir das Datum? Nur über die Übersicht? #if 'sisb' in special: #if 'sise' in special: if 'saname' in special: meeting.type = special['saname'] # head area head = {} for item in root[1].iterchildren(): if item.text: head[item.tag] = h.unescape(item.text) else: head[item.text] = '' if 'sitext' in head: meeting.name = head['sitext'] if 'raname' in head: meeting.room = head['raname'] if 'raort' in head: meeting.address = head['raort'] agendaitems = [] for item in root[2].iterchildren(): elem = {} for e in item.iterchildren(): elem[e.tag] = e.text section = [elem['tofnum'], elem['tofunum'], elem['tofuunum']] section = [x for x in section if x != "0"] elem['section'] = ".".join(section) agendaitem = AgendaItem() agendaitem.originalId = int(elem['tolfdnr']) agendaitem.public = (elem['toostLang'] == u'öffentlich') #agendaitem.name = elem['totext1'] # get agenda detail page # TODO: Own Queue time.sleep(self.config['scraper']['wait_time']) agendaitem_url = ('%sto020.asp?selfaction=ws&template=xyz&TOLFDNR=%s' % (self.config['scraper']['base_url'], agendaitem.originalId)) logging.info("Getting agendaitem %d from %s", agendaitem.originalId, agendaitem_url) agendaitem_r = self.get_url(agendaitem_url) if not agendaitem_r: return if len(agendaitem_r.history): logging.info("Agenda item %d from %s seems to be private", meeting_id, meeting_url) else: agendaitem_xml = agendaitem_r.text.encode('ascii', 'xmlcharrefreplace') # TODO: mixup of agendaitem_parser / parser below? agendaitem_parser = etree.XMLParser(recover=True) agendaitem_root = etree.fromstring(agendaitem_xml, parser=parser) add_agenda_item = {} for add_item in agendaitem_root[0].iterchildren(): if add_item.tag == "rtfWP" and len(add_item) > 0: try: agendaitem.resolution_text = h.unescape( etree.tostring(add_item[0][1][0])) except: logging.warn("Unable to parse resolution text at " "%s", agendaitem_url) else: if add_item.text: add_agenda_item[add_item.tag] = h.unescape( add_item.text) if 'toptext' in add_agenda_item: agendaitem.name = add_agenda_item['toptext'] # there are papers with id = 0. we don't need them. if int(elem['volfdnr']): consult_id = (unicode(agendaitem.originalId) + unicode(int(elem['volfdnr']))) consultation = Consultation(originalId=consult_id) paper_id = int(elem['volfdnr']) if 'voname' in add_agenda_item: consultation.paper = Paper( originalId=paper_id, name=add_agenda_item['voname']) else: consultation.paper = Paper(originalId=paper_id) agendaitem.consultation = [consultation] if 'vobetr' in add_agenda_item: if add_agenda_item['vobetr'] != agendaitem.name: logging.warn("different values for name: %s and %s", agendaitem.name, add_agenda_item['vobetr']) if hasattr(self, 'paper_queue'): self.paper_queue.add(int(elem['volfdnr'])) if 'totyp' in add_agenda_item: agendaitem.result = add_agenda_item['totyp'] agendaitems.append(agendaitem) meeting.agendaItem = agendaitems oid = self.db.save_meeting(meeting) logging.info("Meeting %d stored with _id %s", meeting_id, oid)