def find_meeting(self, start_date=None, end_date=None):
""" Find meetings within a given time frame and add them to the meeting
queue.
"""
meeting_find_url = (
self.config['scraper']['allris']['meeting_find_url'] %
(self.config['scraper']['base_url'],
start_date.strftime("%d.%m.%Y"), end_date.strftime("%d.%m.%Y")))
logging.info("Getting meeting overview from %s", meeting_find_url)
parser = etree.XMLParser(recover=True)
h = HTMLParser.HTMLParser()
r = self.get_url(meeting_find_url)
if not r:
return
xml = r.text.encode('ascii', 'xmlcharrefreplace').replace('</a>', '')
xml = re.sub(r'<a href="([^"]*)" target="_blank" ?>', r'\1', xml)
root = etree.fromstring(xml, parser=parser)
for item in root:
if item.tag == 'list':
root = item
break
for item in root.iterchildren():
raw_meeting = {}
for e in item.iterchildren():
if e.text:
raw_meeting[e.tag] = h.unescape(e.text)
else:
raw_meeting[e.tag] = ''
meeting = Meeting(originalId=int(raw_meeting['silfdnr']))
meeting.start = self.parse_date(raw_meeting['sisbvcs'])
meeting.end = self.parse_date(raw_meeting['sisevcs'])
meeting.name = raw_meeting['siname']
meeting.originalUrl = (
"%sto010.asp?SILFDNR=%s&options=4" %
(self.config['scraper']['base_url'], raw_meeting['silfdnr']))
meeting.name = raw_meeting['sitext']
meeting.organization_name = raw_meeting['grname']
# meeting.description = raw_meeting['sitext'] # WHAT TO DO WITH THIS
self.db.save_meeting(meeting)
self.meeting_queue.add(meeting.originalId)
def find_meeting(self, start_date=None, end_date=None):
""" Find meetings within a given time frame and add them to the meeting
queue.
"""
meeting_find_url = (self.config['scraper']['allris']['meeting_find_url']
% (self.config['scraper']['base_url'],
start_date.strftime("%d.%m.%Y"),
end_date.strftime("%d.%m.%Y")))
logging.info("Getting meeting overview from %s", meeting_find_url)
parser = etree.XMLParser(recover=True)
h = HTMLParser.HTMLParser()
r = self.get_url(meeting_find_url)
if not r:
return
xml = r.text.encode('ascii', 'xmlcharrefreplace').replace('</a>', '')
xml = re.sub(r'<a href="([^"]*)" target="_blank" ?>', r'\1', xml)
root = etree.fromstring(xml, parser=parser)
for item in root:
if item.tag == 'list':
root = item
break
for item in root.iterchildren():
raw_meeting = {}
for e in item.iterchildren():
if e.text:
raw_meeting[e.tag] = h.unescape(e.text)
else:
raw_meeting[e.tag] = ''
meeting = Meeting(originalId=int(raw_meeting['silfdnr']))
meeting.start = self.parse_date(raw_meeting['sisbvcs'])
meeting.end = self.parse_date(raw_meeting['sisevcs'])
meeting.name = raw_meeting['siname']
meeting.originalUrl = ("%sto010.asp?SILFDNR=%s&options=4"
% (self.config['scraper']['base_url'],
raw_meeting['silfdnr']))
meeting.name = raw_meeting['sitext']
meeting.organization_name = raw_meeting['grname']
# meeting.description = raw_meeting['sitext'] # WHAT TO DO WITH THIS
self.db.save_meeting(meeting)
self.meeting_queue.add(meeting.originalId)
def get_meeting(self, meeting_url=None, meeting_id=None):
"""
Load meeting details for the given detail page URL or numeric ID
"""
# Read either meeting_id or meeting_url from the opposite
if meeting_id is not None:
meeting_url = self.urls['MEETING_DETAIL_PRINT_PATTERN'] % (self.config["scraper"]["base_url"], meeting_id)
elif meeting_url is not None:
parsed = parse.search(self.urls['MEETING_DETAIL_PARSE_PATTERN'], meeting_url)
meeting_id = parsed['meeting_id']
logging.info("Getting meeting (session) %d from %s", meeting_id, meeting_url)
meeting = Meeting(originalId=meeting_id)
time.sleep(self.config['scraper']['wait_time'])
response = self.get_url(meeting_url)
if not response:
return
# forms for later document download
mechanize_forms = mechanize.ParseResponse(response, backwards_compat=False)
# seek(0) is necessary to reset response pointer.
response.seek(0)
html = response.read()
html = html.replace(' ', ' ')
parser = etree.HTMLParser()
dom = etree.parse(StringIO(html), parser)
# check for page errors
try:
page_title = dom.xpath('//h1')[0].text
if 'Fehlermeldung' in page_title:
logging.info("Page %s cannot be accessed due to server error", meeting_url)
return
if 'Berechtigungsfehler' in page_title:
logging.info("Page %s cannot be accessed due to permissions", meeting_url)
return
except:
pass
try:
error_h3 = dom.xpath('//h3[@class="smc_h3"]')[0].text.strip()
if 'Keine Daten gefunden' in error_h3:
logging.info("Page %s does not contain any agenda items", meeting_url)
return
if 'Fehlercode: 1104' in error_h3:
logging.info("Page %s cannot be accessed due to permissions", meeting_url)
return
except:
pass
meeting.originalUrl = meeting_url
# Session title
try:
meeting.name = dom.xpath(self.xpath['MEETING_DETAIL_TITLE'])[0].text
except:
logging.critical('Cannot find session title element using XPath MEETING_DETAIL_TITLE')
raise TemplateError('Cannot find session title element using XPath MEETING_DETAIL_TITLE')
# Committe link
#try:
# links = dom.xpath(self.xpath['MEETING_DETAIL_COMMITTEE_LINK'])
# for link in links:
# href = link.get('href')
# parsed = parse.search(self.urls['COMMITTEE_DETAIL_PARSE_PATTERN'], href)
# if parsed is not None:
# meeting.committees = [Commitee(numeric_id=int(parsed['committee_id']))]
# if hasattr(self, 'committee_queue'):
# self.committee_queue.add(int(parsed['committee_id']))
#except:
# logging.critical('Cannot find link to committee detail page using MEETING_DETAIL_COMMITTEE_LINK_XPATH')
# raise TemplateError('Cannot find link to committee detail page using MEETING_DETAIL_COMMITTEE_LINK_XPATH')
# Meeting identifier, date, address etc
tds = dom.xpath(self.xpath['MEETING_DETAIL_IDENTIFIER_TD'])
if len(tds) == 0:
logging.critical('Cannot find table fields using MEETING_DETAIL_IDENTIFIER_TD_XPATH at session ' + meeting_url)
raise TemplateError('Cannot find table fields using MEETING_DETAIL_IDENTIFIER_TD_XPATH at session ' + meeting_url)
else:
for n in range(0, len(tds)):
try:
tdcontent = tds[n].text.strip()
nextcontent = tds[n + 1].text.strip()
except:
continue
if tdcontent == 'Sitzung:':
meeting.shortName = nextcontent
# We don't need this any more because it's scraped in committee detail page(?)
#elif tdcontent == 'Gremium:':
# meeting.committee_name = nextcontent
elif tdcontent == 'Datum:':
start = nextcontent
end = nextcontent
if tds[n + 2].text == 'Zeit:':
if tds[n + 3].text is not None:
times = tds[n + 3].text.replace(' Uhr', '').split('-')
start = start + ' ' + times[0]
if len(times) > 1:
end = end + ' ' + times[1]
else:
end = start
meeting.start = start
meeting.end = end
elif tdcontent == 'Raum:':
meeting.address = " ".join(tds[n + 1].xpath('./text()'))
elif tdcontent == 'Bezeichnung:':
meeting.description = nextcontent
# sense?
#if not hasattr(meeting, 'originalId'):
# logging.critical('Cannot find session identifier using XPath MEETING_DETAIL_IDENTIFIER_TD')
# raise TemplateError('Cannot find session identifier using XPath MEETING_DETAIL_IDENTIFIER_TD')
# Agendaitems
found_files = []
rows = dom.xpath(self.xpath['MEETING_DETAIL_AGENDAITEM_ROWS'])
if len(rows) == 0:
no_agendaitem_check = dom.xpath('//h5')
no_agendaitem = False
for item in no_agendaitem_check:
if item.text.strip() == 'Keine Daten gefunden.':
no_agendaitem = True
if no_agendaitem:
logging.warn("Meeting without agendaitems found in %s" % meeting_url)
else:
logging.critical('Cannot find agenda using XPath MEETING_DETAIL_AGENDAITEM_ROWS at meeting %s' % meeting_url)
raise TemplateError('Cannot find agenda using XPath MEETING_DETAIL_AGENDAITEM_ROWS at %s' % meeting_url)
meeting.agendaitem = []
else:
agendaitems = []
agendaitem_id = None
public = True
agendaitem = None
for row in rows:
row_id = row.get('id')
row_classes = row.get('class').split(' ')
fields = row.xpath('td')
number = fields[0].xpath('./text()')
if len(number) > 0:
number = number[0]
else:
# when theres a updated notice theres an additional spam
number = fields[0].xpath('.//span/text()')
if len(number) > 0:
number = number[0]
if number == []:
number = None
if row_id is not None:
# Agendaitem main row
# first: save agendaitem from before
if agendaitem:
agendaitems.append(agendaitem)
# create new agendaitem
agendaitem = AgendaItem(originalId=int(row_id.rsplit('_', 1)[1]))
if number is not None:
agendaitem.number = number
# in some ris this is a link, sometimes not. test both.
if len(fields[1].xpath('./a/text()')):
agendaitem.name = "; ".join(fields[1].xpath('./a/text()'))
elif len(fields[1].xpath('./text()')):
agendaitem.name = "; ".join(fields[1].xpath('./text()'))
# ignore no agendaitem information
if agendaitem.name == 'keine Tagesordnungspunkte':
agendaitem = None
continue
agendaitem.public = public
# paper links
links = row.xpath(self.xpath['MEETING_DETAIL_AGENDAITEM_ROWS_PAPER_LINK'])
consultations = []
papers = []
for link in links:
href = link.get('href')
if href is None:
continue
# links to papers
parsed = parse.search(self.urls['PAPER_DETAIL_PARSE_PATTERN'], href)
if parsed is not None:
consultation = Consultation(originalId=unicode(agendaitem.originalId) + unicode(parsed['paper_id']))
consultation.paper = Paper(originalId=int(parsed['paper_id'])) #, name=link.text #TODO: da stehen ab und an Zusatzinfos drin. Wohin damit?
consultations.append(consultation)
# Add paper to paper queue
if hasattr(self, 'paper_queue'):
self.paper_queue.add(int(parsed['paper_id']))
if len(consultations) == 1:
agendaitem.consultation = consultations[0]
elif len(consultations) > 1:
logging.warn('Multible Papers found in an Agendaitem at %s' % meeting_url)
"""
Note: we don't scrape agendaitem-related documents for now,
based on the assumption that they are all found via paper
detail pages. All we do here is get a list of document IDs
in found_files
"""
# find links
links = row.xpath('.//a[contains(@href,"getfile.")]')
for link in links:
if not link.xpath('.//img'):
file_link = self.config['scraper']['base_url'] + link.get('href')
file_id = file_link.split('id=')[1].split('&')[0]
found_files.append(file_id)
# find forms
forms = row.xpath('.//form')
for form in forms:
for hidden_field in form.xpath('input'):
if hidden_field.get('name') != 'DT':
continue
file_id = hidden_field.get('value')
found_files.append(file_id)
# Alternative für smc_tophz wegen Version 4.3.5 bi (Layout 3)
elif ('smc_tophz' in row_classes) or (row.get('valign') == 'top' and row.get('debug') == '3'):
# additional (optional row for agendaitem)
label = fields[1].text
value = fields[2].text
if label is not None and value is not None:
label = label.strip()
value = value.strip()
if label in ['Ergebnis:', 'Beschluss:', 'Beratungsergebnis:']:
new_result_string = ''
for result_string in self.config['scraper']['result_strings']:
if result_string[0] == value:
new_result_string = result_string[1]
break
if not new_result_string:
new_result_string = self.db.save_result_string(value)
logging.warn("String '%s' not found in configured RESULT_STRINGS", value)
agendaitem.result = new_result_string
elif label in ['Bemerkung:', 'Abstimmung:']:
agendaitem.result_details = value
# What's this?
#elif label == 'Abstimmung:':
# agendaitems[agendaitem_id]['voting'] = value
else:
logging.critical("Agendaitem info label '%s' is unknown", label)
raise ValueError('Agendaitem info label "%s" is unknown' % label)
elif 'smcrowh' in row_classes:
# Subheading (public / nonpublic part)
if fields[0].text is not None and "Nicht öffentlich" in fields[0].text.encode('utf-8'):
public = False
meeting.agendaItem = agendaitems
# meeting-related documents
containers = dom.xpath(self.xpath['MEETING_DETAIL_FILES'])
for container in containers:
classes = container.get('class')
if classes is None:
continue
classes = classes.split(' ')
if self.xpath['MEETING_DETAIL_FILES_CONTAINER_CLASSNAME'] not in classes:
continue
invitations = []
resultsProtocol = None
verbatimProtocol = None
auxiliaryFile = []
rows = container.xpath('.//tr')
for row in rows:
if not row.xpath('.//form'):
links = row.xpath('.//a')
for link in links:
# ignore additional pdf icon links
if not link.xpath('.//img'):
name = ' '.join(link.xpath('./text()')).strip()
file_link = self.config['scraper']['base_url'] + link.get('href')
file_id = file_link.split('id=')[1].split('&')[0]
if file_id in found_files:
continue
file = File(
originalId=file_id,
name=name,
originalUrl=file_link,
originalDownloadPossible = True)
file = self.get_file(file=file, link=file_link)
if 'Einladung' in name:
invitations.append(file)
elif 'Niederschrift' in name:
if resultsProtocol:
logging.warn('Two resultsProtocols found at %s' % meeting_url)
else:
resultsProtocol = file
else:
auxiliaryFile.append(file)
found_files.append(file_id)
else:
forms = row.xpath('.//form')
for form in forms:
name = " ".join(row.xpath('./td/text()')).strip()
for hidden_field in form.xpath('input'):
if hidden_field.get('name') != 'DT':
continue
file_id = hidden_field.get('value')
# make sure to add only those which aren't agendaitem-related
if file_id not in found_files:
file = File(
originalId=file_id,
name=name,
originalDownloadPossible = False
)
# Traversing the whole mechanize response to submit this form
for mform in mechanize_forms:
for control in mform.controls:
if control.name == 'DT' and control.value == file_id:
file = self.get_file(file, mform)
if 'Einladung' in name:
invitations.append(file)
elif 'Niederschrift' in name:
if resultsProtocol:
logging.warn('Two resultsProtocols found at %s' % meeting_url)
else:
resultsProtocol = file
else:
auxiliaryFile.append(file)
found_files.append(file_id)
if len(invitations):
meeting.invitation = invitations
if resultsProtocol:
meeting.resultsProtocol = resultsProtocol
if verbatimProtocol:
meeting.verbatimProtocol = verbatimProtocol
if len(auxiliaryFile):
meeting.auxiliaryFile = auxiliaryFile
oid = self.db.save_meeting(meeting)
logging.info("Meeting %d stored with _id %s", meeting_id, oid)
def get_meeting(self, meeting_url=None, meeting_id=None):
""" Load meeting details (e.g. agendaitems) for the given detail page
URL or numeric ID
"""
meeting_url = ("%sto010.asp?selfaction=ws&template=xyz&SILFDNR=%s" %
(self.config['scraper']['base_url'], meeting_id))
logging.info("Getting meeting %d from %s", meeting_id, meeting_url)
r = self.get_url(meeting_url)
if not r:
return
# If r.history has an item we have a problem
if len(r.history):
if r.history[0].status_code == 302:
logging.info("Meeting %d from %s seems to be private",
meeting_id, meeting_id)
else:
logging.error(
"Strange redirect %d from %s with status code %s",
meeting_id, meeting_url, r.history[0].status_code)
return
h = HTMLParser.HTMLParser()
xml = str(r.text.encode('ascii', 'xmlcharrefreplace'))
parser = etree.XMLParser(recover=True)
root = etree.fromstring(xml, parser=parser)
meeting = Meeting(originalId=meeting_id)
# special area
special = {}
for item in root[0].iterchildren():
special[item.tag] = item.text
# Woher kriegen wir das Datum? Nur über die Übersicht?
#if 'sisb' in special:
#if 'sise' in special:
if 'saname' in special:
meeting.type = special['saname']
# head area
head = {}
for item in root[1].iterchildren():
if item.text:
head[item.tag] = h.unescape(item.text)
else:
head[item.text] = ''
if 'sitext' in head:
meeting.name = head['sitext']
if 'raname' in head:
meeting.room = head['raname']
if 'raort' in head:
meeting.address = head['raort']
agendaitems = []
for item in root[2].iterchildren():
elem = {}
for e in item.iterchildren():
elem[e.tag] = e.text
section = [elem['tofnum'], elem['tofunum'], elem['tofuunum']]
section = [x for x in section if x != "0"]
elem['section'] = ".".join(section)
agendaitem = AgendaItem()
agendaitem.originalId = int(elem['tolfdnr'])
agendaitem.public = (elem['toostLang'] == u'öffentlich')
#agendaitem.name = elem['totext1']
# get agenda detail page
# TODO: Own Queue
time.sleep(self.config['scraper']['wait_time'])
agendaitem_url = (
'%sto020.asp?selfaction=ws&template=xyz&TOLFDNR=%s' %
(self.config['scraper']['base_url'], agendaitem.originalId))
logging.info("Getting agendaitem %d from %s",
agendaitem.originalId, agendaitem_url)
agendaitem_r = self.get_url(agendaitem_url)
if not agendaitem_r:
return
if len(agendaitem_r.history):
logging.info("Agenda item %d from %s seems to be private",
meeting_id, meeting_url)
else:
agendaitem_xml = agendaitem_r.text.encode(
'ascii', 'xmlcharrefreplace')
# TODO: mixup of agendaitem_parser / parser below?
agendaitem_parser = etree.XMLParser(recover=True)
agendaitem_root = etree.fromstring(agendaitem_xml,
parser=parser)
add_agenda_item = {}
for add_item in agendaitem_root[0].iterchildren():
if add_item.tag == "rtfWP" and len(add_item) > 0:
try:
agendaitem.resolution_text = h.unescape(
etree.tostring(add_item[0][1][0]))
except:
logging.warn(
"Unable to parse resolution text at "
"%s", agendaitem_url)
else:
if add_item.text:
add_agenda_item[add_item.tag] = h.unescape(
add_item.text)
if 'toptext' in add_agenda_item:
agendaitem.name = add_agenda_item['toptext']
# there are papers with id = 0. we don't need them.
if int(elem['volfdnr']):
consult_id = (unicode(agendaitem.originalId) +
unicode(int(elem['volfdnr'])))
consultation = Consultation(originalId=consult_id)
paper_id = int(elem['volfdnr'])
if 'voname' in add_agenda_item:
consultation.paper = Paper(
originalId=paper_id,
name=add_agenda_item['voname'])
else:
consultation.paper = Paper(originalId=paper_id)
agendaitem.consultation = [consultation]
if 'vobetr' in add_agenda_item:
if add_agenda_item['vobetr'] != agendaitem.name:
logging.warn(
"different values for name: %s and %s",
agendaitem.name, add_agenda_item['vobetr'])
if hasattr(self, 'paper_queue'):
self.paper_queue.add(int(elem['volfdnr']))
if 'totyp' in add_agenda_item:
agendaitem.result = add_agenda_item['totyp']
agendaitems.append(agendaitem)
meeting.agendaItem = agendaitems
oid = self.db.save_meeting(meeting)
logging.info("Meeting %d stored with _id %s", meeting_id, oid)
def get_meeting(self, meeting_url=None, meeting_id=None):
""" Load meeting details (e.g. agendaitems) for the given detail page
URL or numeric ID
"""
meeting_url = ("%sto010.asp?selfaction=ws&template=xyz&SILFDNR=%s"
% (self.config['scraper']['base_url'], meeting_id))
logging.info("Getting meeting %d from %s", meeting_id, meeting_url)
r = self.get_url(meeting_url)
if not r:
return
# If r.history has an item we have a problem
if len(r.history):
if r.history[0].status_code == 302:
logging.info("Meeting %d from %s seems to be private",
meeting_id, meeting_id)
else:
logging.error("Strange redirect %d from %s with status code %s",
meeting_id, meeting_url, r.history[0].status_code)
return
h = HTMLParser.HTMLParser()
xml = str(r.text.encode('ascii', 'xmlcharrefreplace'))
parser = etree.XMLParser(recover=True)
root = etree.fromstring(xml, parser=parser)
meeting = Meeting(originalId=meeting_id)
# special area
special = {}
for item in root[0].iterchildren():
special[item.tag] = item.text
# Woher kriegen wir das Datum? Nur über die Übersicht?
#if 'sisb' in special:
#if 'sise' in special:
if 'saname' in special:
meeting.type = special['saname']
# head area
head = {}
for item in root[1].iterchildren():
if item.text:
head[item.tag] = h.unescape(item.text)
else:
head[item.text] = ''
if 'sitext' in head:
meeting.name = head['sitext']
if 'raname' in head:
meeting.room = head['raname']
if 'raort' in head:
meeting.address = head['raort']
agendaitems = []
for item in root[2].iterchildren():
elem = {}
for e in item.iterchildren():
elem[e.tag] = e.text
section = [elem['tofnum'], elem['tofunum'], elem['tofuunum']]
section = [x for x in section if x != "0"]
elem['section'] = ".".join(section)
agendaitem = AgendaItem()
agendaitem.originalId = int(elem['tolfdnr'])
agendaitem.public = (elem['toostLang'] == u'öffentlich')
#agendaitem.name = elem['totext1']
# get agenda detail page
# TODO: Own Queue
time.sleep(self.config['scraper']['wait_time'])
agendaitem_url = ('%sto020.asp?selfaction=ws&template=xyz&TOLFDNR=%s'
% (self.config['scraper']['base_url'],
agendaitem.originalId))
logging.info("Getting agendaitem %d from %s",
agendaitem.originalId, agendaitem_url)
agendaitem_r = self.get_url(agendaitem_url)
if not agendaitem_r:
return
if len(agendaitem_r.history):
logging.info("Agenda item %d from %s seems to be private",
meeting_id, meeting_url)
else:
agendaitem_xml = agendaitem_r.text.encode('ascii',
'xmlcharrefreplace')
# TODO: mixup of agendaitem_parser / parser below?
agendaitem_parser = etree.XMLParser(recover=True)
agendaitem_root = etree.fromstring(agendaitem_xml,
parser=parser)
add_agenda_item = {}
for add_item in agendaitem_root[0].iterchildren():
if add_item.tag == "rtfWP" and len(add_item) > 0:
try:
agendaitem.resolution_text = h.unescape(
etree.tostring(add_item[0][1][0]))
except:
logging.warn("Unable to parse resolution text at "
"%s", agendaitem_url)
else:
if add_item.text:
add_agenda_item[add_item.tag] = h.unescape(
add_item.text)
if 'toptext' in add_agenda_item:
agendaitem.name = add_agenda_item['toptext']
# there are papers with id = 0. we don't need them.
if int(elem['volfdnr']):
consult_id = (unicode(agendaitem.originalId)
+ unicode(int(elem['volfdnr'])))
consultation = Consultation(originalId=consult_id)
paper_id = int(elem['volfdnr'])
if 'voname' in add_agenda_item:
consultation.paper = Paper(
originalId=paper_id, name=add_agenda_item['voname'])
else:
consultation.paper = Paper(originalId=paper_id)
agendaitem.consultation = [consultation]
if 'vobetr' in add_agenda_item:
if add_agenda_item['vobetr'] != agendaitem.name:
logging.warn("different values for name: %s and %s",
agendaitem.name,
add_agenda_item['vobetr'])
if hasattr(self, 'paper_queue'):
self.paper_queue.add(int(elem['volfdnr']))
if 'totyp' in add_agenda_item:
agendaitem.result = add_agenda_item['totyp']
agendaitems.append(agendaitem)
meeting.agendaItem = agendaitems
oid = self.db.save_meeting(meeting)
logging.info("Meeting %d stored with _id %s", meeting_id, oid)
