class Spider(scrapy.Spider):
name = "meeting"
allowed_domains = ["tncc.gov.tw"]
start_urls = ["http://www.tncc.gov.tw",]
download_delay = 0.5
county_abbr = os.path.dirname(os.path.realpath(__file__)).split('/')[-1]
election_year = common.election_year(county_abbr)
output_path = common.meeting_minutes_output_path(county_abbr, election_year)
ads = {'2010': 1, '2014': 2, '2018': 3}
ad = ads[election_year]
def parse(self, response):
return response.follow(response.xpath(u'//a[re:test(., "^出版品$")]/@href').extract_first(), callback=self.parse_list)
def parse_list(self, response):
for tr in response.css('#table2 tr'):
link = tr.xpath(u'descendant::a[re:test(., "^第%d屆")]/@href' % self.ad).extract_first()
if link:
item = {}
item['election_year'] = self.election_year
item['date'] = common.ROC2AD(tr.xpath('td[1]/text()').extract_first())
item['meeting'] = tr.xpath('td[3]/descendant::a/text()').extract_first()
item['meeting'] = item['meeting'].replace('.', u'、')
item['download_url'] = urljoin(response.url, link)
ext = item['download_url'].split('.')[-1]
file_name = '%s.%s' % (item['meeting'], ext)
cmd = 'mkdir -p %s && wget --no-check-certificate -c -O %s%s "%s"' % (self.output_path, self.output_path, file_name, item['download_url'])
retcode = subprocess.call(cmd, shell=True)
yield item
class Spider(scrapy.Spider):
name = "meeting"
allowed_domains = ["www.tycc.gov.tw"]
start_urls = [
"http://www.tycc.gov.tw/content/public/public_main.aspx?wtp=1&wnd=217",
]
download_delay = 0.5
county_abbr = os.path.dirname(os.path.realpath(__file__)).split('/')[-1]
election_year = common.election_year(county_abbr)
output_path = common.meeting_minutes_output_path(county_abbr,
election_year)
election_years_ad = {'2014': '1', '2010': '17'}
ad = election_years_ad[election_year]
def parse(self, response):
nodes = response.xpath(
u'//tr/td[re:test(@title, "第%s屆")]/following-sibling::td/a[re:test(., "會$")]'
% self.ad)
for node in nodes:
item = {}
item['election_year'] = self.election_year
item['download_url'] = urljoin(
response.url,
node.xpath('@href').extract_first().strip())
item['sitting'] = u'第%s屆' % self.ad
item['meeting'] = node.xpath('descendant::*/text()').re(
u'屆(.+會)$')[0]
item['meeting'] = item['meeting'].replace('.', u'、')
ext = node.xpath('@href').extract_first().split('.')[-1]
file_name = '%s_%s.%s' % (item['sitting'], item['meeting'], ext)
cmd = 'mkdir -p %s && wget -c -O %s%s "%s"' % (
self.output_path, self.output_path, file_name,
item['download_url'])
retcode = subprocess.call(cmd, shell=True)
yield item
nodes = response.xpath(
u'//tr/td[re:test(@title, "第%s屆")]/following-sibling::td/a[re:test(., "(冊|pdf)$")]'
% self.ad)
for node in nodes:
item = {}
item['election_year'] = self.election_year
item['download_url'] = urljoin(
response.url,
node.xpath('@href').extract_first().strip())
item['sitting'] = u'第%s屆' % self.ad
item['meeting'] = '%s%s' % (
node.xpath('preceding::td[1]/text()').re(u'屆(.+會)')[0],
node.xpath('descendant::*/text()').re(u'(.+冊)')[0])
item['meeting'] = item['meeting'].replace('.', u'、')
ext = node.xpath('@href').extract_first().split('.')[-1]
file_name = '%s_%s.%s' % (item['sitting'], item['meeting'], ext)
cmd = 'mkdir -p %s && wget -c -O %s%s "%s"' % (
self.output_path, self.output_path, file_name,
item['download_url'])
retcode = subprocess.call(cmd, shell=True)
yield item
class Spider(scrapy.Spider):
name = "meeting"
allowed_domains = ["cissearch.kcc.gov.tw"]
start_urls = [
"http://cissearch.kcc.gov.tw/System/MeetingRecord/Default.aspx",
]
download_delay = 0.5
county_abbr = os.path.dirname(os.path.realpath(__file__)).split('/')[-1]
election_year = common.election_year(county_abbr)
output_path = common.meeting_minutes_output_path(county_abbr,
election_year)
def parse(self, response):
count = response.xpath(
'//span[@id="ContentPlaceHolder1_DataPager1"]/text()').re(
u'共\s*(\d+)\s*筆')[0]
payload = {
'ctl00$ContentPlaceHolder1$DataPager1$ctl02$txtPageSize': count
}
yield scrapy.FormRequest.from_response(response,
response.url,
formdata=payload,
callback=self.parse_profile,
dont_filter=True)
def parse_profile(self, response):
trs = response.xpath('//table[@id="ContentPlaceHolder1_gvIndex"]/tr')
for tr in trs:
item = {}
tds = tr.xpath('td')
if tds:
item['election_year'] = self.election_year
item['date'] = common.ROC2AD(
tds[1].xpath('text()').extract_first())
meeting = tds[2].xpath('text()').extract_first()
item['meeting'] = tds[2].xpath('text()').re(
u'(.+?)[紀記][錄錄]')[0]
item['download_url'] = urljoin(
response.url,
tds[3].xpath('a/@href').extract_first().strip())
ext = item['download_url'].split('.')[-1]
file_name = '%s.%s' % (item['meeting'], ext)
cmd = 'mkdir -p %s && wget -c -O %s%s "%s"' % (
self.output_path, self.output_path, file_name,
item['download_url'])
retcode = subprocess.call(cmd, shell=True)
time.sleep(1)
yield item
class Spider(scrapy.Spider):
name = "meeting"
allowed_domains = ["www.kmc.gov.tw", "ebook.21cms.tw"]
start_urls = ["http://www.kmc.gov.tw/recorder",]
download_delay = 0.5
county_abbr = os.path.dirname(os.path.realpath(__file__)).split('/')[-1]
election_year = common.election_year(county_abbr)
output_path = common.meeting_minutes_output_path(county_abbr, election_year)
ads = {
'2014': u'第十八屆',
'2009': u'第十七屆'
}
ad = ads[election_year]
def parse(self, response):
nodes = response.css('.panel-body').xpath(u'descendant::a[re:test(., "%s")]' % self.ad)
for node in nodes:
link = node.xpath('@href').extract_first()
item = {}
item['election_year'] = self.election_year
item['sitting'] = node.xpath('text()').extract_first().replace(u'(點擊閱讀)', '').replace('>>', '')
item['download_url'] = urljoin(response.url, link)
if re.search('/ebook/', link):
file_name = '%s.pdf' % (item['sitting'], )
cmd = 'mkdir -p %s && wget -A pdf -nd -r --no-parent -O "%s%s" "%s"' % (self.output_path, self.output_path, file_name, urljoin(response.url, link))
retcode = subprocess.call(cmd, shell=True)
yield item
else:
yield response.follow(link, callback=self.parse_iframe, meta={'item': item})
def parse_iframe(self, response):
link = response.css('.article-content iframe').xpath('@src').extract_first()
item = response.meta['item']
file_name = '%s.pdf' % (item['sitting'], )
cmd = 'mkdir -p %s && wget -A pdf -nd -r --no-parent -O "%s%s" "%s"' % (self.output_path, self.output_path, file_name, link)
retcode = subprocess.call(cmd, shell=True)
yield item
class Spider(scrapy.Spider):
name = "meeting"
allowed_domains = ["www.ilcc.gov.tw"]
start_urls = [
"http://www.ilcc.gov.tw/Html/H_06/H_06.asp",
]
download_delay = 0.5
county_abbr = os.path.dirname(os.path.realpath(__file__)).split('/')[-1]
election_year = common.election_year(county_abbr)
output_path = common.meeting_minutes_output_path(county_abbr,
election_year)
payload = {
'ddlcounciltype': u'大會',
}
def parse(self, response):
return response.follow(
response.xpath(u'//area[@alt="會議紀錄"]/@href').extract_first(),
callback=self.parse_frame)
def parse_frame(self, response):
return response.follow(
response.xpath('//frame[@id="FrMain"]/@src').extract_first(),
callback=self.parse_meeting_info)
def parse_meeting_info(self, response):
return scrapy.FormRequest.from_response(response,
response.url,
formdata=self.payload,
callback=self.parse_pages)
def parse_pages(self, response):
pages = response.xpath(
'//select[@name="page"]/option/@value').extract()
for page in pages:
yield scrapy.FormRequest.from_response(response,
response.url,
formdata={
'page': page,
'btSearch': None
},
callback=self.parse_post)
def parse_post(self, response):
trs = response.xpath('//table[@id="dg"]/descendant::tr[position()>1]')
for tr in trs:
item = {}
item['election_year'] = self.election_year
item['date'] = re.sub('\s', '',
tr.xpath('string(td[1])').extract_first())
item['sitting'] = re.sub(
'\s', '', '%s%s' % (tr.xpath('string(td[2])').extract_first(),
tr.xpath('string(td[3])').extract_first()))
item['meeting'] = re.sub('\s', '',
tr.xpath('string(td[5])').extract_first())
yield response.follow(
tr.xpath('td[4]/descendant::a/@href').extract_first(),
callback=self.parse_profile,
meta={'item': item})
def parse_profile(self, response):
item = response.meta['item']
item['download_url'] = response.xpath(
'//td/a[@target="_blank"]/@href').extract_first()
if item['download_url']:
ext = re.search(u'\.(\w+)$', item['download_url']).group(1)
file_name = '%s_%s.%s' % (item['sitting'], item['meeting'], ext)
cmd = 'mkdir -p %s && wget -c -O %s%s "%s"' % (
self.output_path, self.output_path, file_name,
item['download_url'])
retcode = subprocess.call(cmd, shell=True)
else:
logging.error(response.url)
return item
class Spider(scrapy.Spider):
name = "meeting"
allowed_domains = ["www.ntp.gov.tw"]
start_urls = [
'https://www.ntp.gov.tw/content/information/information04.aspx'
]
download_delay = 1
county_abbr = os.path.dirname(os.path.realpath(__file__)).split('/')[-1]
election_year = common.election_year(county_abbr)
output_path = common.meeting_minutes_output_path(county_abbr,
election_year)
def parse(self, response):
for node in response.xpath(
u'//a[contains(@title, "HTML檔")]/@href').extract():
yield response.follow(node, callback=self.parse_sitting)
def parse_sitting(self, response):
for node in response.xpath(u'//td/descendant::a/@href').extract():
yield response.follow(node, callback=self.parse_meeting)
def parse_meeting(self, response):
try:
sitting = response.xpath('//text()').re(u'(.+)日程表')[0]
trs = [
tr for tr in response.xpath('//table/descendant::tr')
if tr.xpath('td[3]/text()').re('\d+')
]
for tr in trs:
item = {}
item['election_year'] = self.election_year
item['date'] = common.ROC2AD(
tr.xpath('td[1]/text()').extract_first())
item['sitting'] = sitting
item['meeting'] = tr.xpath('td[3]/text()').extract_first()
item['download_url'] = tr.xpath(
'td[6]/descendant::a[1]/@href').extract_first()
ext = item['download_url'].split('.')[-1]
file_name = '%s_%s.%s' % (item['sitting'], item['meeting'],
ext)
if ext == 'pdf':
yield response.follow(item['download_url'],
callback=self.download_pdf,
meta={
'item': item,
'file_name': file_name
})
elif ext == 'htm':
yield response.follow(item['download_url'],
callback=self.parse_html,
meta={
'item': item,
'file_name': file_name
})
except scrapy.exceptions.NotSupported:
pass
def download_pdf(self, response):
item = response.meta['item']
item['download_url'] = response.url
cmd = 'mkdir -p %s && wget --no-check-certificate -c -O %s%s "%s"' % (
self.output_path, self.output_path, response.meta['file_name'],
item['download_url'])
retcode = subprocess.call(cmd, shell=True)
return item
def parse_html(self, response):
item = response.meta['item']
item['download_url'] = response.url
text = '\n'.join(response.xpath('//pre/text()').extract())
write_file(
text, '%s%s_%s.txt' %
(self.output_path, item['sitting'], item['meeting']))
return item
class Spider(scrapy.Spider):
name = "meeting"
allowed_domains = ["obas_front.tcc.gov.tw"]
start_urls = [
"http://obas_front.tcc.gov.tw:8080/Agenda/EFileSearch.aspx?FileGrpKind=2&h=600",
]
download_delay = 0.5
county_abbr = os.path.dirname(os.path.realpath(__file__)).split('/')[-1]
election_year = common.election_year(county_abbr)
output_path = common.meeting_minutes_output_path(county_abbr,
election_year)
payload = {
'btnCongress': u'大會',
'txtPageSize': u'300',
}
def parse(self, response):
return scrapy.FormRequest.from_response(response,
response.url,
formdata=self.payload,
callback=self.parse_post)
def parse_post(self, response):
links = response.xpath(
'//table/tr/td/a[contains(@href, "EFileDetail.aspx")]/@href'
).extract()
for link in links:
yield response.follow(link, callback=self.parse_profile)
def parse_profile(self, response):
item = {}
item['election_year'] = self.election_year
nodes = response.xpath('//table/tbody/tr')
ref = {
u'屆別': {
'key': 'sitting',
'path': 'td/span/text()'
},
u'類別': {
'key': 'category',
'path': 'td/span/text()'
},
u'日期': {
'key': 'date',
'path': 'td/span/text()'
},
u'資料名稱': {
'key': 'meeting',
'path': 'td/span/text()'
},
u'檔案': {
'key': 'download_url',
'path': 'td/a/@href',
'extra': 'http://obas_front.tcc.gov.tw:8080/Agenda/'
},
}
for node in nodes:
value = ref.get(node.xpath('th/text()').extract_first().strip())
if value:
item[value['key']] = '%s%s' % (value.get(
'extra', ''), node.xpath(value['path']).extract_first())
item['date'] = common.ROC2AD(item['date'])
ext = re.search(u'FileName=[\w\d]+\.(\w+)&',
item['download_url']).group(1)
file_name = '%s_%s.%s' % (item['sitting'], item['meeting'], ext)
cmd = 'mkdir -p %s && wget -c -O %s%s "%s"' % (
self.output_path, self.output_path, file_name,
item['download_url'])
retcode = subprocess.call(cmd, shell=True)
return item