def list_pages(self):
workbook_path = self.retriever.get_to_file(self.get_excel_url())
yield ExcelDictReader(workbook_path,
sheet_index=0,
header_row_num=0,
start_row_num=1)
os.unlink(workbook_path) # Clean up the temporary file.
def list_pages(self):
for week_end_date in self.week_end_dates:
# They've used a different URL scheme over time.
if week_end_date <= datetime.date(2005, 5, 13):
url = 'http://www.nyc.gov/html/dob/downloads/download/foil/sg%s.xls' % week_end_date.strftime('%m%d%y')
else:
url = 'http://www.nyc.gov/html/dob/downloads/excel/sg%s.xls' % week_end_date.strftime('%m%d%y')
workbook_path = self.retriever.get_to_file(url)
yield ExcelDictReader(workbook_path, sheet_index=0, header_row_num=2, start_row_num=3,
use_last_header_if_duplicate=False)
os.unlink(workbook_path) # Clean up the temporary file.
def list_pages(self):
# Get the HTML page, which includes links to Excel files (one for each
# borough). We do this instead of hard-coding the Excel file names in
# the scraper because the Excel file names tend to change each month.
url = 'http://www.nyc.gov/html/dof/html/property/property_val_sales.shtml'
html = self.get_html(url)
excel_links = re.findall(r'href="([^"]+\.xls)"', html)
if len(excel_links) != 12:
raise ScraperBroken('Got a strange number of Excel links: %s' % len(excel_links))
# The first five links are the "New York City Sales Data" links,
# which is what we want.
for excel_link in excel_links[:5]:
excel_url = urlparse.urljoin(url, excel_link)
workbook_path = self.retriever.get_to_file(excel_url)
reader = ExcelDictReader(workbook_path, sheet_index=0, header_row_num=4, start_row_num=5)
yield reader
os.unlink(workbook_path) # Clean up the temporary file.
def list_pages(self):
for week_end_date in self.week_end_dates:
# They've used a different URL scheme over time.
if week_end_date <= datetime.date(2005, 5, 13):
url = 'http://www.nyc.gov/html/dob/downloads/download/foil/job%s.xls' % week_end_date.strftime(
'%m%d%y')
else:
url = 'http://www.nyc.gov/html/dob/downloads/excel/job%s.xls' % week_end_date.strftime(
'%m%d%y')
try:
workbook_path = self.retriever.get_to_file(url)
yield ExcelDictReader(workbook_path,
sheet_index=0,
header_row_num=2,
start_row_num=3)
os.unlink(workbook_path) # Clean up the temporary file.
except PageNotFoundError:
self.logger.warn("Could not find %s" % url)
def list_pages(self):
file_path = self.retriever.get_to_file(self.excel_url)
reader = ExcelDictReader(file_path, sheet_index=0, header_row_num=0, start_row_num=1)
yield reader
os.unlink(file_path) # Clean up the temporary file.
def list_pages(self):
reader = ExcelDictReader(self.excel_file_name,
sheet_index=0,
header_row_num=0,
start_row_num=1)
yield reader