#!/usr/bin/env python3

#

# Scrape recipes from pepperplate.com.

#

import requests

from bs4 import BeautifulSoup

import json

import time

import getpass

import re

import os

def pp_login(username, password):

return s

def pp_get_page(session, page):

return soup

def scrape_recipe_ids(soup):

ids = [re.findall(r'id=(\d+)', a['href'])[0] for d in soup.find_all('div',{'class':'item'}) for a in d.find_all('a')]

print('Found {} recipes'.format(len(ids)))

return ids

def get_recipe(session, id, imgpath):

return title, soup

def format_recipe(old_soup):

return new_soup

def save_recipe(title, id, soup, savepath):

title = title.replace('/','_').replace('"', '').replace(':','')

with open(savepath + '/{}.{}.html'.format(title, id), 'wb') as f:

f.write(soup.prettify('latin-1'))

if __name__ == '__main__':

import argparse

parser = argparse.ArgumentParser(description='Scrape recipies from Pepperplate')

parser.add_argument('username', help='Username to log in with')

parser.add_argument('password', nargs="?", default=None, help='Password to log in with. If not provided on the command line it will be requested by the program')

parser.add_argument('directory', nargs="?", default='recipes', help='Directory to which download everything. defaults to "recipes"')

args = parser.parse_args()

imgpath = os.path.join(args.directory, 'img')

if not os.path.exists(imgpath):

os.makedirs(imgpath)

session = pp_login(args.username, args.password)

page = 0

soup = pp_get_page(session,page)

ids = scrape_recipe_ids(soup)

while len(ids) > 0:

for id in ids:

time.sleep(1) #sleep 1 second between requests to not mash the server

title, soup = get_recipe(session, id, imgpath)

soup = format_recipe(soup)

save_recipe(title, id, soup, args.directory)

page += 1

soup = pp_get_page(session,page)

ids = scrape_recipe_ids(soup)