here = dirname(abspath(__file__))
# update our python path to be at root of project
base = dirname(here)
sys.path.insert(0, base)
# we default to production
debug = 'debug' in sys.argv
print 'debug: %s' % debug
# CONTEXT
from context import context
# set our config into the context
config = context.get('get_config')(debug, 'scraper')
context.add('config', config)
# gimmie a prefix to check, if ALL is provided we'll
# page through all results
prefix = sys.argv[1]
# what dir are these files going into ?
context.update(out_path=os.path.abspath(sys.argv[2]))
# our functions to do the work
def download_image(obj, out_path, overwrite=False):
out_p = os.path.join(out_path, obj.name)
# atomic operation to check if file is there, if it's
# not than we'll write a place holder file there
# and than download the real file
# bring in the event app framework
from lib.eventapp import EventApp
import lib.eventapp as eventapp
# bring in the event handler functions
import handlers.page as page
import handlers.image as image
# read in the context
from context import context
# add our debug flag to config
context.add('debug', debug)
# set our config into the context
config = context.get('get_config')(debug, 'scraper')
context.add('config', config)
# configure eventapp to have the # of threads / forks we want
thread_count = config.get('knobs').get('threads_per_stage')
fork_count = config.get('knobs').get('forks')
threaded = config.get('knobs').get('threaded')
multiprocess = config.get('knobs').get('multiprocess')
# set up our event app
# CONTEXT
from context import context
# and our helpers
import lib.helpers as helpers
# FLASK it up !
from flask import Flask, Response, redirect
# create our app
app = Flask('image-scraper')
# set our config into the context
config = context.get('get_config')(debug, 'scraper')
context.add('config', config)
# instantiate a revent client
revent_client = context.get('get_revent')(**config.get('revent'))
# add the revent client to context
context.add('revent', revent_client)
# add our debug flag to config
context.add('debug', debug)
# OUR WSGI HANDLERS
@app.route("/blogs/")