Example #1
0
here = dirname(abspath(__file__))

# update our python path to be at root of project
base = dirname(here)
sys.path.insert(0, base)

# we default to production
debug = 'debug' in sys.argv
print 'debug: %s' % debug

# CONTEXT
from context import context

# set our config into the context
config = context.get('get_config')(debug, 'scraper')
context.add('config', config)

# gimmie a prefix to check, if ALL is provided we'll
# page through all results
prefix = sys.argv[1]

# what dir are these files going into ?
context.update(out_path=os.path.abspath(sys.argv[2]))

# our functions to do the work
def download_image(obj, out_path, overwrite=False):
    out_p = os.path.join(out_path, obj.name)

    # atomic operation to check if file is there, if it's
    # not than we'll write a place holder file there
    # and than download the real file
Example #2
0
# bring in the event app framework
from lib.eventapp import EventApp
import lib.eventapp as eventapp


# bring in the event handler functions
import handlers.page as page
import handlers.image as image


# read in the context
from context import context

# add our debug flag to config
context.add('debug', debug)

# set our config into the context
config = context.get('get_config')(debug, 'scraper')
context.add('config', config)


# configure eventapp to have the # of threads / forks we want
thread_count = config.get('knobs').get('threads_per_stage')
fork_count = config.get('knobs').get('forks')

threaded = config.get('knobs').get('threaded')
multiprocess = config.get('knobs').get('multiprocess')


# set up our event app
Example #3
0
# CONTEXT
from context import context


# and our helpers
import lib.helpers as helpers

# FLASK it up !
from flask import Flask, Response, redirect

# create our app
app = Flask('image-scraper')

# set our config into the context
config = context.get('get_config')(debug, 'scraper')
context.add('config', config)

# instantiate a revent client
revent_client = context.get('get_revent')(**config.get('revent'))

# add the revent client to context
context.add('revent', revent_client)

# add our debug flag to config
context.add('debug', debug)



# OUR WSGI HANDLERS

@app.route("/blogs/")