Python run Examples

Example #1

File: run.py Project: Fishermanadg/pyspider

def run_webui():
    import xmlrpclib
    scheduler_rpc = xmlrpclib.ServerProxy('http://localhost:%d' % scheduler_xmlrpc_port)
    fetch_rpc = xmlrpclib.ServerProxy('http://localhost:%d' % fetcher_xmlrpc_port)

    from webui.app import app
    app.config['fetch'] = lambda task: fetch_rpc.fetch(task)
    app.config['projectdb'] = get_projectdb
    app.config['scheduler_rpc'] = scheduler_rpc
    app.run()

Example #2

def run_webui():
    import xmlrpclib
    scheduler_rpc = xmlrpclib.ServerProxy('http://localhost:%d' % scheduler_xmlrpc_port)
    fetch_rpc = xmlrpclib.ServerProxy('http://localhost:%d' % fetcher_xmlrpc_port)

    from webui.app import app
    app.config['fetch'] = lambda task: fetch_rpc.fetch(task)
    app.config['projectdb'] = get_projectdb
    app.config['scheduler_rpc'] = scheduler_rpc
    app.run()

Example #3

File: run.py Project: deyu260/pyspider

def run_webui():
    import xmlrpclib
    import cPickle as pickle
    scheduler_rpc = xmlrpclib.ServerProxy('http://localhost:%d' % scheduler_xmlrpc_port)
    fetch_rpc = xmlrpclib.ServerProxy('http://localhost:%d' % fetcher_xmlrpc_port)

    from webui.app import app
    app.config['fetch'] = lambda task: pickle.loads(fetch_rpc.fetch(task).data)
    app.config['projectdb'] = get_projectdb
    app.config['scheduler_rpc'] = scheduler_rpc
    #app.config['cdn'] = '//cdnjs.cloudflare.com/ajax/libs/'
    app.run('0.0.0.0')

Example #4

File: run.py Project: topcaoagui/pyspider

def run_webui():
    import xmlrpclib
    import cPickle as pickle
    scheduler_rpc = xmlrpclib.ServerProxy('http://localhost:%d' %
                                          scheduler_xmlrpc_port)
    fetch_rpc = xmlrpclib.ServerProxy('http://localhost:%d' %
                                      fetcher_xmlrpc_port)

    from webui.app import app
    app.config['fetch'] = lambda task: pickle.loads(fetch_rpc.fetch(task).data)
    app.config['projectdb'] = get_projectdb
    app.config['scheduler_rpc'] = scheduler_rpc
    #app.config['cdn'] = '//cdnjs.cloudflare.com/ajax/libs/'
    app.run()

Example #5

File: run.py Project: tanleihaoren/pyspider

def run_webui(g=g):
    import cPickle as pickle

    from webui.app import app
    app.config['taskdb'] = g.taskdb
    app.config['projectdb'] = g.projectdb
    app.config['resultdb'] = g.resultdb
    app.config['scheduler_rpc'] = g.scheduler_rpc
    #app.config['cdn'] = '//cdnjs.cloudflare.com/ajax/libs/'
    if g.demo_mode:
        app.config['max_rate'] = 0.2
        app.config['max_burst'] = 3.0
    if not getattr(g, 'all_in_one', False):
        app.debug = True
    app.run(host=g.webui_host, port=g.webui_port)

Example #6

File: run.py Project: kelvin007/pyspider

def run_webui(g=g):
    import cPickle as pickle

    from webui.app import app
    app.config['taskdb'] = g.taskdb
    app.config['projectdb'] = g.projectdb
    app.config['resultdb'] = g.resultdb
    app.config['scheduler_rpc'] = g.scheduler_rpc
    #app.config['cdn'] = '//cdnjs.cloudflare.com/ajax/libs/'
    if g.demo_mode:
        app.config['max_rate'] = 0.2
        app.config['max_burst'] = 3.0
    if 'WEBUI_USERNAME' in os.environ:
        app.config['webui_username'] = os.environ['WEBUI_USERNAME']
        app.config['webui_password'] = os.environ.get('WEBUI_PASSWORD', '')
    if not getattr(g, 'all_in_one', False):
        app.debug = True
    app.run(host=g.webui_host, port=g.webui_port)

Example #7

def run_webui(g=g):
    import cPickle as pickle

    from fetcher.tornado_fetcher import Fetcher
    fetcher = Fetcher(inqueue=None, outqueue=None, async=False)
    fetcher.phantomjs_proxy = g.phantomjs_proxy

    from webui.app import app
    app.config['taskdb'] = g.taskdb
    app.config['projectdb'] = g.projectdb
    app.config['resultdb'] = g.resultdb
    app.config['fetch'] = lambda x: fetcher.fetch(x)[1]
    app.config['scheduler_rpc'] = g.scheduler_rpc
    #app.config['cdn'] = '//cdnjs.cloudflare.com/ajax/libs/'
    if g.demo_mode:
        app.config['max_rate'] = 0.2
        app.config['max_burst'] = 3.0
    if 'WEBUI_USERNAME' in os.environ:
        app.config['webui_username'] = os.environ['WEBUI_USERNAME']
        app.config['webui_password'] = os.environ.get('WEBUI_PASSWORD', '')
    if not getattr(g, 'all_in_one', False):
        app.debug = g.debug
    app.run(host=g.webui_host, port=g.webui_port)

Example #8

File: webui.py Project: 7uk0n/pyspider

#!/usr/bin/env python
# -*- encoding: utf-8 -*-
# vim: set et sw=4 ts=4 sts=4 ff=unix fenc=utf8:
# Author: Binux<*****@*****.**>
#         http://binux.me
# Created on 2014-02-22 23:19:11

import xmlrpclib

from webui.app import app
from fetcher import tornado_fetcher
from database.sqlite import taskdb, projectdb

def fetch(task):
    t, f = tornado_fetcher.Fetcher(None, None, async=False).fetch(task)
    return f

config = {
        'fetch': fetch,
        'projectdb': projectdb.ProjectDB('data/project.db'),
        'taskdb': taskdb.TaskDB('./data/task.db'),
        'scheduler_rpc': xmlrpclib.ServerProxy('http://localhost:23333')
        }

if __name__ == '__main__':
    app.config.update(**config)
    app.debug = True
    app.run()

Example #9

File: webui.py Project: rmaC7/pyspider

#!/usr/bin/env python
# -*- encoding: utf-8 -*-
# vim: set et sw=4 ts=4 sts=4 ff=unix fenc=utf8:
# Author: Binux<*****@*****.**>
#         http://binux.me
# Created on 2014-02-22 23:19:11

import xmlrpclib

from webui.app import app
from fetcher import tornado_fetcher
from database.sqlite import taskdb, projectdb


def fetch(task):
    t, f = tornado_fetcher.Fetcher(None, None, async=False).fetch(task)
    return f


config = {
    'fetch': fetch,
    'projectdb': projectdb.ProjectDB('data/project.db'),
    'taskdb': taskdb.TaskDB('./data/task.db'),
    'scheduler_rpc': xmlrpclib.ServerProxy('http://localhost:23333')
}

if __name__ == '__main__':
    app.config.update(**config)
    app.debug = True
    app.run()