def run(redis_connection, node_url, node_name, queue, datadir, read_only, module): redis_connection_obj = parse_redis_connection(redis_connection) r = redis.StrictRedis(host=redis_connection_obj['host'], port=redis_connection_obj['port'], db=redis_connection_obj['db']) server_manager = Servers(r) settings.setup_server(r, datadir, node_url, node_name, Catalog(r, datadir, node_name), server_manager, _read_only=read_only ) if node_name is None: node_name = node_url if module: mod = __import__(module) if queue is None: queue = ['default'] with Connection(r): queues = [] node_queue = KitchenSinkRedisQueue(node_url) queues.append(node_queue) for q in queue: if '|' in q: raise Exception("queue names cannot contain colons") queues.append(KitchenSinkRedisQueue(q)) queues.append(KitchenSinkRedisQueue("%s|%s" % (q, node_name))) w = KitchenSinkWorker(queues, default_result_ttl=86400) w.work(burst=False)
def run(redis_connection, node_url, node_name, node_port, num_workers, no_redis, queue, module, datadir, read_only): if not node_url.endswith("/"): node_url += "/" if node_name is None: node_name = node_url datadir = abspath(datadir) register_shutdown() redis_connection_info = parse_redis_connection(redis_connection) if node_port is None: node_port = int(urlparse.urlparse(node_url).netloc.split(":")[-1]) print ("**port", node_port) pid_file = "ks-%s.pid" % node_port if not no_redis: print ("Starting redis on %s" % redis_connection) start_redis(pid_file, redis_connection_info['port'], os.getcwd()) time.sleep(1) cmd = [sys.executable, '-m', 'kitchensink.scripts.start_worker', '--node-url', node_url, '--node-name', node_name, '--redis-connection', redis_connection, '--datadir', datadir, ] if module: cmd.extend(('--module', module)) if read_only: cmd.append('--read-only') app = make_app(redis_connection_info, node_port, node_url, node_name, datadir, read_only) app.debug_log_format = FORMAT for c in range(10): try: result = settings.redis_conn.ping() break except ConnectionError: time.sleep(1.0) if queue is None: queue = ['default', 'data'] for q in queue: cmd.extend(['--queue', q]) for c in range(num_workers): print ('**cmd', cmd) ManagedProcess(cmd, 'worker-%s' % c, pid_file) data_rpc = make_data_rpc() register_rpc(data_rpc, 'data') default_rpc = RPC() register_rpc(default_rpc, 'default') admin = make_admin_rpc() register_rpc(admin, 'admin') if module: mod = __import__(module) runserver(gevent=gevent)