Python start_worker示例

示例#1

def worker():
    print('Scannertools: fetching resources...')
    pipelines = cloudpickle.loads(base64.b64decode(os.environ['PIPELINES']))
    for pipeline in pipelines:
        pipeline(None).fetch_resources()

    print('Scannertools: starting worker...')
    scannerpy.start_worker(
        '{}:{}'.format(os.environ['SCANNER_MASTER_SERVICE_HOST'],
                       os.environ['SCANNER_MASTER_SERVICE_PORT']),
        block=True,
        watchdog=False,
        port=5002)

示例#2

文件： kube.py 项目： tarsbase/scanner

def worker():
    machine_params = MachineParameters()
    machine_params.ParseFromString(default_machine_params())
    machine_params.num_load_workers = int(os.environ['NUM_LOAD_WORKERS'])
    machine_params.num_save_workers = int(os.environ['NUM_SAVE_WORKERS'])

    log.info('Scanner: starting worker...')
    scannerpy.start_worker('{}:{}'.format(
        os.environ['SCANNER_MASTER_SERVICE_HOST'],
        os.environ['SCANNER_MASTER_SERVICE_PORT']),
                           machine_params=machine_params.SerializeToString(),
                           block=True,
                           watchdog=False,
                           port=5002)

示例#3

文件： kube.py 项目： bztia/scannertools

def worker():
    log.info('Scannertools: fetching resources...')
    pipelines = cloudpickle.loads(base64.b64decode(os.environ['PIPELINES']))
    for pipeline in pipelines:
        pipeline(None).fetch_resources()

    machine_params = MachineParameters()
    machine_params.ParseFromString(default_machine_params())
    machine_params.num_load_workers = int(os.environ['NUM_LOAD_WORKERS'])
    machine_params.num_save_workers = int(os.environ['NUM_SAVE_WORKERS'])

    log.info('Scannertools: starting worker...')
    scannerpy.start_worker('{}:{}'.format(
        os.environ['SCANNER_MASTER_SERVICE_HOST'],
        os.environ['SCANNER_MASTER_SERVICE_PORT']),
                           machine_params=machine_params.SerializeToString(),
                           block=True,
                           watchdog=False,
                           port=5002)

示例#4

文件： worker.py 项目： sjoerdapp/scannerapps

import scannerpy
import os
import scanner.metadata_pb2 as metadata_types
import scannerpy._python as bindings


mp = bindings.default_machine_params()
mp_proto = metadata_types.MachineParameters()
mp_proto.ParseFromString(mp)
mp_proto.num_load_workers = 32
mp_proto.num_save_workers = 32
mp = mp_proto.SerializeToString()

scannerpy.start_worker(
    '{}:{}'.format(os.environ['SCANNER_MASTER_SERVICE_HOST'],
                   os.environ['SCANNER_MASTER_SERVICE_PORT']),
    block=True,
    watchdog=False,
    port=5002,
    machine_params=mp)

示例#5

文件： spawn_worker.py 项目： swjz/scanner

from scannerpy import ProtobufGenerator, Config, start_worker
import time
import grpc
import sys

c = Config(None)

import scanner.metadata_pb2 as metadata_types
import scanner.engine.rpc_pb2 as rpc_types
import scanner.types_pb2 as misc_types
import scannerpy.libscanner as bindings

con = Config(config_path='/tmp/config_test')
protobufs = ProtobufGenerator(con)

master_address = str(con.master_address) + ':' + str(con.master_port)
port = int(sys.argv[1])

params = bindings.default_machine_params()
mp = protobufs.MachineParameters()
mp.ParseFromString(params)
del mp.gpu_ids[:]
params = mp.SerializeToString()

start_worker(master_address, machine_params=params, config=con, block=True,
             port=port,
             watchdog=False)

示例#6

import scannerpy
import os

scannerpy.start_worker(
    '{}:{}'.format(os.environ['SCANNER_MASTER_SERVICE_HOST'],
                   os.environ['SCANNER_MASTER_SERVICE_PORT']),
    block=True,
    watchdog=False,
    prefetch_table_metadata=True,
    port=5002)

示例#7

from scannerpy import start_worker
import os

start_worker('{}:{}'.format(os.environ['SCANNER_MASTER_SERVICE_HOST'],
                            os.environ['SCANNER_MASTER_SERVICE_PORT']),
             block=True,
             watchdog=False)