def worker():
print('Scannertools: fetching resources...')
pipelines = cloudpickle.loads(base64.b64decode(os.environ['PIPELINES']))
for pipeline in pipelines:
pipeline(None).fetch_resources()
print('Scannertools: starting worker...')
scannerpy.start_worker(
'{}:{}'.format(os.environ['SCANNER_MASTER_SERVICE_HOST'],
os.environ['SCANNER_MASTER_SERVICE_PORT']),
block=True,
watchdog=False,
port=5002)
def worker():
machine_params = MachineParameters()
machine_params.ParseFromString(default_machine_params())
machine_params.num_load_workers = int(os.environ['NUM_LOAD_WORKERS'])
machine_params.num_save_workers = int(os.environ['NUM_SAVE_WORKERS'])
log.info('Scanner: starting worker...')
scannerpy.start_worker('{}:{}'.format(
os.environ['SCANNER_MASTER_SERVICE_HOST'],
os.environ['SCANNER_MASTER_SERVICE_PORT']),
machine_params=machine_params.SerializeToString(),
block=True,
watchdog=False,
port=5002)
def worker():
log.info('Scannertools: fetching resources...')
pipelines = cloudpickle.loads(base64.b64decode(os.environ['PIPELINES']))
for pipeline in pipelines:
pipeline(None).fetch_resources()
machine_params = MachineParameters()
machine_params.ParseFromString(default_machine_params())
machine_params.num_load_workers = int(os.environ['NUM_LOAD_WORKERS'])
machine_params.num_save_workers = int(os.environ['NUM_SAVE_WORKERS'])
log.info('Scannertools: starting worker...')
scannerpy.start_worker('{}:{}'.format(
os.environ['SCANNER_MASTER_SERVICE_HOST'],
os.environ['SCANNER_MASTER_SERVICE_PORT']),
machine_params=machine_params.SerializeToString(),
block=True,
watchdog=False,
port=5002)
import scannerpy
import os
import scanner.metadata_pb2 as metadata_types
import scannerpy._python as bindings
mp = bindings.default_machine_params()
mp_proto = metadata_types.MachineParameters()
mp_proto.ParseFromString(mp)
mp_proto.num_load_workers = 32
mp_proto.num_save_workers = 32
mp = mp_proto.SerializeToString()
scannerpy.start_worker(
'{}:{}'.format(os.environ['SCANNER_MASTER_SERVICE_HOST'],
os.environ['SCANNER_MASTER_SERVICE_PORT']),
block=True,
watchdog=False,
port=5002,
machine_params=mp)
from scannerpy import ProtobufGenerator, Config, start_worker
import time
import grpc
import sys
c = Config(None)
import scanner.metadata_pb2 as metadata_types
import scanner.engine.rpc_pb2 as rpc_types
import scanner.types_pb2 as misc_types
import scannerpy.libscanner as bindings
con = Config(config_path='/tmp/config_test')
protobufs = ProtobufGenerator(con)
master_address = str(con.master_address) + ':' + str(con.master_port)
port = int(sys.argv[1])
params = bindings.default_machine_params()
mp = protobufs.MachineParameters()
mp.ParseFromString(params)
del mp.gpu_ids[:]
params = mp.SerializeToString()
start_worker(master_address, machine_params=params, config=con, block=True,
port=port,
watchdog=False)
import scannerpy
import os
scannerpy.start_worker(
'{}:{}'.format(os.environ['SCANNER_MASTER_SERVICE_HOST'],
os.environ['SCANNER_MASTER_SERVICE_PORT']),
block=True,
watchdog=False,
prefetch_table_metadata=True,
port=5002)
from scannerpy import start_worker
import os
start_worker('{}:{}'.format(os.environ['SCANNER_MASTER_SERVICE_HOST'],
os.environ['SCANNER_MASTER_SERVICE_PORT']),
block=True,
watchdog=False)