def main(killed_event=None):
p = argparse.ArgumentParser(
description='Run a Manhattan worker with a TimeRotatingLog.')
p.add_argument('-v',
'--verbose',
dest='verbose',
action='store_true',
default=False,
help='Print detailed output')
p.add_argument('-p',
'--path',
dest='input_log_path',
type=str,
help='Input Manhattan log path')
p.add_argument('--log',
dest='error_log_path',
type=str,
help='Path to error/debug log')
p.add_argument('-u', '--url', dest='url', type=str, help='SQL backend URL')
p.add_argument('-c',
'--complex',
dest='complex',
action='append',
help='Configure complex goal, like '
'name|include a, include b|exclude a')
p.add_argument('--bind',
type=str,
default='tcp://127.0.0.1:5555',
help='ZeroMQ socket description to bind to')
p.add_argument('--config',
type=str,
help='Python namespace to use for configuration')
args = p.parse_args()
if args.config:
config = load_python_config(args.config)
else:
config = load_args_config(args)
logging.config.dictConfig(
logging_config(config.pop('verbose'), config.pop('error_log_path')))
input_log_path = config.pop('input_log_path')
bind = config.pop('bind')
backend = Backend(**config)
manhattan.server_backend = backend
mhlog = TimeRotatingLog(input_log_path)
worker = Worker(mhlog, backend, stats_every=5000)
server = Server(backend, bind=bind)
server.start()
try:
worker.run(stay_alive=True, killed_event=killed_event)
finally:
server.kill()
def test_memory_log(self):
log = MemoryLog()
data.run_clickstream(log)
backend = self._get_backend(reset=True)
worker1 = Worker(log, backend)
worker1.run(resume=False)
self._check_backend_queries(backend)
def test_memory_log(self):
log = MemoryLog()
data.run_clickstream(log)
backend = self._get_backend(reset=True)
worker1 = Worker(log, backend)
worker1.run(resume=False)
self._check_backend_queries(backend)
def test_basic(self):
path = work_path('basic')
backend = self._get_backend(reset=True)
log_w = TimeRotatingLog(path)
data.run_clickstream(log_w)
log_r = TimeRotatingLog(path)
worker1 = Worker(log_r, backend)
worker1.run()
self._check_backend_queries(backend)
def test_basic(self):
path = work_path('basic')
backend = self._get_backend(reset=True)
log_w = TimeRotatingLog(path)
data.run_clickstream(log_w)
log_r = TimeRotatingLog(path)
worker1 = Worker(log_r, backend)
worker1.run()
self._check_backend_queries(backend)
def test_resume(self):
path = work_path('resume')
backend = self._get_backend(reset=True)
log_w = TimeRotatingLog(path)
data.run_clickstream(log_w, first=0, last=25)
log_r1 = TimeRotatingLog(path)
worker1 = Worker(log_r1, backend)
worker1.run()
first_pointer = backend.get_pointer()
self.assertIsNotNone(first_pointer)
backend = self._get_backend(reset=False)
data.run_clickstream(log_w, first=25)
log_r2 = TimeRotatingLog(path)
worker2 = Worker(log_r2, backend)
worker2.run(resume=True)
second_pointer = backend.get_pointer()
self.assertIsNotNone(second_pointer)
self._check_backend_queries(backend)
def main(killed_event=None):
p = argparse.ArgumentParser(
description='Run a Manhattan worker with a TimeRotatingLog.')
p.add_argument('-v', '--verbose', dest='verbose', action='store_true',
default=False, help='Print detailed output')
p.add_argument('-p', '--path', dest='input_log_path', type=str,
help='Input Manhattan log path')
p.add_argument('--log', dest='error_log_path', type=str,
help='Path to error/debug log')
p.add_argument('-u', '--url', dest='url', type=str,
help='SQL backend URL')
p.add_argument('-c', '--complex', dest='complex', action='append',
help='Configure complex goal, like '
'name|include a, include b|exclude a')
p.add_argument('--bind', type=str,
default='tcp://127.0.0.1:5555',
help='ZeroMQ socket description to bind to')
p.add_argument('--config', type=str,
help='Python namespace to use for configuration')
args = p.parse_args()
if args.config:
config = load_python_config(args.config)
else:
config = load_args_config(args)
logging.config.dictConfig(logging_config(config.pop('verbose'),
config.pop('error_log_path')))
input_log_path = config.pop('input_log_path')
bind = config.pop('bind')
backend = Backend(**config)
manhattan.server_backend = backend
mhlog = TimeRotatingLog(input_log_path)
worker = Worker(mhlog, backend, stats_every=5000)
server = Server(backend, bind=bind)
server.start()
try:
worker.run(stay_alive=True, killed_event=killed_event)
finally:
server.kill()
def test_resume(self):
path = work_path('resume')
backend = self._get_backend(reset=True)
log_w = TimeRotatingLog(path)
data.run_clickstream(log_w, first=0, last=25)
log_r1 = TimeRotatingLog(path)
worker1 = Worker(log_r1, backend)
worker1.run()
first_pointer = backend.get_pointer()
self.assertIsNotNone(first_pointer)
backend = self._get_backend(reset=False)
data.run_clickstream(log_w, first=25)
log_r2 = TimeRotatingLog(path)
worker2 = Worker(log_r2, backend)
worker2.run(resume=True)
second_pointer = backend.get_pointer()
self.assertIsNotNone(second_pointer)
self._check_backend_queries(backend)
