def main(killed_event=None): p = argparse.ArgumentParser( description='Run a Manhattan worker with a TimeRotatingLog.') p.add_argument('-v', '--verbose', dest='verbose', action='store_true', default=False, help='Print detailed output') p.add_argument('-p', '--path', dest='input_log_path', type=str, help='Input Manhattan log path') p.add_argument('--log', dest='error_log_path', type=str, help='Path to error/debug log') p.add_argument('-u', '--url', dest='url', type=str, help='SQL backend URL') p.add_argument('-c', '--complex', dest='complex', action='append', help='Configure complex goal, like ' 'name|include a, include b|exclude a') p.add_argument('--bind', type=str, default='tcp://127.0.0.1:5555', help='ZeroMQ socket description to bind to') p.add_argument('--config', type=str, help='Python namespace to use for configuration') args = p.parse_args() if args.config: config = load_python_config(args.config) else: config = load_args_config(args) logging.config.dictConfig( logging_config(config.pop('verbose'), config.pop('error_log_path'))) input_log_path = config.pop('input_log_path') bind = config.pop('bind') backend = Backend(**config) manhattan.server_backend = backend mhlog = TimeRotatingLog(input_log_path) worker = Worker(mhlog, backend, stats_every=5000) server = Server(backend, bind=bind) server.start() try: worker.run(stay_alive=True, killed_event=killed_event) finally: server.kill()
def test_memory_log(self): log = MemoryLog() data.run_clickstream(log) backend = self._get_backend(reset=True) worker1 = Worker(log, backend) worker1.run(resume=False) self._check_backend_queries(backend)
def test_basic(self): path = work_path('basic') backend = self._get_backend(reset=True) log_w = TimeRotatingLog(path) data.run_clickstream(log_w) log_r = TimeRotatingLog(path) worker1 = Worker(log_r, backend) worker1.run() self._check_backend_queries(backend)
def test_resume(self): path = work_path('resume') backend = self._get_backend(reset=True) log_w = TimeRotatingLog(path) data.run_clickstream(log_w, first=0, last=25) log_r1 = TimeRotatingLog(path) worker1 = Worker(log_r1, backend) worker1.run() first_pointer = backend.get_pointer() self.assertIsNotNone(first_pointer) backend = self._get_backend(reset=False) data.run_clickstream(log_w, first=25) log_r2 = TimeRotatingLog(path) worker2 = Worker(log_r2, backend) worker2.run(resume=True) second_pointer = backend.get_pointer() self.assertIsNotNone(second_pointer) self._check_backend_queries(backend)
def main(killed_event=None): p = argparse.ArgumentParser( description='Run a Manhattan worker with a TimeRotatingLog.') p.add_argument('-v', '--verbose', dest='verbose', action='store_true', default=False, help='Print detailed output') p.add_argument('-p', '--path', dest='input_log_path', type=str, help='Input Manhattan log path') p.add_argument('--log', dest='error_log_path', type=str, help='Path to error/debug log') p.add_argument('-u', '--url', dest='url', type=str, help='SQL backend URL') p.add_argument('-c', '--complex', dest='complex', action='append', help='Configure complex goal, like ' 'name|include a, include b|exclude a') p.add_argument('--bind', type=str, default='tcp://127.0.0.1:5555', help='ZeroMQ socket description to bind to') p.add_argument('--config', type=str, help='Python namespace to use for configuration') args = p.parse_args() if args.config: config = load_python_config(args.config) else: config = load_args_config(args) logging.config.dictConfig(logging_config(config.pop('verbose'), config.pop('error_log_path'))) input_log_path = config.pop('input_log_path') bind = config.pop('bind') backend = Backend(**config) manhattan.server_backend = backend mhlog = TimeRotatingLog(input_log_path) worker = Worker(mhlog, backend, stats_every=5000) server = Server(backend, bind=bind) server.start() try: worker.run(stay_alive=True, killed_event=killed_event) finally: server.kill()