def main(args): bs = get_binstar(args, cls=BinstarBuildAPI) worker_config = WorkerConfiguration.load(args.worker_id, bs, warn=True) WorkerConfiguration.validate_worker_name(bs, args.worker_id) if worker_config.hostname != WorkerConfiguration.HOSTNAME: log.warn(WRONG_HOSTNAME_MSG.format(worker_config.hostname, WorkerConfiguration.HOSTNAME)) args.conda_build_dir = args.conda_build_dir.format(platform=worker_config.platform) setup_logging(logging.getLogger('binstar_build_client'), args.log_level, args.color, show_tb=args.show_traceback) log.info("Using conda build directory: {}".format(args.conda_build_dir)) log.info(str(worker_config)) worker = Worker(bs, worker_config, args) worker.write_status(True, "Starting") worker.write_stats() try: with worker_config.running(): worker.work_forever() finally: worker.write_status(False, "Exited")
def main(args, context="worker"): bs = get_binstar(args, cls=BinstarBuildAPI) if args.all: WorkerConfiguration.deregister_all(bs) elif args.worker_id: wconfig = WorkerConfiguration.load(args.worker_id, bs) wconfig.deregister(bs) else: log.info(context_info)
def main(args): if docker is None: raise errors.UserError("anaconda worker docker_run requires docker and docker-py to be installed\n" "Run:\n\tpip install docker-py") bs = get_binstar(args, cls=BinstarBuildAPI) worker_config = WorkerConfiguration.load(args.worker_id, bs, warn=True) WorkerConfiguration.validate_worker_name(bs, args.worker_id) if worker_config.hostname != WorkerConfiguration.HOSTNAME: log.warn(WRONG_HOSTNAME_MSG.format(worker_config.hostname, WorkerConfiguration.HOSTNAME)) worker = DockerWorker(bs, worker_config, args) worker.work_forever()
def test_register_backwards_compat(self): worker_file = os.path.join(WorkerConfiguration.REGISTERED_WORKERS_DIR, 'worker_name_1') worker_id = '123456789' worker_id_pid = '{}.123'.format(worker_id) with open(worker_file, 'w') as f: f.write(yaml.safe_dump({'worker_id': worker_id})) worker_id_to_name = WorkerConfiguration.backwards_compat_lookup() self.assertIn(worker_id, worker_id_to_name) if os.path.exists(worker_file): os.unlink(worker_file) self.assertEqual(worker_id_to_name[worker_id], 'worker_name_1') worker_id_to_name = WorkerConfiguration.backwards_compat_lookup() self.assertEqual(worker_id_to_name.get(worker_id, None), None)
def test_running(self): wc = WorkerConfiguration( 'worker_name', 'worker_id', 'username', 'queue', 'platform', 'hostname', 'dist' ) self.assertFalse(wc.is_running()) with wc.running(): self.assertIsNotNone(wc.pid) self.assertTrue(wc.is_running()) self.assertFalse(wc.is_running())
def test_duplicate_worker_name(self, registered): worker_configs = [Namespace(name='abc', worker_id='id_' + worker, username='******', queue='queue', platform='platform', hostname='hostname', dist='dist') for worker in ('a', 'b')] for worker_config in worker_configs: worker_config.to_dict = lambda : worker_config.__dict__ registered.return_value = iter(worker_configs) bs = get_binstar(Namespace(), cls=BinstarBuildAPI) with self.assertRaises(errors.BinstarError): WorkerConfiguration.validate_worker_name(bs, 'abc')
def main(args): if docker is None: raise errors.UserError("anaconda worker docker_run requires docker and docker-py to be installed\n" "Run:\n\tpip install docker-py") bs = get_binstar(args, cls=BinstarBuildAPI) worker_config = WorkerConfiguration.load(args.worker_id, bs, warn=True) WorkerConfiguration.validate_worker_name(bs, args.worker_id) if worker_config.hostname != WorkerConfiguration.HOSTNAME: log.warn(WRONG_HOSTNAME_MSG.format(worker_config.hostname, WorkerConfiguration.HOSTNAME)) setup_logging(logging.getLogger('binstar_build_client'), args.log_level, args.color, show_tb=args.show_traceback) worker = DockerWorker(bs, worker_config, args) worker.write_stats() worker.work_forever()
def main(args): if docker is None: raise errors.UserError("anaconda worker docker_run requires docker and docker-py to be installed\n" "Run:\n\tpip install docker-py") bs = get_binstar(args, cls=BinstarBuildAPI) worker_config = WorkerConfiguration.load(args.worker_id, bs) worker = DockerWorker(bs, worker_config, args) worker.work_forever()
def main(args): args.username, args.queue = split_queue_arg(args.queue) bs = get_binstar(args, cls=BinstarBuildAPI) worker_config = WorkerConfiguration.register( bs, args.username, args.queue, args.platform, args.hostname, args.dist, name=args.name, ) log.info('When running, worker PID files will be at {}.<PID>.'.format(worker_config.filename)) log.info('Now run:\n\tanaconda worker run {}'.format(worker_config.name))
def main(args): if docker is None: raise errors.UserError( "anaconda worker docker_run requires docker and docker-py to be installed\n" "Run:\n\tpip install docker-py") bs = get_binstar(args, cls=BinstarBuildAPI) worker_config = WorkerConfiguration.load(args.worker_id, bs, warn=True) WorkerConfiguration.validate_worker_name(bs, args.worker_id) if worker_config.hostname != WorkerConfiguration.HOSTNAME: log.warn( WRONG_HOSTNAME_MSG.format(worker_config.hostname, WorkerConfiguration.HOSTNAME)) setup_logging(logging.getLogger('binstar_build_client'), args.log_level, args.color, show_tb=args.show_traceback) worker = DockerWorker(bs, worker_config, args) worker.write_stats() worker.work_forever()
def main(args): bs = get_binstar(args, cls=BinstarBuildAPI) worker_config = WorkerConfiguration.load(args.worker_id, bs, warn=True) WorkerConfiguration.validate_worker_name(bs, args.worker_id) if worker_config.hostname != WorkerConfiguration.HOSTNAME: log.warn( WRONG_HOSTNAME_MSG.format(worker_config.hostname, WorkerConfiguration.HOSTNAME)) args.conda_build_dir = args.conda_build_dir.format( platform=worker_config.platform) log.info("Using conda build directory: {}".format(args.conda_build_dir)) log.info(str(worker_config)) worker = Worker(bs, worker_config, args) worker.write_status(True, "Starting") try: with worker_config.running(): worker.work_forever() finally: worker.write_status(False, "Exited")
def __init__(self): self.SLEEP_TIME = 0 bs = Mock() bs.log_build_output.return_value = False args = Mock() args.status_file = None args.timeout = 100 worker_config = WorkerConfiguration('worker_name', 'worker_id', 'username', 'queue', 'test_platform', 'test_hostname', 'dist') Worker.__init__(self, bs, worker_config, args)
def main(args): bs = get_binstar(args, cls=BinstarBuildAPI) worker_config = WorkerConfiguration.load(args.worker_id, bs) args.conda_build_dir = args.conda_build_dir.format(platform=worker_config.platform) log.info("Using conda build directory: {}".format(args.conda_build_dir)) log.info(str(worker_config)) worker = Worker(bs, worker_config, args) worker.write_status(True, "Starting") try: with worker_config.running(): worker.work_forever() finally: worker.write_status(False, "Exited")
def test_str(self): wc = WorkerConfiguration('worker_name', 'worker_id', 'username', 'queue', 'platform', 'hostname', 'dist') expected = """WorkerConfiguration \tpid: None \tdist: dist \thostname: hostname \tname: worker_name \tplatform: platform \tqueue: queue \tusername: username \tworker_id: worker_id """ self.assertEqual(str(wc), expected)
def __init__(self): self.SLEEP_TIME = 0 bs = Mock() bs.log_build_output.return_value = False args = Mock() args.status_file = None args.timeout = 100 args.show_new_procs = False args.cwd = tempfile.mkdtemp() worker_config = WorkerConfiguration('worker_name', 'worker_id', 'username', 'queue', 'test_platform', 'test_hostname', 'dist') super(MyWorker, self).__init__(bs, worker_config, args)
def __init__(self): self.SLEEP_TIME = 0 bs = Mock() bs.log_build_output.return_value = False args = Mock() args.status_file = None args.timeout = 100 args.show_new_procs = False args.image = 'binstar/linux-64' worker_config = WorkerConfiguration('worker_id', 'worker_id', 'username', 'queue', 'test_platform', 'test_hostname', 'dist') super(TestDockerWorker, self).__init__(bs, worker_config, args)
def main(args): bs = get_binstar(args, cls=BinstarBuildAPI) worker_config = WorkerConfiguration.load(args.worker_id, bs) args.conda_build_dir = args.conda_build_dir.format( platform=worker_config.platform) log.info("Using conda build directory: {}".format(args.conda_build_dir)) log.info(str(worker_config)) worker = Worker(bs, worker_config, args) worker.write_status(True, "Starting") try: with worker_config.running(): worker.work_forever() finally: worker.write_status(False, "Exited")
def test_running(self): wc = WorkerConfiguration('worker_name', 'worker_id', 'username', 'queue', 'platform', 'hostname', 'dist') self.assertFalse(wc.is_running()) with wc.running(): self.assertIsNotNone(wc.pid) self.assertTrue(wc.is_running()) self.assertFalse(wc.is_running())
def main(args): args.username, args.queue = split_queue_arg(args.queue) bs = get_binstar(args, cls=BinstarBuildAPI) worker_config = WorkerConfiguration.register( bs, args.username, args.queue, args.platform, args.hostname, args.dist, name=args.name, ) log.info('When running, worker PID files will be at {}.<PID>.'.format( worker_config.filename)) log.info('Now run:\n\tanaconda worker run {}'.format(worker_config.name))
def test_already_running(self): wc = WorkerConfiguration('worker_name', 'worker_id', 'username', 'queue', 'platform', 'hostname', 'dist') self.assertFalse(wc.is_running()) with wc.running(): with self.assertRaises(errors.BinstarError): with wc.running(): pass self.assertTrue(wc.is_running()) self.assertFalse(wc.is_running())
def main(args): args.username, args.queue = split_queue_arg(args.queue) bs = get_binstar(args, cls=BinstarBuildAPI) if args.name: if not is_valid_name(args.name): raise errors.BinstarError('Invalid name for ' 'worker: {}. Must start' ' with a letter and contain' ' only numbers, letters, -, and _'.format(args.name)) worker_config = WorkerConfiguration.register( bs, args.username, args.queue, args.platform, args.hostname, args.dist, name=args.name, ) log.info('When running, worker PID files will be at {}.<PID>.'.format(worker_config.filename)) log.info('Now run:\n\tanaconda worker run {}'.format(worker_config.name))
def test_register_backwards_compat_pid(self): '''Test .workers files that when yaml loaded will error out or not return a dict.''' folder = WorkerConfiguration.REGISTERED_WORKERS_DIR for f in os.listdir(folder): os.unlink(os.path.join(folder, f)) test_cases = [ ('worker1.123', ''), ('worker2.234', 'user99'), # su worker uses usernames in pid files ('worker3.1234', '{bad_dict: [abc,'), # this shouldn't happen but just in case ] for pid_file, content in test_cases: worker_id_pid = os.path.join(folder, pid_file) with open(worker_id_pid, 'w') as f: f.write(content) worker_id_to_name = WorkerConfiguration.backwards_compat_lookup() # the above should be len zero because folder started # empty and added only pid files. No # worker yaml's that have worker_id's # in them were added. It should skip without error # over bad or irrelevant files (non-yaml worker configs) self.assertEqual(len(worker_id_to_name), 0)
def test_register_backwards_compat_pid(self): '''Test .workers files that when yaml loaded will error out or not return a dict.''' folder = WorkerConfiguration.REGISTERED_WORKERS_DIR for f in os.listdir(folder): os.unlink(os.path.join(folder, f)) test_cases = [ ('worker1.123', ''), ('worker2.234', 'user99'), # su worker uses usernames in pid files ('worker3.1234', '{bad_dict: [abc,'), # this shouldn't happen but just in case ] for pid_file, content in test_cases: worker_id_pid = os.path.join(folder, pid_file) with open(worker_id_pid, 'w') as f: f.write(content) worker_id_to_name = WorkerConfiguration.backwards_compat_lookup() # the above should be len zero because folder started # empty and added only pid files. No # worker yaml's that have worker_id's # in them were added. It should skip without error # over bad or irrelevant files (non-yaml worker configs) self.assertEqual(len(worker_id_to_name), 0)
def print_registered_workers(bs, args): has_workers = False log.info('Registered workers:') if args.queue: user, args.queue = split_queue_arg(args.queue) for wconfig in WorkerConfiguration.registered_workers(bs): has_workers = True if args.this_host_only and wconfig.hostname != WorkerConfiguration.HOSTNAME: continue if args.queue and args.queue != wconfig.queue: continue if args.org and args.org != wconfig.username: continue msg = '{name}, id:{worker_id}, hostname:{hostname}, queue:{username}/{queue}'.format(**wconfig.to_dict()) if wconfig.pid: msg += ' (running with pid: {})'.format(wconfig.pid) log.info(msg) if not has_workers: log.info('(No registered workers)')
def print_registered_workers(bs, args): has_workers = False log.info('Registered workers:') if args.queue: user, args.queue = split_queue_arg(args.queue) for wconfig in WorkerConfiguration.registered_workers(bs): has_workers = True if args.this_host_only and wconfig.hostname != WorkerConfiguration.HOSTNAME: continue if args.queue and args.queue != wconfig.queue: continue if args.org and args.org != wconfig.username: continue msg = '{name}, id:{worker_id}, hostname:{hostname}, queue:{username}/{queue}'.format( **wconfig.to_dict()) if wconfig.pid: msg += ' (running with pid: {})'.format(wconfig.pid) log.info(msg) if not has_workers: log.info('(No registered workers)')
def test_already_running(self): wc = WorkerConfiguration( 'worker_name', 'worker_id', 'username', 'queue', 'platform', 'hostname', 'dist' ) self.assertFalse(wc.is_running()) with wc.running(): with self.assertRaises(errors.BinstarError): with wc.running(): pass self.assertTrue(wc.is_running()) self.assertFalse(wc.is_running())
def main(args): bs = get_binstar(args, cls=BinstarBuildAPI) WorkerConfiguration.print_registered_workers(bs)