def main(args):
bs = get_binstar(args, cls=BinstarBuildAPI)
worker_config = WorkerConfiguration.load(args.worker_id, bs, warn=True)
WorkerConfiguration.validate_worker_name(bs, args.worker_id)
if worker_config.hostname != WorkerConfiguration.HOSTNAME:
log.warn(WRONG_HOSTNAME_MSG.format(worker_config.hostname,
WorkerConfiguration.HOSTNAME))
args.conda_build_dir = args.conda_build_dir.format(platform=worker_config.platform)
setup_logging(logging.getLogger('binstar_build_client'), args.log_level,
args.color, show_tb=args.show_traceback)
log.info("Using conda build directory: {}".format(args.conda_build_dir))
log.info(str(worker_config))
worker = Worker(bs, worker_config, args)
worker.write_status(True, "Starting")
worker.write_stats()
try:
with worker_config.running():
worker.work_forever()
finally:
worker.write_status(False, "Exited")
def main(args, context="worker"):
bs = get_binstar(args, cls=BinstarBuildAPI)
if args.all:
WorkerConfiguration.deregister_all(bs)
elif args.worker_id:
wconfig = WorkerConfiguration.load(args.worker_id, bs)
wconfig.deregister(bs)
else:
log.info(context_info)
def main(args):
if docker is None:
raise errors.UserError("anaconda worker docker_run requires docker and docker-py to be installed\n"
"Run:\n\tpip install docker-py")
bs = get_binstar(args, cls=BinstarBuildAPI)
worker_config = WorkerConfiguration.load(args.worker_id, bs, warn=True)
WorkerConfiguration.validate_worker_name(bs, args.worker_id)
if worker_config.hostname != WorkerConfiguration.HOSTNAME:
log.warn(WRONG_HOSTNAME_MSG.format(worker_config.hostname,
WorkerConfiguration.HOSTNAME))
worker = DockerWorker(bs, worker_config, args)
worker.work_forever()
def test_register_backwards_compat(self):
worker_file = os.path.join(WorkerConfiguration.REGISTERED_WORKERS_DIR,
'worker_name_1')
worker_id = '123456789'
worker_id_pid = '{}.123'.format(worker_id)
with open(worker_file, 'w') as f:
f.write(yaml.safe_dump({'worker_id': worker_id}))
worker_id_to_name = WorkerConfiguration.backwards_compat_lookup()
self.assertIn(worker_id, worker_id_to_name)
if os.path.exists(worker_file):
os.unlink(worker_file)
self.assertEqual(worker_id_to_name[worker_id], 'worker_name_1')
worker_id_to_name = WorkerConfiguration.backwards_compat_lookup()
self.assertEqual(worker_id_to_name.get(worker_id, None), None)
def test_running(self):
wc = WorkerConfiguration(
'worker_name',
'worker_id', 'username', 'queue',
'platform', 'hostname', 'dist'
)
self.assertFalse(wc.is_running())
with wc.running():
self.assertIsNotNone(wc.pid)
self.assertTrue(wc.is_running())
self.assertFalse(wc.is_running())
def test_duplicate_worker_name(self, registered):
worker_configs = [Namespace(name='abc',
worker_id='id_' + worker,
username='******',
queue='queue',
platform='platform',
hostname='hostname',
dist='dist') for worker in ('a', 'b')]
for worker_config in worker_configs:
worker_config.to_dict = lambda : worker_config.__dict__
registered.return_value = iter(worker_configs)
bs = get_binstar(Namespace(), cls=BinstarBuildAPI)
with self.assertRaises(errors.BinstarError):
WorkerConfiguration.validate_worker_name(bs, 'abc')
def main(args):
if docker is None:
raise errors.UserError("anaconda worker docker_run requires docker and docker-py to be installed\n"
"Run:\n\tpip install docker-py")
bs = get_binstar(args, cls=BinstarBuildAPI)
worker_config = WorkerConfiguration.load(args.worker_id, bs, warn=True)
WorkerConfiguration.validate_worker_name(bs, args.worker_id)
if worker_config.hostname != WorkerConfiguration.HOSTNAME:
log.warn(WRONG_HOSTNAME_MSG.format(worker_config.hostname,
WorkerConfiguration.HOSTNAME))
setup_logging(logging.getLogger('binstar_build_client'), args.log_level,
args.color, show_tb=args.show_traceback)
worker = DockerWorker(bs, worker_config, args)
worker.write_stats()
worker.work_forever()
def main(args):
if docker is None:
raise errors.UserError("anaconda worker docker_run requires docker and docker-py to be installed\n"
"Run:\n\tpip install docker-py")
bs = get_binstar(args, cls=BinstarBuildAPI)
worker_config = WorkerConfiguration.load(args.worker_id, bs)
worker = DockerWorker(bs, worker_config, args)
worker.work_forever()
def main(args):
args.username, args.queue = split_queue_arg(args.queue)
bs = get_binstar(args, cls=BinstarBuildAPI)
worker_config = WorkerConfiguration.register(
bs, args.username, args.queue,
args.platform, args.hostname, args.dist,
name=args.name,
)
log.info('When running, worker PID files will be at {}.<PID>.'.format(worker_config.filename))
log.info('Now run:\n\tanaconda worker run {}'.format(worker_config.name))
def main(args):
if docker is None:
raise errors.UserError(
"anaconda worker docker_run requires docker and docker-py to be installed\n"
"Run:\n\tpip install docker-py")
bs = get_binstar(args, cls=BinstarBuildAPI)
worker_config = WorkerConfiguration.load(args.worker_id, bs, warn=True)
WorkerConfiguration.validate_worker_name(bs, args.worker_id)
if worker_config.hostname != WorkerConfiguration.HOSTNAME:
log.warn(
WRONG_HOSTNAME_MSG.format(worker_config.hostname,
WorkerConfiguration.HOSTNAME))
setup_logging(logging.getLogger('binstar_build_client'),
args.log_level,
args.color,
show_tb=args.show_traceback)
worker = DockerWorker(bs, worker_config, args)
worker.write_stats()
worker.work_forever()
def main(args):
bs = get_binstar(args, cls=BinstarBuildAPI)
worker_config = WorkerConfiguration.load(args.worker_id, bs, warn=True)
WorkerConfiguration.validate_worker_name(bs, args.worker_id)
if worker_config.hostname != WorkerConfiguration.HOSTNAME:
log.warn(
WRONG_HOSTNAME_MSG.format(worker_config.hostname,
WorkerConfiguration.HOSTNAME))
args.conda_build_dir = args.conda_build_dir.format(
platform=worker_config.platform)
log.info("Using conda build directory: {}".format(args.conda_build_dir))
log.info(str(worker_config))
worker = Worker(bs, worker_config, args)
worker.write_status(True, "Starting")
try:
with worker_config.running():
worker.work_forever()
finally:
worker.write_status(False, "Exited")
def __init__(self):
self.SLEEP_TIME = 0
bs = Mock()
bs.log_build_output.return_value = False
args = Mock()
args.status_file = None
args.timeout = 100
worker_config = WorkerConfiguration('worker_name', 'worker_id',
'username', 'queue',
'test_platform', 'test_hostname',
'dist')
Worker.__init__(self, bs, worker_config, args)
def main(args):
bs = get_binstar(args, cls=BinstarBuildAPI)
worker_config = WorkerConfiguration.load(args.worker_id, bs)
args.conda_build_dir = args.conda_build_dir.format(platform=worker_config.platform)
log.info("Using conda build directory: {}".format(args.conda_build_dir))
log.info(str(worker_config))
worker = Worker(bs, worker_config, args)
worker.write_status(True, "Starting")
try:
with worker_config.running():
worker.work_forever()
finally:
worker.write_status(False, "Exited")
def test_str(self):
wc = WorkerConfiguration('worker_name', 'worker_id', 'username',
'queue', 'platform', 'hostname', 'dist')
expected = """WorkerConfiguration
\tpid: None
\tdist: dist
\thostname: hostname
\tname: worker_name
\tplatform: platform
\tqueue: queue
\tusername: username
\tworker_id: worker_id
"""
self.assertEqual(str(wc), expected)
def __init__(self):
self.SLEEP_TIME = 0
bs = Mock()
bs.log_build_output.return_value = False
args = Mock()
args.status_file = None
args.timeout = 100
args.show_new_procs = False
args.cwd = tempfile.mkdtemp()
worker_config = WorkerConfiguration('worker_name', 'worker_id',
'username', 'queue',
'test_platform', 'test_hostname',
'dist')
super(MyWorker, self).__init__(bs, worker_config, args)
def __init__(self):
self.SLEEP_TIME = 0
bs = Mock()
bs.log_build_output.return_value = False
args = Mock()
args.status_file = None
args.timeout = 100
args.show_new_procs = False
args.image = 'binstar/linux-64'
worker_config = WorkerConfiguration('worker_id', 'worker_id',
'username', 'queue',
'test_platform', 'test_hostname',
'dist')
super(TestDockerWorker, self).__init__(bs, worker_config, args)
def main(args):
bs = get_binstar(args, cls=BinstarBuildAPI)
worker_config = WorkerConfiguration.load(args.worker_id, bs)
args.conda_build_dir = args.conda_build_dir.format(
platform=worker_config.platform)
log.info("Using conda build directory: {}".format(args.conda_build_dir))
log.info(str(worker_config))
worker = Worker(bs, worker_config, args)
worker.write_status(True, "Starting")
try:
with worker_config.running():
worker.work_forever()
finally:
worker.write_status(False, "Exited")
def test_running(self):
wc = WorkerConfiguration('worker_name', 'worker_id', 'username',
'queue', 'platform', 'hostname', 'dist')
self.assertFalse(wc.is_running())
with wc.running():
self.assertIsNotNone(wc.pid)
self.assertTrue(wc.is_running())
self.assertFalse(wc.is_running())
def main(args):
args.username, args.queue = split_queue_arg(args.queue)
bs = get_binstar(args, cls=BinstarBuildAPI)
worker_config = WorkerConfiguration.register(
bs,
args.username,
args.queue,
args.platform,
args.hostname,
args.dist,
name=args.name,
)
log.info('When running, worker PID files will be at {}.<PID>.'.format(
worker_config.filename))
log.info('Now run:\n\tanaconda worker run {}'.format(worker_config.name))
def test_already_running(self):
wc = WorkerConfiguration('worker_name', 'worker_id', 'username',
'queue', 'platform', 'hostname', 'dist')
self.assertFalse(wc.is_running())
with wc.running():
with self.assertRaises(errors.BinstarError):
with wc.running():
pass
self.assertTrue(wc.is_running())
self.assertFalse(wc.is_running())
def main(args):
args.username, args.queue = split_queue_arg(args.queue)
bs = get_binstar(args, cls=BinstarBuildAPI)
if args.name:
if not is_valid_name(args.name):
raise errors.BinstarError('Invalid name for '
'worker: {}. Must start'
' with a letter and contain'
' only numbers, letters, -, and _'.format(args.name))
worker_config = WorkerConfiguration.register(
bs, args.username, args.queue,
args.platform, args.hostname, args.dist,
name=args.name,
)
log.info('When running, worker PID files will be at {}.<PID>.'.format(worker_config.filename))
log.info('Now run:\n\tanaconda worker run {}'.format(worker_config.name))
def test_register_backwards_compat_pid(self):
'''Test .workers files that when yaml loaded
will error out or not return a dict.'''
folder = WorkerConfiguration.REGISTERED_WORKERS_DIR
for f in os.listdir(folder):
os.unlink(os.path.join(folder, f))
test_cases = [
('worker1.123', ''),
('worker2.234', 'user99'), # su worker uses usernames in pid files
('worker3.1234', '{bad_dict: [abc,'), # this shouldn't happen but just in case
]
for pid_file, content in test_cases:
worker_id_pid = os.path.join(folder, pid_file)
with open(worker_id_pid, 'w') as f:
f.write(content)
worker_id_to_name = WorkerConfiguration.backwards_compat_lookup()
# the above should be len zero because folder started
# empty and added only pid files. No
# worker yaml's that have worker_id's
# in them were added. It should skip without error
# over bad or irrelevant files (non-yaml worker configs)
self.assertEqual(len(worker_id_to_name), 0)
def test_register_backwards_compat_pid(self):
'''Test .workers files that when yaml loaded
will error out or not return a dict.'''
folder = WorkerConfiguration.REGISTERED_WORKERS_DIR
for f in os.listdir(folder):
os.unlink(os.path.join(folder, f))
test_cases = [
('worker1.123', ''),
('worker2.234', 'user99'), # su worker uses usernames in pid files
('worker3.1234', '{bad_dict: [abc,'), # this shouldn't happen but just in case
]
for pid_file, content in test_cases:
worker_id_pid = os.path.join(folder, pid_file)
with open(worker_id_pid, 'w') as f:
f.write(content)
worker_id_to_name = WorkerConfiguration.backwards_compat_lookup()
# the above should be len zero because folder started
# empty and added only pid files. No
# worker yaml's that have worker_id's
# in them were added. It should skip without error
# over bad or irrelevant files (non-yaml worker configs)
self.assertEqual(len(worker_id_to_name), 0)
def print_registered_workers(bs, args):
has_workers = False
log.info('Registered workers:')
if args.queue:
user, args.queue = split_queue_arg(args.queue)
for wconfig in WorkerConfiguration.registered_workers(bs):
has_workers = True
if args.this_host_only and wconfig.hostname != WorkerConfiguration.HOSTNAME:
continue
if args.queue and args.queue != wconfig.queue:
continue
if args.org and args.org != wconfig.username:
continue
msg = '{name}, id:{worker_id}, hostname:{hostname}, queue:{username}/{queue}'.format(**wconfig.to_dict())
if wconfig.pid:
msg += ' (running with pid: {})'.format(wconfig.pid)
log.info(msg)
if not has_workers:
log.info('(No registered workers)')
def print_registered_workers(bs, args):
has_workers = False
log.info('Registered workers:')
if args.queue:
user, args.queue = split_queue_arg(args.queue)
for wconfig in WorkerConfiguration.registered_workers(bs):
has_workers = True
if args.this_host_only and wconfig.hostname != WorkerConfiguration.HOSTNAME:
continue
if args.queue and args.queue != wconfig.queue:
continue
if args.org and args.org != wconfig.username:
continue
msg = '{name}, id:{worker_id}, hostname:{hostname}, queue:{username}/{queue}'.format(
**wconfig.to_dict())
if wconfig.pid:
msg += ' (running with pid: {})'.format(wconfig.pid)
log.info(msg)
if not has_workers:
log.info('(No registered workers)')
def test_already_running(self):
wc = WorkerConfiguration(
'worker_name',
'worker_id', 'username', 'queue',
'platform', 'hostname', 'dist'
)
self.assertFalse(wc.is_running())
with wc.running():
with self.assertRaises(errors.BinstarError):
with wc.running():
pass
self.assertTrue(wc.is_running())
self.assertFalse(wc.is_running())
def main(args):
bs = get_binstar(args, cls=BinstarBuildAPI)
WorkerConfiguration.print_registered_workers(bs)