def __init__(self, id, tag, work_dir, max_work_dir_size_bytes,
max_images_bytes, shared_file_system, slots, bundle_service,
docker):
self.id = id
self._tag = tag
self.shared_file_system = shared_file_system
self._bundle_service = bundle_service
self._docker = docker
self._slots = slots
self._worker_state_manager = WorkerStateManager(
work_dir, self.shared_file_system)
if not self.shared_file_system:
# Manages which dependencies are available.
self._dependency_manager = DependencyManager(
work_dir, max_work_dir_size_bytes,
self._worker_state_manager.previous_runs.keys())
self._image_manager = DockerImageManager(self._docker, work_dir,
max_images_bytes)
self._max_images_bytes = max_images_bytes
self._exiting_lock = threading.Lock()
self._exiting = False
self._should_upgrade = False
self._last_checkin_successful = False
def __init__(self, worker_id, tag, work_dir,
max_work_dir_size_bytes, max_dependencies_serialized_length,
shared_file_system, bundle_service, create_run_manager):
self.id = worker_id
self._tag = tag
self.shared_file_system = shared_file_system
self._bundle_service = bundle_service
self._resource_lock = threading.Lock() # lock for cpuset and gpuset
self._run_manager = create_run_manager(self)
self._worker_state_manager = WorkerStateManager(
work_dir=work_dir,
run_manager=self._run_manager,
shared_file_system=self.shared_file_system
)
if not self.shared_file_system:
# Manages which dependencies are available.
self._dependency_manager = DependencyManager(
work_dir,
max_work_dir_size_bytes,
max_dependencies_serialized_length,
self._worker_state_manager.previous_runs.keys()
)
self._exiting_lock = threading.Lock()
self._exiting = False
self._should_upgrade = False
self._last_checkin_successful = False
def __init__(self,
id,
tag,
work_dir,
cpuset,
gpuset,
max_work_dir_size_bytes,
max_dependencies_serialized_length,
max_images_bytes,
shared_file_system,
bundle_service,
docker,
docker_network_prefix='codalab_worker_network'):
self.id = id
self._tag = tag
self.shared_file_system = shared_file_system
self._bundle_service = bundle_service
self._docker = docker
self._docker_network_prefix = docker_network_prefix
self._resource_lock = threading.Lock() # lock for cpuset and gpuset
self._cpuset = cpuset
self._gpuset = gpuset
self._cpuset_free = set(
self._cpuset) # make a copy of self._cpuset as initial value
self._gpuset_free = set(
self._gpuset) # make a copy of self._gpuset as initial value
self._worker_state_manager = WorkerStateManager(
work_dir, self.shared_file_system)
if not self.shared_file_system:
# Manages which dependencies are available.
self._dependency_manager = DependencyManager(
work_dir, max_work_dir_size_bytes,
max_dependencies_serialized_length,
self._worker_state_manager.previous_runs.keys())
self._image_manager = DockerImageManager(self._docker, work_dir,
max_images_bytes)
self._max_images_bytes = max_images_bytes
self._exiting_lock = threading.Lock()
self._exiting = False
self._should_upgrade = False
self._last_checkin_successful = False
# set up docker networks for running bundles: one with external network access and one without
self.docker_network_external_name = self._docker_network_prefix + "_ext"
if self.docker_network_external_name not in self._docker.list_networks(
):
logger.debug('Creating docker network: {}'.format(
self.docker_network_external_name))
self._docker.create_network(self.docker_network_external_name,
internal=False)
else:
logger.debug(
'Docker network already exists, not creating: {}'.format(
self.docker_network_external_name))
self.docker_network_internal_name = self._docker_network_prefix + "_int"
if self.docker_network_internal_name not in self._docker.list_networks(
):
logger.debug('Creating docker network: {}'.format(
self.docker_network_internal_name))
self._docker.create_network(self.docker_network_internal_name)
else:
logger.debug(
'Docker network already exists, not creating: {}'.format(
self.docker_network_internal_name))