def __init__(self, commit_file, bundle_service, worker_dir, max_cache_size_bytes):
super(DependencyManager, self).__init__()
self.add_transition(DependencyStage.DOWNLOADING, self._transition_from_DOWNLOADING)
self.add_terminal(DependencyStage.READY)
self.add_terminal(DependencyStage.FAILED)
self._state_committer = JsonStateCommitter(commit_file)
self._bundle_service = bundle_service
self._max_cache_size_bytes = max_cache_size_bytes
self.dependencies_dir = os.path.join(worker_dir, DependencyManager.DEPENDENCIES_DIR_NAME)
if not os.path.exists(self.dependencies_dir):
logger.info('{} doesn\'t exist, creating.'.format(self.dependencies_dir))
os.makedirs(self.dependencies_dir, 0o770)
# Locks for concurrency
self._dependency_locks = dict() # type: Dict[DependencyKey, threading.RLock]
self._global_lock = threading.RLock() # Used for add/remove actions
self._paths_lock = threading.RLock() # Used for path name computations
# File paths that are currently being used to store dependencies. Used to prevent conflicts
self._paths = set()
# DependencyKey -> DependencyState
self._dependencies = dict()
# DependencyKey -> WorkerThread(thread, success, failure_message)
self._downloading = ThreadDict(fields={'success': False, 'failure_message': None})
self._load_state()
# Sync states between dependency-state.json and dependency directories on the local file system.
self._sync_state()
self._stop = False
self._main_thread = None
def __init__(self, commit_file: str, max_image_cache_size: int,
max_image_size: int):
"""
Initializes a DockerImageManager
:param commit_file: String path to where the state file should be committed
:param max_image_cache_size: Total size in bytes that the image cache can use
:param max_image_size: Total size in bytes that the image can have
"""
super().__init__(max_image_size, max_image_cache_size)
self._state_committer = JsonStateCommitter(
commit_file) # type: JsonStateCommitter
self._docker = docker.from_env(
timeout=DEFAULT_DOCKER_TIMEOUT) # type: DockerClient
def __init__(self, commit_file, max_image_cache_size, max_image_size):
"""
Initializes a DockerImageManager
:param commit_file: String path to where the state file should be committed
:param max_image_cache_size: Total size in bytes that the image cache can use
:param max_image_size: Total size in bytes that the image can have
"""
self._state_committer = JsonStateCommitter(commit_file) # type: JsonStateCommitter
self._docker = docker.from_env(timeout=DEFAULT_DOCKER_TIMEOUT) # type: DockerClient
self._downloading = ThreadDict(
fields={'success': False, 'status': 'Download starting'}, lock=True
)
self._max_image_cache_size = max_image_cache_size
self._max_image_size = max_image_size
self._stop = False
self._sleep_secs = 10
self._cleanup_thread = None
def __init__(
self,
commit_file: str,
bundle_service: BundleServiceClient,
worker_dir: str,
max_cache_size_bytes: int,
download_dependencies_max_retries: int,
):
super(DependencyManager, self).__init__()
self.add_transition(DependencyStage.DOWNLOADING, self._transition_from_DOWNLOADING)
self.add_terminal(DependencyStage.READY)
self.add_terminal(DependencyStage.FAILED)
self._id: str = "worker-dependency-manager-{}".format(uuid.uuid4().hex[:8])
self._state_committer = JsonStateCommitter(commit_file)
self._bundle_service = bundle_service
self._max_cache_size_bytes = max_cache_size_bytes
self.dependencies_dir = os.path.join(worker_dir, DependencyManager.DEPENDENCIES_DIR_NAME)
self._download_dependencies_max_retries = download_dependencies_max_retries
if not os.path.exists(self.dependencies_dir):
logger.info('{} doesn\'t exist, creating.'.format(self.dependencies_dir))
os.makedirs(self.dependencies_dir, 0o770)
# Create a lock for concurrency over NFS
# Create a separate locks directory to hold the lock files.
# Each lock file is created when a process tries to claim the main lock.
locks_claims_dir: str = os.path.join(worker_dir, 'locks_claims')
try:
os.makedirs(locks_claims_dir)
except FileExistsError:
logger.info(f"A locks directory at {locks_claims_dir} already exists.")
self._state_lock = NFSLock(os.path.join(locks_claims_dir, 'state.lock'))
# File paths that are currently being used to store dependencies. Used to prevent conflicts
self._paths: Set[str] = set()
# DependencyKey -> WorkerThread(thread, success, failure_message)
self._downloading = ThreadDict(fields={'success': False, 'failure_message': None})
# Sync states between dependency-state.json and dependency directories on the local file system.
self._sync_state()
self._stop = False
self._main_thread = None
logger.info(f"Initialized Dependency Manager with ID: {self._id}")
def __init__(
self,
worker, # type: Worker
image_manager, # type: DockerImageManager
dependency_manager, # type: LocalFileSystemDependencyManager
commit_file, # type: str
cpuset, # type: Set[str]
gpuset, # type: Set[str]
work_dir, # type: str
docker_runtime=docker_utils.DEFAULT_RUNTIME, # type: str
docker_network_prefix='codalab_worker_network', # type: str
):
self._worker = worker
self._state_committer = JsonStateCommitter(commit_file)
self._reader = LocalReader()
self._docker = docker.from_env()
self._bundles_dir = os.path.join(work_dir, LocalRunManager.BUNDLES_DIR_NAME)
if not os.path.exists(self._bundles_dir):
logger.info('{} doesn\'t exist, creating.'.format(self._bundles_dir))
os.makedirs(self._bundles_dir, 0o770)
self._image_manager = image_manager
self._dependency_manager = dependency_manager
self._cpuset = cpuset
self._gpuset = gpuset
self._stop = False
self._work_dir = work_dir
self._runs = {} # bundle_uuid -> LocalRunState
self._lock = threading.RLock()
self._init_docker_networks(docker_network_prefix)
self._run_state_manager = LocalRunStateMachine(
docker_image_manager=self._image_manager,
dependency_manager=self._dependency_manager,
worker_docker_network=self.worker_docker_network,
docker_network_internal=self.docker_network_internal,
docker_network_external=self.docker_network_external,
docker_runtime=docker_runtime,
upload_bundle_callback=self._worker.upload_bundle_contents,
assign_cpu_and_gpu_sets_fn=self.assign_cpu_and_gpu_sets,
)
def __init__(self, commit_file, max_image_cache_size):
"""
Initializes a DockerImageManager
:param commit_file: String path to where the state file should be committed
:param max_image_cache_size: Total size in bytes that the image cache can use
"""
self._state_committer = JsonStateCommitter(
commit_file) # type: JsonStateCommitter
self._docker = docker.from_env() # type: DockerClient
self._image_cache = {} # type: Dict[str, ImageCacheEntry]
self._downloading = ThreadDict(fields={
'success': False,
'status': 'Download starting.'
},
lock=True)
self._max_image_cache_size = max_image_cache_size
self._lock = threading.RLock()
self._stop = False
self._sleep_secs = 10
self._cleanup_thread = None
self._load_state()
def setUp(self):
self.test_dir = tempfile.mkdtemp()
self.state_file = 'test-state.json'
self.state_path = os.path.join(self.test_dir, self.state_file)
self.committer = JsonStateCommitter(self.state_path)