def main():
args = parse_args()
if args.tag and not args.tag.replace("-", "").isalnum():
raise argparse.ArgumentTypeError(
"Worker tag must only contain letters, numbers or hyphens."
)
# Configure logging
log_format: str = '%(asctime)s %(message)s'
if args.verbose:
log_format += ' %(pathname)s %(lineno)d'
log_level = logging.DEBUG
else:
log_level = logging.INFO
logging.basicConfig(format=log_format, level=log_level)
logging.getLogger('urllib3').setLevel(logging.INFO)
# Initialize sentry logging
if using_sentry():
initialize_sentry()
# This quits if connection unsuccessful
bundle_service = connect_to_codalab_server(args.server, args.password_file)
# Load some data into sentry
if using_sentry():
load_sentry_data(username=bundle_service._username, **vars(args))
if args.shared_file_system:
# No need to store bundles locally if filesystems are shared
local_bundles_dir = None
# Also no need to download dependencies if they're on the filesystem already
dependency_manager = None
else:
local_bundles_dir = os.path.join(args.work_dir, 'runs')
dependency_manager = DependencyManager(
os.path.join(args.work_dir, 'dependencies-state.json'),
bundle_service,
args.work_dir,
args.max_work_dir_size,
args.download_dependencies_max_retries,
)
if args.container_runtime == "singularity":
singularity_folder = os.path.join(args.work_dir, 'codalab_singularity_images')
if not os.path.exists(singularity_folder):
logger.info(
'Local singularity image location %s doesn\'t exist, creating.', singularity_folder,
)
os.makedirs(singularity_folder, 0o770)
image_manager = SingularityImageManager(
args.max_image_size, args.max_image_cache_size, singularity_folder,
)
# todo workers with singularity don't work because this is set to none -- handle this
docker_runtime = None
else:
image_manager = DockerImageManager(
os.path.join(args.work_dir, 'images-state.json'),
args.max_image_cache_size,
args.max_image_size,
)
docker_runtime = docker_utils.get_available_runtime()
# Set up local directories
if not os.path.exists(args.work_dir):
logging.debug('Work dir %s doesn\'t exist, creating.', args.work_dir)
os.makedirs(args.work_dir, 0o770)
if local_bundles_dir and not os.path.exists(local_bundles_dir):
logger.info('%s doesn\'t exist, creating.', local_bundles_dir)
os.makedirs(local_bundles_dir, 0o770)
worker = Worker(
image_manager,
dependency_manager,
# Include the worker ID in the worker state JSON path, so multiple workers
# sharing the same work directory maintain their own state.
os.path.join(args.work_dir, f'worker-state-{args.id}.json'),
args.cpuset,
args.gpuset,
args.max_memory,
args.id,
args.tag,
args.work_dir,
local_bundles_dir,
args.exit_when_idle,
args.exit_after_num_runs,
args.idle_seconds,
args.checkin_frequency_seconds,
bundle_service,
args.shared_file_system,
args.tag_exclusive,
args.group,
docker_runtime=docker_runtime,
docker_network_prefix=args.network_prefix,
pass_down_termination=args.pass_down_termination,
delete_work_dir_on_exit=args.delete_work_dir_on_exit,
exit_on_exception=args.exit_on_exception,
shared_memory_size_gb=args.shared_memory_size_gb,
preemptible=args.preemptible,
)
# Register a signal handler to ensure safe shutdown.
for sig in [signal.SIGTERM, signal.SIGINT, signal.SIGHUP]:
signal.signal(sig, lambda signup, frame: worker.signal())
# BEGIN: DO NOT CHANGE THIS LINE UNLESS YOU KNOW WHAT YOU ARE DOING
# THIS IS HERE TO KEEP TEST-CLI FROM HANGING
logger.info('Worker started!')
# END
worker.start()
def main():
args = parse_args()
# This quits if connection unsuccessful
bundle_service = connect_to_codalab_server(args.server, args.password_file)
# Configure logging
logging.basicConfig(
format='%(asctime)s %(message)s',
level=(logging.DEBUG if args.verbose else logging.INFO))
if args.shared_file_system:
# No need to store bundles locally if filesystems are shared
local_bundles_dir = None
# Also no need to download dependencies if they're on the filesystem already
dependency_manager = None
else:
local_bundles_dir = os.path.join(args.work_dir, 'runs')
dependency_manager = DependencyManager(
os.path.join(args.work_dir, 'dependencies-state.json'),
bundle_service,
args.work_dir,
args.max_work_dir_size,
)
# Set up local directories
if not os.path.exists(args.work_dir):
logging.debug('Work dir %s doesn\'t exist, creating.', args.work_dir)
os.makedirs(args.work_dir, 0o770)
if local_bundles_dir and not os.path.exists(local_bundles_dir):
logger.info('%s doesn\'t exist, creating.', local_bundles_dir)
os.makedirs(local_bundles_dir, 0o770)
docker_runtime = docker_utils.get_available_runtime()
image_manager = DockerImageManager(
os.path.join(args.work_dir, 'images-state.json'),
args.max_image_cache_size,
args.max_image_size,
)
worker = Worker(
image_manager,
dependency_manager,
os.path.join(args.work_dir, 'worker-state.json'),
args.cpuset,
args.gpuset,
args.id,
args.tag,
args.work_dir,
local_bundles_dir,
args.exit_when_idle,
args.idle_seconds,
bundle_service,
args.shared_file_system,
docker_runtime=docker_runtime,
docker_network_prefix=args.network_prefix,
)
# Register a signal handler to ensure safe shutdown.
for sig in [signal.SIGTERM, signal.SIGINT, signal.SIGHUP]:
signal.signal(sig, lambda signup, frame: worker.signal())
# BEGIN: DO NOT CHANGE THIS LINE UNLESS YOU KNOW WHAT YOU ARE DOING
# THIS IS HERE TO KEEP TEST-CLI FROM HANGING
logger.info('Worker started!')
# END
worker.start()
def main():
args = parse_args()
if args.tag and not args.tag.isalnum():
raise argparse.ArgumentTypeError(
"Worker tag must be alphanumeric (only contain letters and numbers)."
)
# Configure logging
logging.basicConfig(
format='%(asctime)s %(message)s %(pathname)s %(lineno)d',
level=(logging.DEBUG if args.verbose else logging.INFO),
)
logging.getLogger('urllib3').setLevel(logging.INFO)
# Initialize sentry logging
if using_sentry():
initialize_sentry()
# This quits if connection unsuccessful
bundle_service = connect_to_codalab_server(args.server, args.password_file)
# Load some data into sentry
if using_sentry():
load_sentry_data(username=bundle_service._username, **vars(args))
if args.shared_file_system:
# No need to store bundles locally if filesystems are shared
local_bundles_dir = None
# Also no need to download dependencies if they're on the filesystem already
dependency_manager = None
else:
local_bundles_dir = os.path.join(args.work_dir, 'runs')
dependency_manager = DependencyManager(
os.path.join(args.work_dir, 'dependencies-state.json'),
bundle_service,
args.work_dir,
args.max_work_dir_size,
)
# Set up local directories
if not os.path.exists(args.work_dir):
logging.debug('Work dir %s doesn\'t exist, creating.', args.work_dir)
os.makedirs(args.work_dir, 0o770)
if local_bundles_dir and not os.path.exists(local_bundles_dir):
logger.info('%s doesn\'t exist, creating.', local_bundles_dir)
os.makedirs(local_bundles_dir, 0o770)
docker_runtime = docker_utils.get_available_runtime()
image_manager = DockerImageManager(
os.path.join(args.work_dir, 'images-state.json'),
args.max_image_cache_size,
args.max_image_size,
)
worker = Worker(
image_manager,
dependency_manager,
os.path.join(args.work_dir, 'worker-state.json'),
args.cpuset,
args.gpuset,
args.max_memory,
args.id,
args.tag,
args.work_dir,
local_bundles_dir,
args.exit_when_idle,
args.exit_after_num_runs,
args.idle_seconds,
args.checkin_frequency_seconds,
bundle_service,
args.shared_file_system,
args.tag_exclusive,
args.group,
docker_runtime=docker_runtime,
docker_network_prefix=args.network_prefix,
pass_down_termination=args.pass_down_termination,
delete_work_dir_on_exit=args.delete_work_dir_on_exit,
exit_on_exception=args.exit_on_exception,
)
# Register a signal handler to ensure safe shutdown.
for sig in [signal.SIGTERM, signal.SIGINT, signal.SIGHUP]:
signal.signal(sig, lambda signup, frame: worker.signal())
# BEGIN: DO NOT CHANGE THIS LINE UNLESS YOU KNOW WHAT YOU ARE DOING
# THIS IS HERE TO KEEP TEST-CLI FROM HANGING
logger.info('Worker started!')
# END
worker.start()