def hydra_main(overrides: List[Any]):
######################################################################################
# DO NOT MOVE THIS IMPORT TO TOP LEVEL: submitit processes will not be initialized
# correctly (MKL_THREADING_LAYER will be set to INTEL instead of GNU)
######################################################################################
from vissl.hooks import default_hook_generator
######################################################################################
print(f"####### overrides: {overrides}")
with initialize_config_module(config_module="vissl.config"):
cfg = compose("defaults", overrides=overrides)
args, config = convert_to_attrdict(cfg)
if config.SLURM.USE_SLURM:
assert (
is_submitit_available()
), "Please 'pip install submitit' to schedule jobs on SLURM"
launch_distributed_on_slurm(engine_name=args.engine_name, cfg=config)
else:
launch_distributed(
cfg=config,
node_id=args.node_id,
engine_name=args.engine_name,
hook_generator=default_hook_generator,
)
def main(args: Namespace, config: AttrDict):
if config.SLURM.USE_SLURM:
assert (
is_submitit_available()
), "Please 'pip install submitit' to schedule jobs on SLURM"
extract_features_and_run_knn_on_slurm(config)
else:
extract_features_and_run_knn(args.node_id, config)
executor.update_parameters(
name=slurm_options.NAME,
slurm_comment=slurm_options.COMMENT,
slurm_partition=slurm_options.PARTITION,
slurm_constraint=slurm_options.CONSTRAINT,
timeout_min=slurm_options.TIMEOUT_MIN,
nodes=1,
cpus_per_task=slurm_options.CPUS_PER_TASK,
tasks_per_node=1,
mem_gb=slurm_options.MEM_GB,
slurm_additional_parameters=slurm_options.ADDITIONAL_PARAMETERS,
)
job = executor.submit(benchmark_suite_scheduler_job)
print(f"SUBMITTED EVALUATION JOB: {job.job_id}")
if __name__ == "__main__":
"""
Example usage:
python -u "./vissl/engines/benchmark_suite_scheduler.py" \
"/path/to/benchmark_suite_scheduler_example.json"
"""
assert_hydra_dependency()
assert (is_submitit_available()
), "Please 'pip install submitit' to schedule jobs on SLURM"
config_file = sys.argv[1]
launch_benchmark_suite_scheduler(config_file)