def new_experiment(kwargs):
args = kwargs["args"]
sample_idx = str(kwargs["idx"])
test_name = os.path.basename(args.test_config).split(".yaml")[0]
while True:
with FileLock(test_name + GPU_IN_USE, timeout=np.inf, delay=1) as lock:
gpu_usage = pkl.load(open(test_name + GPU_IN_USE, "rb"))
found_gpu = False
for k, v in gpu_usage.items():
if v < args.exp_per_gpu:
gpu_usage[k] += 1
gpu_id = k
found_gpu = True
break
if found_gpu:
pkl.dump(gpu_usage, open(test_name + GPU_IN_USE, "wb"))
break
else:
time.sleep(1)
job_env = os.environ.copy()
job_env["CUDA_VISIBLE_DEVICES"] = gpu_id
mem_ratio = 1 / args.exp_per_gpu
print(" [*] Submitting a sample {} to GPU id = {}".format(
sample_idx, gpu_id))
# So run subprocess.Popen, which creates legit process, which has access to GPUs
FNULL = open(os.devnull, 'w') # suppress all outputs
proc = subprocess.Popen([
"python",
"test.py",
"--test-config",
args.test_config,
"--model-config",
args.model_config,
"--inv-start-idx",
sample_idx,
"--try-restrict-memory",
str(mem_ratio),
],
env=job_env,
stdout=FNULL,
stderr=subprocess.PIPE)
output, error = proc.communicate() # Blocking
if proc.returncode != 0:
print("[!] Error at sample_id {} (return {}):".format(
sample_idx, proc.returncode))
print(error.decode("utf-8"))
with FileLock(test_name + ERROR_LOGS, timeout=np.inf, delay=1) as lock:
with open(test_name + ERROR_LOGS, "a") as f:
f.write("[!] Error at sample_id {}\n".format(sample_idx))
f.write(error.decode("utf-8") + "\n")
f.write("\n")
with FileLock(test_name + GPU_IN_USE, timeout=np.inf, delay=1) as lock:
gpu_usage = pkl.load(open(test_name + GPU_IN_USE, "rb"))
gpu_usage[gpu_id] -= 1
pkl.dump(gpu_usage, open(test_name + GPU_IN_USE, "wb"))
def check_update(ref_name, old_rev, new_rev):
"""General handler of the given update.
Raises InvalidUpdate if the update cannot be accepted (usually
because one of the commits fails a style-check, for instance).
PARAMETERS
ref_name: The name of the reference being update (Eg:
refs/heads/master).
old_rev: The commit SHA1 of the reference before the update.
new_rev: The new commit SHA1 that the reference will point to
if the update is accepted.
REMARKS
This function assumes that scratch_dir has been initialized.
"""
debug('check_update(ref_name=%s, old_rev=%s, new_rev=%s)'
% (ref_name, old_rev, new_rev),
level=2)
update_cls = new_update(ref_name, old_rev, new_rev, git_show_ref(),
submitter_email=None)
if update_cls is None:
raise InvalidUpdate(
"This type of update (%s,%s) is currently unsupported."
% (ref_name, get_object_type(new_rev)))
with FileLock('git-hooks::update.token'):
update_cls.validate()
def modifiableSingleton(cls, filepath):
"""Allocates, as necessary, and returns a modifiable, singleton
Configuration instance for the specified filepath. Separate entities can
thus share one in memory copy of the configuration file allowing for
encapsulation of per-entity manipulation of the configuration.
Args:
filepath (str): path to config file
"""
config = None
with FileLock(cls.singletonLock, "r+") as f:
config = cls.modifiableSingltons.get(filepath)
if config is None:
config = Configuration(filepath, readonly=False)
cls.modifiableSingltons[filepath] = config
return config
def _removeFile(self):
"""Deletes the current configuration file.
In noRun mode, pretend that we're doing an rm of the file."""
if Command.noRunMode():
runCommand(['rm', self.filepath])
return
if os.path.exists(self.filepath):
os.remove(self.filepath)
self._fsyncDirectory()
try:
with FileLock(self.singletonLock, "r+") as f:
del Configuration.modifiableSingltons[self.filepath]
except KeyError:
pass
def check_update(ref_name, old_rev, new_rev):
"""General handler of the given update.
Raises InvalidUpdate if the update cannot be accepted (usually
because one of the commits fails a style-check, for instance).
PARAMETERS
ref_name: The name of the reference being update (Eg:
refs/heads/master).
old_rev: The commit SHA1 of the reference before the update.
new_rev: The new commit SHA1 that the reference will point to
if the update is accepted.
REMARKS
This function assumes that scratch_dir has been initialized.
"""
debug(
"check_update(ref_name=%s, old_rev=%s, new_rev=%s)"
% (ref_name, old_rev, new_rev),
level=2,
)
check_minimum_system_requirements()
# Do nothing if the reference is in the hooks.ignore-refs list.
ignore_refs_match = utils.search_config_option_list("hooks.ignore-refs", ref_name)
if ignore_refs_match is not None:
debug(f"{ref_name} ignored due to hooks.ignore-refs" f" ({ignore_refs_match})")
return
update_cls = new_update(
ref_name, old_rev, new_rev, git_show_ref(), submitter_email=None
)
if update_cls is None:
# Report an error. We could look more precisely into what
# might be the reason behind this error, and print more precise
# diagnostics, but it does not seem like this would be worth
# the effort: It requires some pretty far-fetched scenarios
# for this to trigger; so, this should happen only very seldomly,
# and when a user does something very unusual.
raise InvalidUpdate(
"This type of update (%s,%s) is not valid."
% (ref_name, get_object_type(new_rev))
)
with FileLock("git-hooks::update.token"):
update_cls.validate()
maybe_update_hook(ref_name, old_rev, new_rev)
def __init__(self, env):
self.env = env
self.lock = FileLock(self.env.lock_file())
def __init__(self, env):
self.env = env
self._user_info = []
self._update_task = gevent.spawn(self._update)
self.file_lock = FileLock(self.env.lock_file())