def new_experiment(kwargs):
args = kwargs["args"]
sample_idx = str(kwargs["idx"])
test_name = os.path.basename(args.test_config).split(".yaml")[0]
while True:
with FileLock(test_name + GPU_IN_USE, timeout=np.inf, delay=1) as lock:
gpu_usage = pkl.load(open(test_name + GPU_IN_USE, "rb"))
found_gpu = False
for k, v in gpu_usage.items():
if v < args.exp_per_gpu:
gpu_usage[k] += 1
gpu_id = k
found_gpu = True
break
if found_gpu:
pkl.dump(gpu_usage, open(test_name + GPU_IN_USE, "wb"))
break
else:
time.sleep(1)
job_env = os.environ.copy()
job_env["CUDA_VISIBLE_DEVICES"] = gpu_id
mem_ratio = 1 / args.exp_per_gpu
print(" [*] Submitting a sample {} to GPU id = {}".format(
sample_idx, gpu_id))
# So run subprocess.Popen, which creates legit process, which has access to GPUs
FNULL = open(os.devnull, 'w') # suppress all outputs
proc = subprocess.Popen([
"python",
"test.py",
"--test-config",
args.test_config,
"--model-config",
args.model_config,
"--inv-start-idx",
sample_idx,
"--try-restrict-memory",
str(mem_ratio),
],
env=job_env,
stdout=FNULL,
stderr=subprocess.PIPE)
output, error = proc.communicate() # Blocking
if proc.returncode != 0:
print("[!] Error at sample_id {} (return {}):".format(
sample_idx, proc.returncode))
print(error.decode("utf-8"))
with FileLock(test_name + ERROR_LOGS, timeout=np.inf, delay=1) as lock:
with open(test_name + ERROR_LOGS, "a") as f:
f.write("[!] Error at sample_id {}\n".format(sample_idx))
f.write(error.decode("utf-8") + "\n")
f.write("\n")
with FileLock(test_name + GPU_IN_USE, timeout=np.inf, delay=1) as lock:
gpu_usage = pkl.load(open(test_name + GPU_IN_USE, "rb"))
gpu_usage[gpu_id] -= 1
pkl.dump(gpu_usage, open(test_name + GPU_IN_USE, "wb"))
class UserInfoMgr(object):
USER_INFO_UPDATE_INTERVAL = 5
def __init__(self, env):
self.env = env
self._user_info = []
self._update_task = gevent.spawn(self._update)
self.file_lock = FileLock(self.env.lock_file())
def user_info(self):
return self._user_info
def _update(self):
while True:
try:
self.file_lock.lock()
f = open(self.env.user_info_file())
s = f.read()
if len(s) != 0:
self._user_info = pickle.loads(s)
f.close()
self.file_lock.unlock()
except IOError:
os.mknod(self.env.lock_file())
time.sleep(UserInfoMgr.USER_INFO_UPDATE_INTERVAL)
class UserInfoExport(object):
def __init__(self, env):
self.env = env
self.lock = FileLock(self.env.lock_file())
def export(self, user_info):
self.lock.lock()
f = open(self.env.user_info_file(), "w")
pickle.dump(user_info, f)
f.close()
self.lock.unlock()
def check_update(ref_name, old_rev, new_rev):
"""General handler of the given update.
Raises InvalidUpdate if the update cannot be accepted (usually
because one of the commits fails a style-check, for instance).
PARAMETERS
ref_name: The name of the reference being update (Eg:
refs/heads/master).
old_rev: The commit SHA1 of the reference before the update.
new_rev: The new commit SHA1 that the reference will point to
if the update is accepted.
REMARKS
This function assumes that scratch_dir has been initialized.
"""
debug('check_update(ref_name=%s, old_rev=%s, new_rev=%s)'
% (ref_name, old_rev, new_rev),
level=2)
update_cls = new_update(ref_name, old_rev, new_rev, git_show_ref(),
submitter_email=None)
if update_cls is None:
raise InvalidUpdate(
"This type of update (%s,%s) is currently unsupported."
% (ref_name, get_object_type(new_rev)))
with FileLock('git-hooks::update.token'):
update_cls.validate()
def modifiableSingleton(cls, filepath):
"""Allocates, as necessary, and returns a modifiable, singleton
Configuration instance for the specified filepath. Separate entities can
thus share one in memory copy of the configuration file allowing for
encapsulation of per-entity manipulation of the configuration.
Args:
filepath (str): path to config file
"""
config = None
with FileLock(cls.singletonLock, "r+") as f:
config = cls.modifiableSingltons.get(filepath)
if config is None:
config = Configuration(filepath, readonly=False)
cls.modifiableSingltons[filepath] = config
return config
def _removeFile(self):
"""Deletes the current configuration file.
In noRun mode, pretend that we're doing an rm of the file."""
if Command.noRunMode():
runCommand(['rm', self.filepath])
return
if os.path.exists(self.filepath):
os.remove(self.filepath)
self._fsyncDirectory()
try:
with FileLock(self.singletonLock, "r+") as f:
del Configuration.modifiableSingltons[self.filepath]
except KeyError:
pass
def check_update(ref_name, old_rev, new_rev):
"""General handler of the given update.
Raises InvalidUpdate if the update cannot be accepted (usually
because one of the commits fails a style-check, for instance).
PARAMETERS
ref_name: The name of the reference being update (Eg:
refs/heads/master).
old_rev: The commit SHA1 of the reference before the update.
new_rev: The new commit SHA1 that the reference will point to
if the update is accepted.
REMARKS
This function assumes that scratch_dir has been initialized.
"""
debug(
"check_update(ref_name=%s, old_rev=%s, new_rev=%s)"
% (ref_name, old_rev, new_rev),
level=2,
)
check_minimum_system_requirements()
# Do nothing if the reference is in the hooks.ignore-refs list.
ignore_refs_match = utils.search_config_option_list("hooks.ignore-refs", ref_name)
if ignore_refs_match is not None:
debug(f"{ref_name} ignored due to hooks.ignore-refs" f" ({ignore_refs_match})")
return
update_cls = new_update(
ref_name, old_rev, new_rev, git_show_ref(), submitter_email=None
)
if update_cls is None:
# Report an error. We could look more precisely into what
# might be the reason behind this error, and print more precise
# diagnostics, but it does not seem like this would be worth
# the effort: It requires some pretty far-fetched scenarios
# for this to trigger; so, this should happen only very seldomly,
# and when a user does something very unusual.
raise InvalidUpdate(
"This type of update (%s,%s) is not valid."
% (ref_name, get_object_type(new_rev))
)
with FileLock("git-hooks::update.token"):
update_cls.validate()
maybe_update_hook(ref_name, old_rev, new_rev)
def __init__(self, env):
self.env = env
self.lock = FileLock(self.env.lock_file())
def POST(self, request):
logging.debug("proxy.post")
proxy_nodes = []
storage_nodes = []
self.get_nodes(proxy_nodes, storage_nodes)
print("storage_node: %s" % storage_nodes)
print("proxy_node: %s" % proxy_nodes)
#request.showVariables()
response = {"headers": [], "status": "201 OK", "body": ""}
if not request.file_name:
response['status'] = http_code['400']
return response
if not self.group.is_leader:
response['status'] = http_code['304']
response['body'] = "No leader. Try to find the leader"
logging.debug("I'm no leader. Try to find the leader")
return response
#client_id = request.metadata.get("HTTP_CLIENT_ID", "test:test")
#logging.debug("HTTP_CLIENT_ID: %s" % client_id)
replicas_count = request.metadata["HTTP_REPLICAS_COUNT"]
metadata = MetadataManager(self.data_path + "/" + request.file_name,
request.file_name,
request.metadata["HTTP_DATA_LENGTH"],
replicas_count,
request.metadata["HTTP_ETAG"])
if not metadata.restore():
response['status'] = http_code['304']
response['body'] = "File does not exist"
logging.debug("File does not exist")
return response
logging.debug(self.data_path + "/" + request.file_name)
fl = FileLock(self.data_path + "/" + request.file_name)
if metadata.lock_list:
metadata.lock_list.pop()
metadata.save()
fl.unlock()
logging.debug("self.proxy_nodes: %s" % self.proxy_nodes)
for proxy_replicas in proxy_nodes:
if proxy_replicas[0] == self.port_ip[0] and\
proxy_replicas[1] == self.port_ip[1]:
continue
headers = {}
headers['file_name'] = request.file_name
headers['data_length'] = request.content_length
headers['etag'] = request.file_name
headers['replicas_count'] = replicas_count
headers['client_id'] = "%s:%s" % (request.remote_addr,
request.remote_port)
headers["replication"] = "1"
metadata_copy = json.dumps(metadata.resource_dict)
response = self._put_request(request.file_name,
proxy_replicas,
data=metadata_copy,
headers=headers)
logging.debug("response: %s" % response)
return response
def __init__(self, env):
self.env = env
self._user_info = []
self._update_task = gevent.spawn(self._update)
self.file_lock = FileLock(self.env.lock_file())