def _register_instance(cls, unique_worker_id=None, worker_name=None, api_client=None, allow_double=False):
if cls.worker_id:
return cls.worker_id, cls.instance_slot
# make sure we have a unique name
instance_num = 0
slots = {}
for pid, uid, slot, file in cls.get_running_pids():
worker = None
if api_client and ENV_DOCKER_HOST_MOUNT.get() and uid:
try:
worker = [w for w in api_client.workers.get_all() if w.id == uid]
except Exception:
worker = None
# count active instances and delete dead files
if not worker and pid < 0:
# delete the file
try:
os.remove(os.path.join(file))
except Exception:
pass
continue
instance_num += 1
if slot is None:
continue
if uid == unique_worker_id:
if allow_double:
warning('Instance with the same WORKER_ID [{}] was found on this machine. '
'We are ignoring it, make sure this not a mistake.'.format(unique_worker_id))
else:
return None, None
slots[slot] = uid
# get a new slot
if not slots:
cls.instance_slot = 0
else:
# guarantee we have the minimal slot possible
for i in range(max(slots.keys())+2):
if i not in slots:
cls.instance_slot = i
break
# build worker id based on slot
if not unique_worker_id:
unique_worker_id = worker_name + cls.worker_name_sep + str(cls.instance_slot)
# create lock
cls._pid = str(os.getpid())
cls._pid_file = NamedTemporaryFile(
dir=cls._get_temp_folder(), prefix=cls.prefix + cls.sep + cls._pid + cls.sep, suffix=cls.ext)
cls._pid_file.write(('{}\n{}'.format(unique_worker_id, cls.instance_slot)).encode())
cls._pid_file.flush()
cls.worker_id = unique_worker_id
return cls.worker_id, cls.instance_slot
def replace_one(i, req):
# type: (int, MarkerRequirement) -> Optional[Text]
try:
return self._replace_one(req)
except FatalSpecsResolutionError:
warning(
'could not resolve python wheel replacement for {}'.format(
req))
raise
except Exception:
warning(
'could not resolve python wheel replacement for \"{}\", '
'using original requirements line: {}'.format(req, i))
return None
def register_instance(cls,
unique_worker_id=None,
worker_name=None,
api_client=None,
allow_double=False):
"""
# Exit the process if another instance of us is using the same worker_id
:param unique_worker_id: if already exists, return negative
:param worker_name: slot number will be added to worker name, based on the available instance slot
:return: (str worker_id, int slot_number) Return None value on instance already running
"""
# try to lock file
lock_file = cls.get_lock_filename()
timeout = 0
while os.path.exists(lock_file):
if timeout > cls._lock_timeout:
warning('lock file timed out {}sec - clearing lock'.format(
cls._lock_timeout))
try:
os.remove(lock_file)
except Exception:
pass
break
sleep(1)
timeout += 1
with open(lock_file, 'wb') as f:
f.write(bytes(os.getpid()))
f.flush()
try:
ret = cls._register_instance(unique_worker_id=unique_worker_id,
worker_name=worker_name,
api_client=api_client,
allow_double=allow_double)
except:
ret = None, None
try:
os.remove(lock_file)
except Exception:
pass
return ret
def __call__(self, parser, namespace, values, option_string=None):
warning('argument "{}" is deprecated'.format(option_string))