def replace_one(i, req): # type: (int, MarkerRequirement) -> Optional[Text] try: return self._replace_one(req) except FatalSpecsResolutionError: warning('could not resolve python wheel replacement for {}'.format(req)) raise except Exception: warning('could not resolve python wheel replacement for \"{}\", ' 'using original requirements line: {}'.format(req, i)) return None
def register_instance(cls, unique_worker_id=None, worker_name=None, api_client=None, allow_double=False): """ # Exit the process if another instance of us is using the same worker_id :param unique_worker_id: if already exists, return negative :param worker_name: slot number will be added to worker name, based on the available instance slot :return: (str worker_id, int slot_number) Return None value on instance already running """ # try to lock file lock_file = os.path.join(cls._get_temp_folder(), cls._lock_file_name) timeout = 0 while os.path.exists(lock_file): if timeout > cls._lock_timeout: warning('lock file timed out {}sec - clearing lock'.format( cls._lock_timeout)) try: os.remove(lock_file) except Exception: pass break sleep(1) timeout += 1 with open(lock_file, 'wb') as f: f.write(bytes(os.getpid())) f.flush() try: ret = cls._register_instance(unique_worker_id=unique_worker_id, worker_name=worker_name, api_client=api_client, allow_double=allow_double) except: ret = None, None try: os.remove(lock_file) except Exception: pass return ret
def __call__(self, parser, namespace, values, option_string=None): warning('argument "{}" is deprecated'.format(option_string))
def _register_instance(cls, unique_worker_id=None, worker_name=None, api_client=None, allow_double=False): if cls.worker_id: return cls.worker_id, cls.instance_slot # make sure we have a unique name instance_num = 0 temp_folder = cls._get_temp_folder() files = glob( os.path.join(temp_folder, cls.prefix + cls.sep + '*' + cls.ext)) slots = {} for file in files: parts = file.split(cls.sep) try: pid = int(parts[1]) except Exception: # something is wrong, use non existing pid and delete the file pid = -1 uid, slot = None, None try: with open(file, 'r') as f: uid, slot = str(f.read()).split('\n') slot = int(slot) except Exception: pass worker = None if api_client and ENV_DOCKER_HOST_MOUNT.get() and uid: try: worker = [ w for w in api_client.workers.get_all() if w.id == uid ] except Exception: worker = None # count active instances and delete dead files if not worker and not psutil.pid_exists(pid): # delete the file try: os.remove(os.path.join(file)) except Exception: pass continue instance_num += 1 if slot is None: continue if uid == unique_worker_id: if allow_double: warning( 'Instance with the same WORKER_ID [{}] was found on this machine. ' 'We are ignoring it, make sure this not a mistake.'. format(unique_worker_id)) else: return None, None slots[slot] = uid # get a new slot if not slots: cls.instance_slot = 0 else: # guarantee we have the minimal slot possible for i in range(max(slots.keys()) + 2): if i not in slots: cls.instance_slot = i break # build worker id based on slot if not unique_worker_id: unique_worker_id = worker_name + cls.worker_name_sep + str( cls.instance_slot) # create lock cls._pid = str(os.getpid()) cls._pid_file = NamedTemporaryFile(dir=cls._get_temp_folder(), prefix=cls.prefix + cls.sep + cls._pid + cls.sep, suffix=cls.ext) cls._pid_file.write(('{}\n{}'.format(unique_worker_id, cls.instance_slot)).encode()) cls._pid_file.flush() cls.worker_id = unique_worker_id return cls.worker_id, cls.instance_slot