def _monitor_hanging_pods_daemon(self):
while True:
output = get_bash_output(
'kubectl get pods -n {namespace} -o=JSON'.format(
namespace=self.namespace))
output = '' if not output else output if isinstance(
output, str) else output.decode('utf-8')
try:
output_config = json.loads(output)
except Exception as ex:
self.log.warning(
'K8S Glue pods monitor: Failed parsing kubectl output:\n{}\nEx: {}'
.format(output, ex))
sleep(self._polling_interval)
continue
pods = output_config.get('items', [])
for pod in pods:
try:
reason = functools.reduce(
lambda a, b: a[b], ('status', 'containerStatuses', 0,
'state', 'waiting', 'reason'), pod)
except (IndexError, KeyError):
continue
if reason == 'ImagePullBackOff':
pod_name = pod.get('metadata', {}).get('name', None)
if pod_name:
task_id = pod_name.rpartition('-')[-1]
delete_pod_cmd = 'kubectl delete pods {} -n {}'.format(
pod_name, self.namespace)
get_bash_output(delete_pod_cmd)
try:
self._session.api_client.tasks.failed(
task=task_id,
status_reason=
"K8S glue error due to ImagePullBackOff",
status_message="Changed by K8S glue",
force=True)
except Exception as ex:
self.log.warning(
'K8S Glue pods monitor: Failed deleting task "{}"\nEX: {}'
.format(task_id, ex))
sleep(self._polling_interval)
def run_tasks_loop(self, queues: List[Text], worker_params, **kwargs):
"""
:summary: Pull and run tasks from queues.
:description: 1. Go through ``queues`` by order.
2. Try getting the next task for each and run the first one that returns.
3. Go to step 1
:param queues: IDs of queues to pull tasks from
:type queues: list of ``Text``
:param worker_params: Worker command line arguments
:type worker_params: ``clearml_agent.helper.process.WorkerParams``
"""
events_service = self.get_service(Events)
# make sure we have a k8s pending queue
# noinspection PyBroadException
try:
self._session.api_client.queues.create(self.k8s_pending_queue_name)
except Exception:
pass
# get queue id
self.k8s_pending_queue_name = self._resolve_name(self.k8s_pending_queue_name, "queues")
_last_machine_update_ts = 0
while True:
# iterate over queues (priority style, queues[0] is highest)
for queue in queues:
# delete old completed / failed pods
get_bash_output(self.KUBECTL_DELETE_CMD.format(namespace=self.namespace))
# get next task in queue
try:
response = self._session.api_client.queues.get_next_task(queue=queue)
except Exception as e:
print("Warning: Could not access task queue [{}], error: {}".format(queue, e))
continue
else:
try:
task_id = response.entry.task
except AttributeError:
print("No tasks in queue {}".format(queue))
continue
events_service.send_log_events(
self.worker_id,
task_id=task_id,
lines="task {} pulled from {} by worker {}".format(
task_id, queue, self.worker_id
),
level="INFO",
)
self.report_monitor(ResourceMonitor.StatusReport(queues=queues, queue=queue, task=task_id))
self.run_one_task(queue, task_id, worker_params)
self.report_monitor(ResourceMonitor.StatusReport(queues=self.queues))
break
else:
# sleep and retry polling
print("No tasks in Queues, sleeping for {:.1f} seconds".format(self._polling_interval))
sleep(self._polling_interval)
if self._session.config["agent.reload_config"]:
self.reload_config()
def old_replace(session, line):
try:
cuda_ver_suffix, cuda_ver, cuda_cudnn_ver = old_get_suffix(session)
except Exception:
return line
if line.lstrip().startswith('#'):
return line
for package_name in _cuda_based_packages_hack:
if package_name not in line:
continue
try:
line_lstrip = line.lstrip()
if line_lstrip.startswith('http://') or line_lstrip.startswith(
'https://'):
pos = line.find(package_name) + len(package_name)
# patch line with specific version
line = line[:pos] + \
line[pos:].replace('-cp', cuda_ver_suffix + '-cp', 1)
else:
# this is a pypi package
tokens = line.replace('=', ' ').replace('<', ' ').replace('>', ' ').replace(';', ' '). \
replace('!', ' ').split()
if package_name != tokens[0]:
# how did we get here, probably a mistake
found_cuda_based_package = False
continue
version_number = None
if len(tokens) > 1:
# get the package version info
test_version_number = tokens[1]
# check if we have a valid version, i.e. does not contain post/dev
version_number = '.'.join([
v for v in test_version_number.split('.')
if v and '0' <= v[0] <= '9'
])
if version_number != test_version_number:
raise ValueError()
# we have no version, but we have to have one
if not version_number:
# get the latest version from the extra index list
pip_search_cmd = ['pip', 'search']
if Worker._pip_extra_index_url:
pip_search_cmd.extend(
chain.from_iterable(
('-i', x)
for x in Worker._pip_extra_index_url))
pip_search_cmd += [package_name]
pip_search_output = get_bash_output(
' '.join(pip_search_cmd), strip=True)
version_number = pip_search_output.split(package_name)[1]
version_number = version_number.replace('(', ' ').replace(
')', ' ').split()[0]
version_number = '.'.join([
v for v in version_number.split('.')
if v and '0' <= v[0] <= '9'
])
if not version_number:
# somewhere along the way we failed
raise ValueError()
package_name_version = package_name + '==' + version_number + cuda_ver_suffix
if version_number in line:
# make sure we have the specific version not >=
tokens = line.split(';')
line = ';'.join([package_name_version] + tokens[1:])
else:
# add version to the package_name
line = line.replace(package_name, package_name_version, 1)
# print('pip install %s using CUDA v%s CuDNN v%s' % (package_name, cuda_ver, cuda_cudnn_ver))
except ValueError:
pass
# print('Warning! could not find installed CUDA/CuDNN version for %s, '
# 'using original requirements line: %s' % (package_name, line))
# add the current line into the cuda requirements list
return line