def get_compatibility(key: str, service: str, version: str, is_cli=True):
if not key:
key = uuid.uuid4().hex
try:
version = version.lstrip("v").replace(".", "-")[:5]
except Exception as e:
CliConfigManager.reset(last_check=now())
if is_cli:
handle_cli_error(
e,
message="Could parse the version {}.".format(version),
)
polyaxon_client = PolyaxonClient(config=ClientConfig(), token=NO_AUTH)
try:
return polyaxon_client.versions_v1.get_compatibility(uuid=key,
service=service,
version=version)
except ApiException as e:
if e.status == 403:
session_expired()
CliConfigManager.reset(last_check=now())
if is_cli:
handle_cli_error(
e,
message="Could not reach the compatibility API.",
)
except HTTPError:
CliConfigManager.reset(last_check=now())
if is_cli:
Printer.print_error(
"Could not connect to remote server to fetch compatibility versions.",
)
def test_has_datetime_timestamp(self):
log_result = V1Log.process_log_line(
timestamp=now(),
value="foo",
node="node1",
pod="pod1",
container="container1",
)
assert log_result.timestamp.date() == now().date()
def test_should_check(self):
config = ChecksConfig(last_check=now())
assert config.should_check(-1) is False
assert config.should_check(0) is True
config.last_check = now() - timedelta(seconds=10000)
assert config.should_check(-1) is False
assert config.should_check() is True
assert config.should_check(100) is True
assert config.should_check(100000) is False
def get_log_handler(polyaxon_client=None):
polyaxon_client = polyaxon_client or PolyaxonClient()
try:
return polyaxon_client.versions_v1.get_log_handler()
except ApiException as e:
if e.status == 403:
session_expired()
CliConfigManager.reset(last_check=now())
handle_cli_error(e, message="Could not get cli version.")
except HTTPError:
CliConfigManager.reset(last_check=now())
Printer.print_error("Could not connect to remote server to fetch log handler.")
def get_compatibility(
key: str,
service: str,
version: str,
is_cli: bool = True,
set_config: bool = True,
):
if not key:
installation = CliConfigManager.get_value("installation") or {}
key = installation.get("key") or uuid.uuid4().hex
try:
version = clean_version_for_compatibility(version)
except Exception as e:
if set_config:
CliConfigManager.reset(last_check=now())
if is_cli:
handle_cli_error(
e,
message="Could parse the version {}.".format(version),
)
polyaxon_client = PolyaxonClient(config=ClientConfig(), token=NO_AUTH)
try:
return polyaxon_client.versions_v1.get_compatibility(
uuid=key,
service=service,
version=version,
_request_timeout=2,
)
except ApiException as e:
if e.status == 403 and is_cli:
session_expired()
if set_config:
CliConfigManager.reset(last_check=now())
if is_cli:
handle_cli_error(
e,
message="Could not reach the compatibility API.",
)
except HTTPError:
if set_config:
CliConfigManager.reset(last_check=now())
if is_cli:
Printer.print_error(
"Could not connect to remote server to fetch compatibility versions.",
)
except Exception as e:
if set_config:
CliConfigManager.reset(last_check=now())
if is_cli:
Printer.print_error(
"Unexpected error %s, "
"could not connect to remote server to fetch compatibility versions."
% e, )
def setUp(self):
super().setUp()
self.notification = NotificationSpec(
kind=self.webhook.notification_key,
owner="onwer",
project="project",
uuid=uuid.uuid4().hex,
name="test",
condition=V1StatusCondition(
type=V1Statuses.FAILED,
reason="reason",
message="message",
last_update_time=now(),
last_transition_time=now(),
),
)
async def query_k8s_operation_logs(
k8s_manager: AsyncK8SManager,
instance: str,
last_time: Optional[AwareDT],
stream: bool = False,
) -> Tuple[List[V1Log], Optional[AwareDT]]:
new_time = now()
params = {}
if last_time:
since_seconds = (new_time - last_time).total_seconds() - 1
params["since_seconds"] = int(since_seconds)
if stream:
params["tail_lines"] = V1Logs.CHUNK_SIZE
logs = []
pods = await k8s_manager.list_pods(
label_selector=get_label_selector(instance))
for pod in pods:
logs += await handle_pod_logs(
k8s_manager=k8s_manager,
pod=pod,
**params,
)
return logs, new_time
def test_logs_with_files(self):
logs = V1Logs(
last_file=1000,
last_time=now(),
files=["file1", "file2"],
logs=[
V1Log(
value="foo",
timestamp=parse_datetime("2018-12-11 10:24:57 UTC"),
node="node1",
pod="pod1",
container="container1",
),
V1Log(
value="foo",
timestamp=parse_datetime("2018-12-11 10:24:57 UTC"),
node="node1",
pod="pod1",
container="container1",
),
V1Log(
value="foo",
timestamp=parse_datetime("2018-12-11 10:24:57 UTC"),
node="node1",
pod="pod1",
container="container1",
),
],
)
logs_dict = logs.to_light_dict()
assert logs_dict == logs.from_dict(logs_dict).to_light_dict()
assert logs_dict == logs.read(logs.to_dict(dump=True)).to_light_dict()
def set_versions_config(
polyaxon_client=None,
set_installation: bool = True,
set_compatibility: bool = True,
set_handler: bool = False,
service=PolyaxonServices.CLI,
version=pkg.VERSION,
key: str = None,
):
polyaxon_client = polyaxon_client or PolyaxonClient()
server_installation = None
if set_installation:
server_installation = get_server_installation(
polyaxon_client=polyaxon_client)
if not key and server_installation and server_installation.key:
key = server_installation.key
compatibility = None
if set_compatibility:
compatibility = get_compatibility(key=key,
service=service,
version=version)
log_handler = None
if set_handler:
log_handler = get_log_handler(polyaxon_client=polyaxon_client)
return CliConfigManager.reset(
last_check=now(),
current_version=version,
installation=server_installation.to_dict()
if server_installation else {},
compatibility=compatibility.to_dict() if compatibility else {},
log_handler=log_handler.to_dict() if log_handler else {},
)
def sync_logs(
k8s_manager: K8SManager,
client: RunClient,
last_check: Optional[datetime],
pod_id: str,
container_id: str,
owner: str,
project: str,
run_uuid: str,
):
new_check = now()
since_seconds = None
if last_check:
since_seconds = (new_check - last_check).total_seconds()
if since_seconds < 1:
return last_check
filepath = str(new_check.timestamp())
created = process_logs(
k8s_manager=k8s_manager,
pod_id=pod_id,
container_id=container_id,
since_seconds=since_seconds,
filepath=filepath,
)
if created:
client.client.upload_run_logs(owner,
project,
run_uuid,
uploadfile=filepath,
path=filepath)
return new_check
return last_check
def test_exact_time_schedule(self):
config_dict = {"startAt": "foo"}
with self.assertRaises(ValidationError):
V1ExactTimeSchedule.from_dict(config_dict)
config_dict = {"kind": "exact_time", "startAt": now().isoformat()}
V1ExactTimeSchedule.from_dict(config_dict).to_dict()
def should_check(self, interval: int = None):
interval = self.get_interval(interval=interval)
if interval == -1:
return False
if (now() - self.last_check).total_seconds() > interval:
return True
return False
def test_executable(self):
config_dict = {
"startAt": "foo",
"run": {"run": {"kind": V1RunKind.JOB, "container": {"image": "test"}}},
}
with self.assertRaises(ValidationError):
V1CompiledOperation.from_dict(config_dict)
config_dict = {
"schedule": {"startAt": "foo"},
"run": {"kind": V1RunKind.JOB, "container": {"image": "test"}},
}
with self.assertRaises(ValidationError):
V1CompiledOperation.from_dict(config_dict)
config_dict = {"timeout": 2}
with self.assertRaises(ValidationError):
V1CompiledOperation.from_dict(config_dict)
config_dict = {
"schedule": {"kind": "datetime", "startAt": now().isoformat()},
"termination": {"timeout": 2},
"run": {"kind": V1RunKind.JOB, "container": {"image": "test"}},
}
V1CompiledOperation.from_dict(config_dict)
def __init__(self):
self.created_at = now()
self.started_at = None
self.finished_at = None
self.status = None
self.wait_time = None
self.duration = None
def process_log_line(
cls,
value: Text,
node: Optional[str],
pod: Optional[str],
container: Optional[str],
timestamp=None,
) -> "V1Log":
if not value:
return None
if not isinstance(value, str):
value = value.decode("utf-8")
value = value.strip()
if not timestamp:
value, timestamp = timestamp_search_regex(ISO_DATETIME_REGEX, value)
if not timestamp:
value, timestamp = timestamp_search_regex(DATETIME_REGEX, value)
if isinstance(timestamp, str):
try:
timestamp = dt_parser.parse(timestamp)
except Exception as e:
raise ValidationError("Received an invalid timestamp") from e
return cls(
timestamp=timestamp if timestamp else now(tzinfo=True),
node=node,
pod=pod,
container=container,
value=value,
)
def set_finished_at(cls, entity) -> bool:
if cls.is_done(entity.status) and entity.finished_at is None:
entity.finished_at = now()
if entity.started_at is None: # We should not have this case
entity.started_at = entity.created_at
# Update duration
if entity.duration is None:
entity.duration = (entity.finished_at - entity.started_at).seconds
return True
return False
def test_should_check(self):
with patch.object(CliConfigManager, "reset") as patch_fct:
result = CliConfigManager.should_check()
assert patch_fct.call_count == 1
assert result is True
CliConfigManager.reset(
last_check=now(),
current_version="0.0.5",
installation={"key": "uuid", "version": "1.1.4-rc11", "dist": "foo"},
compatibility={"cli": {"min": "0.0.4", "latest": "1.1.4"}},
)
with patch.object(CliConfigManager, "reset") as patch_fct:
result = CliConfigManager.should_check()
assert patch_fct.call_count == 0
assert result is False
CliConfigManager.reset(
last_check=now() - timedelta(1000),
current_version="0.0.5",
installation={"key": "uuid", "version": "1.1.4-rc11", "dist": "foo"},
compatibility={"cli": {"min": "0.0.4", "latest": "1.1.4"}},
)
with patch.object(CliConfigManager, "reset") as patch_fct:
result = CliConfigManager.should_check()
assert patch_fct.call_count == 1
assert result is True
CliConfigManager.reset(
last_check=now(),
current_version="0.0.2",
installation={"key": "uuid", "version": "1.1.4-rc11", "dist": "foo"},
compatibility={"cli": {"min": "0.0.4", "latest": "1.1.4"}},
)
with patch.object(CliConfigManager, "reset") as patch_fct:
result = CliConfigManager.should_check()
# Although condition for showing a message, do not reset
assert patch_fct.call_count == 0
assert result is False
def test_schedule(self):
configs = [
{
"kind": "interval",
"frequency": 2,
"startAt": now().isoformat(),
"endAt": now().isoformat(),
"dependsOnPast": False,
},
{
"kind": "cron",
"cron": "0 0 * * *",
"startAt": now().isoformat(),
"endAt": now().isoformat(),
"dependsOnPast": False,
},
]
ScheduleSchema().load(configs, many=True)
async def get_k8s_operation_logs(
k8s_manager: AsyncK8SManager,
operation: str,
last_time: Optional[AwareDT],
stream: bool = False,
) -> Tuple[List[V1Log], Optional[AwareDT]]:
new_time = now()
params = {}
if last_time:
since_seconds = (new_time - last_time).total_seconds() - 1
params["since_seconds"] = int(since_seconds)
if stream:
params["tail_lines"] = V1Logs.CHUNK_SIZE
logs = []
pods = await k8s_manager.list_pods(label_selector=get_label_selector(operation))
async def handle_container_logs():
resp = None
try:
resp = await k8s_manager.k8s_api.read_namespaced_pod_log(
pod.metadata.name,
k8s_manager.namespace,
container=container.name,
timestamps=True,
**params,
)
except ApiException:
pass
if not resp:
return None, None
for log_line in resp.split("\n"):
if log_line:
logs.append(
V1Log.process_log_line(
value=log_line,
node=pod.spec.node_name,
pod=pod.metadata.name,
container=container.name,
)
)
for pod in pods:
for container in pod.spec.init_containers or []:
await handle_container_logs()
for container in pod.spec.containers or []:
await handle_container_logs()
if logs:
last_time = logs[-1].timestamp
return logs, last_time
def test_cron_schedule(self):
config_dict = {"cron": 2, "startAt": "foo"}
with self.assertRaises(ValidationError):
V1CronSchedule.from_dict(config_dict)
config_dict = {
"kind": "interval",
"cron": "0 0 * * *",
"startAt": now().isoformat(),
"endAt": now().isoformat(),
}
with self.assertRaises(ValidationError):
V1CronSchedule.from_dict(config_dict)
config_dict = {"cron": "0 0 * * *"}
V1CronSchedule.from_dict(config_dict)
config_dict = {
"cron": "0 0 * * *",
"startAt": now().isoformat(),
"endAt": now().isoformat(),
}
V1CronSchedule.from_dict(config_dict)
config_dict = {
"kind": "cron",
"cron": "0 0 * * *",
"startAt": now().isoformat(),
"endAt": now().isoformat(),
"dependsOnPast": False,
}
V1CronSchedule.from_dict(config_dict)
def set_started_at(cls, entity) -> bool:
# We allow to override started_at if the value is running
if entity.started_at is not None:
return False
if cls.is_running(entity.status):
entity.started_at = now()
# Update wait_time
if entity.wait_time is None:
entity.wait_time = (entity.started_at - entity.created_at).seconds
return True
return False
def test_component_base_attrs(self):
config_dict = {
"concurrency": "foo",
"run": {"kind": V1RunKind.JOB, "container": {"image": "test"}},
}
with self.assertRaises(ValidationError):
V1Component.from_dict(config_dict)
config_dict = {
"concurrency": 2,
"run": {"kind": V1RunKind.JOB, "container": {"image": "test"}},
}
with self.assertRaises(ValidationError):
V1Component.from_dict(config_dict)
config_dict = {
"kind": "component",
"matrix": {
"concurrency": 2,
"kind": "mapping",
"values": [{"a": 1}, {"a": 1}],
},
"run": {"kind": V1RunKind.JOB, "container": {"image": "test"}},
}
with self.assertRaises(ValidationError):
V1Component.from_dict(config_dict)
config_dict = {
"kind": "component",
"matrix": {
"concurrency": 2,
"kind": "mapping",
"values": [{"a": 1}, {"a": 1}],
},
"schedule": {
"kind": "datetime",
"startAt": local_datetime(now()).isoformat(),
},
"termination": {"timeout": 1000},
"run": {"kind": V1RunKind.JOB, "container": {"image": "test"}},
}
with self.assertRaises(ValidationError):
V1Component.from_dict(config_dict)
config_dict = {
"kind": "component",
"termination": {"timeout": 1000},
"run": {"kind": V1RunKind.JOB, "container": {"image": "test"}},
}
config = V1Component.from_dict(config_dict)
assert config.to_dict() == config_dict
def get(self, request, *args, **kwargs):
self.init_config()
config = self.get_config()
if config and config.should_check():
config.version = pkg.VERSION
key = conf.get(ORGANIZATION_KEY) or get_dummy_key()
config.compatibility = get_compatibility(
key=key,
service=PolyaxonServices.PLATFORM,
version=config.version,
is_cli=False,
)
config.last_check = now()
self.write_config(config)
return Response(status=status.HTTP_200_OK)
def sync_artifacts(last_check: Optional[datetime], run_uuid: str):
new_check = now()
connection_type = get_artifacts_connection()
path_from = CONTEXT_MOUNT_ARTIFACTS_FORMAT.format(run_uuid)
# check if there's a path to sync
if os.path.exists(path_from):
path_to = os.path.join(connection_type.store_path, run_uuid)
upload_file_or_dir(
path_from=path_from,
path_to=path_to,
is_file=False,
workers=5,
last_time=last_check,
connection_type=connection_type,
)
return new_check
def test_pipelines_base_attrs(self):
config_dict = {
"concurrency": "foo",
"run": {"kind": V1RunKind.JOB, "container": {"image": "test"}},
}
with self.assertRaises(ValidationError):
V1CompiledOperation.from_dict(config_dict)
config_dict = {
"concurrency": 2,
"run": {"run": {"kind": V1RunKind.JOB, "container": {"image": "test"}}},
}
with self.assertRaises(ValidationError):
V1CompiledOperation.from_dict(config_dict)
config_dict = {
"kind": "compiled_operation",
"matrix": {
"concurrency": 2,
"kind": "mapping",
"values": [{"a": 1}, {"a": 1}],
},
"run": {"kind": V1RunKind.JOB, "container": {"image": "test"}},
}
config = V1CompiledOperation.from_dict(config_dict)
assert config.to_dict()["run"] == config_dict["run"]
assert config.to_dict()["matrix"] == config_dict["matrix"]
config_dict = {
"kind": "compiled_operation",
"matrix": {
"concurrency": 2,
"kind": "mapping",
"values": [{"a": 1}, {"a": 1}],
},
"schedule": {"kind": "datetime", "startAt": now().isoformat()},
"termination": {"timeout": 1000},
"run": {"kind": V1RunKind.JOB, "container": {"image": "test"}},
}
config = V1CompiledOperation.from_dict(config_dict)
config_to_light = config.to_light_dict()
config_to_light["schedule"].pop("startAt")
config_dict["schedule"].pop("startAt")
assert config_to_light == config_dict
def humanize_timesince(start_time): # pylint:disable=too-many-return-statements
"""Creates a string representation of time since the given `start_time`."""
if not start_time:
return start_time
start_time = parse_datetime(start_time)
delta = now() - start_time
# assumption: negative delta values originate from clock
# differences on different app server machines
if delta.total_seconds() < 0:
return "a few seconds ago"
num_years = delta.days // 365
if num_years > 0:
return "{} year{} ago".format(*((num_years,
"s") if num_years > 1 else (num_years,
"")))
num_weeks = delta.days // 7
if num_weeks > 0:
return "{} week{} ago".format(*((num_weeks,
"s") if num_weeks > 1 else (num_weeks,
"")))
num_days = delta.days
if num_days > 0:
return "{} day{} ago".format(*((num_days,
"s") if num_days > 1 else (num_days,
"")))
num_hours = delta.seconds // 3600
if num_hours > 0:
return "{} hour{} ago".format(*((num_hours,
"s") if num_hours > 1 else (num_hours,
"")))
num_minutes = delta.seconds // 60
if num_minutes > 0:
return "{} minute{} ago".format(
*((num_minutes, "s") if num_minutes > 1 else (num_minutes, "")))
return "a few seconds ago"
async def query_k8s_pod_logs(
k8s_manager: AsyncK8SManager,
pod: V1Pod,
last_time: Optional[AwareDT],
stream: bool = False,
) -> Tuple[List[V1Log], Optional[AwareDT]]:
new_time = now()
params = {}
if last_time:
since_seconds = (new_time - last_time).total_seconds() - 1
params["since_seconds"] = int(since_seconds)
if stream:
params["tail_lines"] = V1Logs.CHUNK_SIZE
logs = await handle_pod_logs(k8s_manager=k8s_manager, pod=pod, **params)
if logs:
last_time = logs[-1].timestamp
return logs, last_time
def get_condition(
cls,
type=None, # noqa
status=None,
last_update_time=None,
last_transition_time=None,
reason=None,
message=None,
) -> "V1StatusCondition":
current_time = now()
last_update_time = last_update_time or current_time
last_transition_time = last_transition_time or current_time
return cls(
type=type.lower() if type else type,
status=status,
last_update_time=last_update_time,
last_transition_time=last_transition_time,
reason=reason,
message=message,
)
def test_interval_schedule(self):
config_dict = {"frequency": 2, "startAt": "foo"}
with self.assertRaises(ValidationError):
V1IntervalSchedule.from_dict(config_dict)
config_dict = {"frequency": "foo", "startAt": now().isoformat()}
with self.assertRaises(ValidationError):
V1IntervalSchedule.from_dict(config_dict)
config_dict = {
"kind": "cron",
"frequency": 2,
"startAt": now().isoformat(),
"endAt": now().isoformat(),
}
with self.assertRaises(ValidationError):
V1IntervalSchedule.from_dict(config_dict)
config_dict = {"frequency": 2, "startAt": now().isoformat()}
V1IntervalSchedule.from_dict(config_dict)
config_dict = {
"frequency": 2,
"startAt": now().isoformat(),
"endAt": now().isoformat(),
}
V1IntervalSchedule.from_dict(config_dict)
config_dict = {
"kind": "interval",
"frequency": 2,
"startAt": now().isoformat(),
"endAt": now().isoformat(),
"maxRuns": 123,
"dependsOnPast": False,
}
V1IntervalSchedule.from_dict(config_dict)
def make(
cls,
step: int = None,
timestamp=None,
metric: float = None,
image: V1EventImage = None,
histogram: V1EventHistogram = None,
audio: V1EventAudio = None,
video: V1EventVideo = None,
html: str = None,
text: str = None,
chart: V1EventChart = None,
curve: V1EventCurve = None,
artifact: V1EventArtifact = None,
model: V1EventModel = None,
dataframe: V1EventDataframe = None,
) -> "V1Event":
if isinstance(timestamp, str):
try:
timestamp = parse_datetime(timestamp)
except Exception as e:
raise ValidationError("Received an invalid timestamp") from e
return cls(
timestamp=timestamp if timestamp else now(tzinfo=True),
step=step,
metric=metric,
image=image,
histogram=histogram,
audio=audio,
video=video,
html=html,
text=text,
chart=chart,
curve=curve,
artifact=artifact,
model=model,
dataframe=dataframe,
)