from apiserver.apierrors import errors from apiserver.bll.event.event_common import ( EventType, EventSettings, search_company_events, check_empty_data, ) from apiserver.bll.event.scalar_key import ScalarKey, ScalarKeyEnum from apiserver.config_repo import config from apiserver.database.errors import translate_errors_context from apiserver.database.model.task.task import Task from apiserver.timing_context import TimingContext from apiserver.tools import safe_get log = config.logger(__file__) class EventMetrics: MAX_AGGS_ELEMENTS_COUNT = 50 MAX_SAMPLE_BUCKETS = 6000 def __init__(self, es: Elasticsearch): self.es = es def get_scalar_metrics_average_per_iter(self, company_id: str, task_id: str, samples: int, key: ScalarKeyEnum) -> dict: """ Get scalar metric histogram per metric and variant The amount of points in each histogram should not exceed
from pathlib import Path from typing import Sequence, Union from apiserver.config_repo import config from apiserver.config.info import get_default_company from apiserver.database.model.auth import Role from apiserver.service_repo.auth.fixed_user import FixedUser from .migration import _apply_migrations, check_mongo_empty, get_last_server_version from .pre_populate import PrePopulate from .user import ensure_fixed_user, _ensure_auth_user, _ensure_backend_user from .util import _ensure_company, _ensure_default_queue, _ensure_uuid log = config.logger(__package__) def _pre_populate(company_id: str, zip_file: str): if not zip_file or not Path(zip_file).is_file(): msg = f"Invalid pre-populate zip file: {zip_file}" if config.get("apiserver.pre_populate.fail_on_error", False): log.error(msg) raise ValueError(msg) else: log.warning(msg) else: log.info(f"Pre-populating using {zip_file}") PrePopulate.import_from_zip( zip_file, artifacts_path=config.get("apiserver.pre_populate.artifacts_path", None), )
import os from threading import Thread from time import sleep from typing import Optional import attr import requests from semantic_version import Version from apiserver.config_repo import config from apiserver.config.info import get_version from apiserver.database.model.settings import Settings from apiserver.utilities.threads_manager import ThreadsManager log = config.logger(__name__) class CheckUpdatesThread(Thread): _enabled = bool(config.get("apiserver.check_for_updates.enabled", True)) @attr.s(auto_attribs=True) class _VersionResponse: version: str patch_upgrade: bool description: str = None def __init__(self): super(CheckUpdatesThread, self).__init__(target=self._check_updates, daemon=True) def start(self) -> None:
from datetime import datetime from apiserver import database from apiserver.apierrors import errors from apiserver.apimodels.auth import GetTokenResponse, CreateUserRequest, Credentials as CredModel from apiserver.apimodels.users import CreateRequest as Users_CreateRequest from apiserver.bll.user import UserBLL from apiserver.config_repo import config from apiserver.config.info import get_version, get_build_number from apiserver.database.errors import translate_errors_context from apiserver.database.model.auth import User, Role, Credentials from apiserver.database.model.company import Company from apiserver.service_repo import APICall, ServiceRepo from apiserver.service_repo.auth import Identity, Token, get_client_id, get_secret_key log = config.logger("AuthBLL") class AuthBLL: @staticmethod def get_token_for_user( user_id: str, company_id: str = None, expiration_sec: int = None, entities: dict = None, ) -> GetTokenResponse: with translate_errors_context(): query = dict(id=user_id) if company_id:
from os import getenv from boltons.iterutils import first from furl import furl from jsonmodels import models from jsonmodels.errors import ValidationError from jsonmodels.fields import StringField from mongoengine import register_connection from mongoengine.connection import get_connection, disconnect from apiserver.config_repo import config from .defs import Database from .utils import get_items log = config.logger("database") strict = config.get("apiserver.mongo.strict", True) OVERRIDE_HOST_ENV_KEY = ( "CLEARML_MONGODB_SERVICE_HOST", "TRAINS_MONGODB_SERVICE_HOST", "MONGODB_SERVICE_HOST", "MONGODB_SERVICE_SERVICE_HOST", ) OVERRIDE_PORT_ENV_KEY = ( "CLEARML_MONGODB_SERVICE_PORT", "TRAINS_MONGODB_SERVICE_PORT", "MONGODB_SERVICE_PORT", )
from apiserver.apierrors import errors from apiserver.apierrors.base import BaseError from apiserver.config_repo import config from apiserver.database.errors import MakeGetAllQueryError from apiserver.database.projection import project_dict, ProjectionHelper from apiserver.database.props import PropsMixin from apiserver.database.query import RegexQ, RegexWrapper from apiserver.database.utils import ( get_company_or_none_constraint, get_fields_choices, field_does_not_exist, field_exists, ) log = config.logger("dbmodel") ACCESS_REGEX = re.compile(r"^(?P<prefix>>=|>|<=|<)?(?P<value>.*)$") ACCESS_MODIFIER = {">=": "gte", ">": "gt", "<=": "lte", "<": "lt"} ABSTRACT_FLAG = {"abstract": True} class AuthDocument(Document): meta = ABSTRACT_FLAG class ProperDictMixin(object): def to_proper_dict( self: Union["ProperDictMixin", Document], strip_private=True,