from apiserver.apierrors import errors
from apiserver.bll.event.event_common import (
EventType,
EventSettings,
search_company_events,
check_empty_data,
)
from apiserver.bll.event.scalar_key import ScalarKey, ScalarKeyEnum
from apiserver.config_repo import config
from apiserver.database.errors import translate_errors_context
from apiserver.database.model.task.task import Task
from apiserver.timing_context import TimingContext
from apiserver.tools import safe_get
log = config.logger(__file__)
class EventMetrics:
MAX_AGGS_ELEMENTS_COUNT = 50
MAX_SAMPLE_BUCKETS = 6000
def __init__(self, es: Elasticsearch):
self.es = es
def get_scalar_metrics_average_per_iter(self, company_id: str,
task_id: str, samples: int,
key: ScalarKeyEnum) -> dict:
"""
Get scalar metric histogram per metric and variant
The amount of points in each histogram should not exceed
from pathlib import Path
from typing import Sequence, Union
from apiserver.config_repo import config
from apiserver.config.info import get_default_company
from apiserver.database.model.auth import Role
from apiserver.service_repo.auth.fixed_user import FixedUser
from .migration import _apply_migrations, check_mongo_empty, get_last_server_version
from .pre_populate import PrePopulate
from .user import ensure_fixed_user, _ensure_auth_user, _ensure_backend_user
from .util import _ensure_company, _ensure_default_queue, _ensure_uuid
log = config.logger(__package__)
def _pre_populate(company_id: str, zip_file: str):
if not zip_file or not Path(zip_file).is_file():
msg = f"Invalid pre-populate zip file: {zip_file}"
if config.get("apiserver.pre_populate.fail_on_error", False):
log.error(msg)
raise ValueError(msg)
else:
log.warning(msg)
else:
log.info(f"Pre-populating using {zip_file}")
PrePopulate.import_from_zip(
zip_file,
artifacts_path=config.get("apiserver.pre_populate.artifacts_path",
None),
)
import os
from threading import Thread
from time import sleep
from typing import Optional
import attr
import requests
from semantic_version import Version
from apiserver.config_repo import config
from apiserver.config.info import get_version
from apiserver.database.model.settings import Settings
from apiserver.utilities.threads_manager import ThreadsManager
log = config.logger(__name__)
class CheckUpdatesThread(Thread):
_enabled = bool(config.get("apiserver.check_for_updates.enabled", True))
@attr.s(auto_attribs=True)
class _VersionResponse:
version: str
patch_upgrade: bool
description: str = None
def __init__(self):
super(CheckUpdatesThread, self).__init__(target=self._check_updates,
daemon=True)
def start(self) -> None:
from datetime import datetime
from apiserver import database
from apiserver.apierrors import errors
from apiserver.apimodels.auth import GetTokenResponse, CreateUserRequest, Credentials as CredModel
from apiserver.apimodels.users import CreateRequest as Users_CreateRequest
from apiserver.bll.user import UserBLL
from apiserver.config_repo import config
from apiserver.config.info import get_version, get_build_number
from apiserver.database.errors import translate_errors_context
from apiserver.database.model.auth import User, Role, Credentials
from apiserver.database.model.company import Company
from apiserver.service_repo import APICall, ServiceRepo
from apiserver.service_repo.auth import Identity, Token, get_client_id, get_secret_key
log = config.logger("AuthBLL")
class AuthBLL:
@staticmethod
def get_token_for_user(
user_id: str,
company_id: str = None,
expiration_sec: int = None,
entities: dict = None,
) -> GetTokenResponse:
with translate_errors_context():
query = dict(id=user_id)
if company_id:
from os import getenv
from boltons.iterutils import first
from furl import furl
from jsonmodels import models
from jsonmodels.errors import ValidationError
from jsonmodels.fields import StringField
from mongoengine import register_connection
from mongoengine.connection import get_connection, disconnect
from apiserver.config_repo import config
from .defs import Database
from .utils import get_items
log = config.logger("database")
strict = config.get("apiserver.mongo.strict", True)
OVERRIDE_HOST_ENV_KEY = (
"CLEARML_MONGODB_SERVICE_HOST",
"TRAINS_MONGODB_SERVICE_HOST",
"MONGODB_SERVICE_HOST",
"MONGODB_SERVICE_SERVICE_HOST",
)
OVERRIDE_PORT_ENV_KEY = (
"CLEARML_MONGODB_SERVICE_PORT",
"TRAINS_MONGODB_SERVICE_PORT",
"MONGODB_SERVICE_PORT",
)
from apiserver.apierrors import errors
from apiserver.apierrors.base import BaseError
from apiserver.config_repo import config
from apiserver.database.errors import MakeGetAllQueryError
from apiserver.database.projection import project_dict, ProjectionHelper
from apiserver.database.props import PropsMixin
from apiserver.database.query import RegexQ, RegexWrapper
from apiserver.database.utils import (
get_company_or_none_constraint,
get_fields_choices,
field_does_not_exist,
field_exists,
)
log = config.logger("dbmodel")
ACCESS_REGEX = re.compile(r"^(?P<prefix>>=|>|<=|<)?(?P<value>.*)$")
ACCESS_MODIFIER = {">=": "gte", ">": "gt", "<=": "lte", "<": "lt"}
ABSTRACT_FLAG = {"abstract": True}
class AuthDocument(Document):
meta = ABSTRACT_FLAG
class ProperDictMixin(object):
def to_proper_dict(
self: Union["ProperDictMixin", Document],
strip_private=True,
