def test_initialize_logger(): mock_package_name = 'mock_package' default_log_level = logging.DEBUG default_stream_handler_log_level = logging.CRITICAL mock_logger = logger.initialize_logger(mock_package_name) assert mock_logger assert logging.getLogger( mock_package_name).getEffectiveLevel() == default_log_level assert logger.STREAM_HANDLER.level == default_stream_handler_log_level
import socket import requests from requests.exceptions import ConnectionError import time import psutil from util.k8s import kubectl from util.app_names import NAUTAAppNames from util.logger import initialize_logger from util.exceptions import K8sProxyOpenError, K8sProxyCloseError, LocalPortOccupiedError, KubectlConnectionError from cli_text_consts import UtilK8sProxyTexts as Texts logger = initialize_logger(__name__) class TunnelSetupError(RuntimeError): pass class K8sProxy: def __init__(self, nauta_app_name: NAUTAAppNames, port: int = None, app_name: str = None, number_of_retries: int = 0, namespace: str = None, number_of_retries_wait_for_readiness: int = 30): self.nauta_app_name = nauta_app_name
RUN_START_DATE = "Start date" RUN_END_DATE = "End date" RUN_SUBMITTER = "Owner" RUN_TEMPLATE_NAME = "Template name" JUPYTER_NOTEBOOK_TEMPLATES_NAMES = ["jupyter", "jupyter-py2"] EXP_SUB_SEMAPHORE_FILENAME = ".underSubmission" EXPERIMENTS_LIST_HEADERS = [RUN_NAME, RUN_PARAMETERS, RUN_METRICS, RUN_SUBMISSION_DATE, RUN_START_DATE, RUN_END_DATE, RUN_SUBMITTER, RUN_STATUS, RUN_TEMPLATE_NAME] CHART_YAML_FILENAME = "Chart.yaml" TEMPL_FOLDER_NAME = "templates" log = initialize_logger('commands.common') PrepareExperimentResult = namedtuple('PrepareExperimentResult', ['folder_name', 'script_name', 'pod_count']) submitted_runs = [] submitted_experiment = "" submitted_namespace = "" def ctrl_c_handler_for_submit(sig, frame): log.debug("ctrl-c pressed while submitting") try: with spinner(text=Texts.CTRL_C_PURGING_PROGRESS_MSG): if submitted_runs: for run in submitted_runs:
import sys from typing import List, Tuple import errno import socket import dateutil from datetime import timedelta import signal import platform from distutils.version import LooseVersion import click import distro from util.logger import initialize_logger, get_verbosity_level from cli_text_consts import UtilSystemTexts as Texts, VERBOSE_RERUN_MSG log = initialize_logger('util.system') WINDOWS_EDITIONS = { 0: "undefined", 1: "ultimate", 2: "home_basic", 3: "home_premium", 4: "enterprise", 5: "home_basic_n", 6: "business", 7: "standard_server", 8: "datacenter_server", 9: "small_business_server", 10: "enterprise_server", 11: "starter", 12: "datacenter_server_core",
from util.cli_state import common_options from tensorboard.client import TensorboardServiceClient, TensorboardStatus, build_tensorboard_run_list from util.spinner import spinner from util.aliascmd import AliasCmd, AliasGroup from util.app_names import NAUTAAppNames from util.exceptions import LaunchError, ProxyClosingError from util.k8s.k8s_info import get_kubectl_current_context_namespace from util.k8s.k8s_proxy_context_manager import K8sProxy from util.launcher import launch_app from util.logger import initialize_logger from util.system import handle_error from cli_text_consts import LaunchCmdTexts as Texts logger = initialize_logger('commands.launch') FORWARDED_URL = 'http://*****:*****@click.command(cls=AliasCmd, alias='ui', short_help=Texts.WEBUI_HELP, help=Texts.WEBUI_HELP, options_metavar='[options]') @common_options() @click.pass_context @click.option('-n', '--no-launch', is_flag=True, help=Texts.HELP_N) @click.option('-pn', '--port-number', type=click.IntRange(1024, 65535), help=Texts.HELP_P) def webui(ctx: click.Context, no_launch: bool, port_number: int):
import base64 from enum import Enum from http import HTTPStatus from typing import List, Dict from kubernetes.client.rest import ApiException from kubernetes import config, client from kubernetes.client import configuration, V1DeleteOptions, V1Secret, V1ServiceAccount from util.logger import initialize_logger from util.exceptions import KubernetesError from util.app_names import NAUTAAppNames from cli_text_consts import UtilK8sInfoTexts as Texts logger = initialize_logger('util.kubectl') PREFIX_VALUES = { "E": 10**18, "P": 10**15, "T": 10**12, "G": 10**9, "M": 10**6, "K": 10**3 } PREFIX_I_VALUES = { "Ei": 2**60, "Pi": 2**50, "Ti": 2**40, "Gi": 2**30, "Mi": 2**20,
from commands.experiment.common import RUN_NAME, RUN_PARAMETERS, RUN_STATUS, RUN_MESSAGE, RunKinds, \ validate_env_paramater from util.cli_state import common_options, pass_state, State from util.config import TBLT_TABLE_FORMAT from util.logger import initialize_logger from commands.experiment.common import submit_experiment from util.aliascmd import AliasCmd from util.exceptions import SubmitExperimentError, K8sProxyCloseError from commands.experiment.common import validate_experiment_name, validate_pack_params_names, validate_template_name, \ validate_pack from platform_resources.run import RunStatus from util.system import handle_error from cli_text_consts import ExperimentSubmitCmdTexts as Texts logger = initialize_logger('commands.submit') DEFAULT_SCRIPT_NAME = "experiment.py" DEFAULT_REQUIREMENTS_NAME = "requirements.txt" def validate_script_location(script_location: str): if not (os.path.isfile(script_location) or os.path.isdir(script_location)): handle_error(user_msg=Texts.SCRIPT_NOT_FOUND_ERROR_MSG.format( script_location=script_location)) exit(2) def get_default_script_location(script_directory: str) -> str: default_script_location = os.path.join(script_directory, DEFAULT_SCRIPT_NAME)
def initialize(): initialize_logger() initialize_path() initialize_collection()
import docker import docker.errors import yaml import toml from util.k8s import k8s_info from util.logger import initialize_logger from util.config import FOLDER_DIR_NAME from util.config import NAUTAConfigMap from util.spinner import spinner import packs.common as common import dpath.util as dutil from cli_text_consts import PacksTfTrainingTexts as Texts log = initialize_logger('packs.tf_training') WORK_CNT_PARAM = "workersCount" P_SERV_CNT_PARAM = "pServersCount" POD_COUNT_PARAM = "podCount" def update_configuration(run_folder: str, script_location: str, script_parameters: Tuple[str, ...], experiment_name: str, run_name: str, local_registry_port: int, cluster_registry_port: int, pack_type: str, pack_params: List[Tuple[str, str]] = None,
# import sys import click from tabulate import tabulate from commands.experiment.common import RUN_TEMPLATE_NAME, get_list_of_packs from util.cli_state import common_options from util.aliascmd import AliasCmd from util.logger import initialize_logger from util.system import handle_error from cli_text_consts import ExperimentTemplateListCmdTexts as Texts log = initialize_logger('commands.template_list') CHART_YAML_FILENAME = "Chart.yaml" TEMPL_FOLDER_NAME = "templates" @click.command("template-list", short_help=Texts.HELP, help=Texts.HELP, cls=AliasCmd, alias='t', options_metavar='[options]') @common_options() def template_list(): list_of_packs = get_list_of_packs() if list_of_packs: click.echo(tabulate([[row] for row in list_of_packs], headers=[RUN_TEMPLATE_NAME], tablefmt="orgtbl"))
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # import logging import os from typing import List import re from util.config import Config from util.system import execute_system_command from util.logger import initialize_logger, get_verbosity_level from cli_text_consts import DraftCmdTexts as Texts logger = initialize_logger('draft.cmd') DRAFT_BIN = 'draft' DRAFT_HOME_FOLDER = ".draft" DRAFT_LOGS_FOLDER = "logs" DOCKER_IP_ADDRESS = "127.0.0.1" def call_draft(args: List[str], cwd: str = None, namespace: str = None, logs_size: int = 0) \ -> (str, int, str): config_path = Config().config_path full_command = [os.path.join(config_path, DRAFT_BIN)] full_command.extend(args) if get_verbosity_level() == logging.DEBUG: full_command.append('--debug')
import logging import os import sys import util.dbconnect as db import util.logger as utl import util.argparser as arg import util.readFiles as rfiles import os.path # Initialize log object logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) utl.initialize_logger(os.getcwd(), logger) def main(): tmpColumns = [ 'cell_type_category', 'cell_type', 'cell_type_track_name', 'cell_type_short', 'assay_category', 'assay', 'assay_track_name', 'assay_short,donor', 'time_point', 'view', 'track_name', 'track_type', 'track_density', 'provider_institution', 'source_server', 'source_path_to_file', 'server,path_to_file', 'new_file_name' ] args = arg.callArgparser() conn = db.connect_db(args.db, logger) #1 if args.command == "createdb":