def test_initialize_logger():
mock_package_name = 'mock_package'
default_log_level = logging.DEBUG
default_stream_handler_log_level = logging.CRITICAL
mock_logger = logger.initialize_logger(mock_package_name)
assert mock_logger
assert logging.getLogger(
mock_package_name).getEffectiveLevel() == default_log_level
assert logger.STREAM_HANDLER.level == default_stream_handler_log_level
import socket
import requests
from requests.exceptions import ConnectionError
import time
import psutil
from util.k8s import kubectl
from util.app_names import NAUTAAppNames
from util.logger import initialize_logger
from util.exceptions import K8sProxyOpenError, K8sProxyCloseError, LocalPortOccupiedError, KubectlConnectionError
from cli_text_consts import UtilK8sProxyTexts as Texts
logger = initialize_logger(__name__)
class TunnelSetupError(RuntimeError):
pass
class K8sProxy:
def __init__(self,
nauta_app_name: NAUTAAppNames,
port: int = None,
app_name: str = None,
number_of_retries: int = 0,
namespace: str = None,
number_of_retries_wait_for_readiness: int = 30):
self.nauta_app_name = nauta_app_name
RUN_START_DATE = "Start date"
RUN_END_DATE = "End date"
RUN_SUBMITTER = "Owner"
RUN_TEMPLATE_NAME = "Template name"
JUPYTER_NOTEBOOK_TEMPLATES_NAMES = ["jupyter", "jupyter-py2"]
EXP_SUB_SEMAPHORE_FILENAME = ".underSubmission"
EXPERIMENTS_LIST_HEADERS = [RUN_NAME, RUN_PARAMETERS, RUN_METRICS, RUN_SUBMISSION_DATE, RUN_START_DATE, RUN_END_DATE,
RUN_SUBMITTER, RUN_STATUS, RUN_TEMPLATE_NAME]
CHART_YAML_FILENAME = "Chart.yaml"
TEMPL_FOLDER_NAME = "templates"
log = initialize_logger('commands.common')
PrepareExperimentResult = namedtuple('PrepareExperimentResult', ['folder_name', 'script_name', 'pod_count'])
submitted_runs = []
submitted_experiment = ""
submitted_namespace = ""
def ctrl_c_handler_for_submit(sig, frame):
log.debug("ctrl-c pressed while submitting")
try:
with spinner(text=Texts.CTRL_C_PURGING_PROGRESS_MSG):
if submitted_runs:
for run in submitted_runs:
import sys
from typing import List, Tuple
import errno
import socket
import dateutil
from datetime import timedelta
import signal
import platform
from distutils.version import LooseVersion
import click
import distro
from util.logger import initialize_logger, get_verbosity_level
from cli_text_consts import UtilSystemTexts as Texts, VERBOSE_RERUN_MSG
log = initialize_logger('util.system')
WINDOWS_EDITIONS = {
0: "undefined",
1: "ultimate",
2: "home_basic",
3: "home_premium",
4: "enterprise",
5: "home_basic_n",
6: "business",
7: "standard_server",
8: "datacenter_server",
9: "small_business_server",
10: "enterprise_server",
11: "starter",
12: "datacenter_server_core",
from util.cli_state import common_options
from tensorboard.client import TensorboardServiceClient, TensorboardStatus, build_tensorboard_run_list
from util.spinner import spinner
from util.aliascmd import AliasCmd, AliasGroup
from util.app_names import NAUTAAppNames
from util.exceptions import LaunchError, ProxyClosingError
from util.k8s.k8s_info import get_kubectl_current_context_namespace
from util.k8s.k8s_proxy_context_manager import K8sProxy
from util.launcher import launch_app
from util.logger import initialize_logger
from util.system import handle_error
from cli_text_consts import LaunchCmdTexts as Texts
logger = initialize_logger('commands.launch')
FORWARDED_URL = 'http://*****:*****@click.command(cls=AliasCmd, alias='ui', short_help=Texts.WEBUI_HELP, help=Texts.WEBUI_HELP,
options_metavar='[options]')
@common_options()
@click.pass_context
@click.option('-n', '--no-launch', is_flag=True, help=Texts.HELP_N)
@click.option('-pn', '--port-number', type=click.IntRange(1024, 65535), help=Texts.HELP_P)
def webui(ctx: click.Context, no_launch: bool, port_number: int):
import base64
from enum import Enum
from http import HTTPStatus
from typing import List, Dict
from kubernetes.client.rest import ApiException
from kubernetes import config, client
from kubernetes.client import configuration, V1DeleteOptions, V1Secret, V1ServiceAccount
from util.logger import initialize_logger
from util.exceptions import KubernetesError
from util.app_names import NAUTAAppNames
from cli_text_consts import UtilK8sInfoTexts as Texts
logger = initialize_logger('util.kubectl')
PREFIX_VALUES = {
"E": 10**18,
"P": 10**15,
"T": 10**12,
"G": 10**9,
"M": 10**6,
"K": 10**3
}
PREFIX_I_VALUES = {
"Ei": 2**60,
"Pi": 2**50,
"Ti": 2**40,
"Gi": 2**30,
"Mi": 2**20,
from commands.experiment.common import RUN_NAME, RUN_PARAMETERS, RUN_STATUS, RUN_MESSAGE, RunKinds, \
validate_env_paramater
from util.cli_state import common_options, pass_state, State
from util.config import TBLT_TABLE_FORMAT
from util.logger import initialize_logger
from commands.experiment.common import submit_experiment
from util.aliascmd import AliasCmd
from util.exceptions import SubmitExperimentError, K8sProxyCloseError
from commands.experiment.common import validate_experiment_name, validate_pack_params_names, validate_template_name, \
validate_pack
from platform_resources.run import RunStatus
from util.system import handle_error
from cli_text_consts import ExperimentSubmitCmdTexts as Texts
logger = initialize_logger('commands.submit')
DEFAULT_SCRIPT_NAME = "experiment.py"
DEFAULT_REQUIREMENTS_NAME = "requirements.txt"
def validate_script_location(script_location: str):
if not (os.path.isfile(script_location) or os.path.isdir(script_location)):
handle_error(user_msg=Texts.SCRIPT_NOT_FOUND_ERROR_MSG.format(
script_location=script_location))
exit(2)
def get_default_script_location(script_directory: str) -> str:
default_script_location = os.path.join(script_directory,
DEFAULT_SCRIPT_NAME)
def initialize():
initialize_logger()
initialize_path()
initialize_collection()
import docker
import docker.errors
import yaml
import toml
from util.k8s import k8s_info
from util.logger import initialize_logger
from util.config import FOLDER_DIR_NAME
from util.config import NAUTAConfigMap
from util.spinner import spinner
import packs.common as common
import dpath.util as dutil
from cli_text_consts import PacksTfTrainingTexts as Texts
log = initialize_logger('packs.tf_training')
WORK_CNT_PARAM = "workersCount"
P_SERV_CNT_PARAM = "pServersCount"
POD_COUNT_PARAM = "podCount"
def update_configuration(run_folder: str,
script_location: str,
script_parameters: Tuple[str, ...],
experiment_name: str,
run_name: str,
local_registry_port: int,
cluster_registry_port: int,
pack_type: str,
pack_params: List[Tuple[str, str]] = None,
#
import sys
import click
from tabulate import tabulate
from commands.experiment.common import RUN_TEMPLATE_NAME, get_list_of_packs
from util.cli_state import common_options
from util.aliascmd import AliasCmd
from util.logger import initialize_logger
from util.system import handle_error
from cli_text_consts import ExperimentTemplateListCmdTexts as Texts
log = initialize_logger('commands.template_list')
CHART_YAML_FILENAME = "Chart.yaml"
TEMPL_FOLDER_NAME = "templates"
@click.command("template-list", short_help=Texts.HELP, help=Texts.HELP, cls=AliasCmd, alias='t',
options_metavar='[options]')
@common_options()
def template_list():
list_of_packs = get_list_of_packs()
if list_of_packs:
click.echo(tabulate([[row] for row in list_of_packs],
headers=[RUN_TEMPLATE_NAME],
tablefmt="orgtbl"))
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import logging
import os
from typing import List
import re
from util.config import Config
from util.system import execute_system_command
from util.logger import initialize_logger, get_verbosity_level
from cli_text_consts import DraftCmdTexts as Texts
logger = initialize_logger('draft.cmd')
DRAFT_BIN = 'draft'
DRAFT_HOME_FOLDER = ".draft"
DRAFT_LOGS_FOLDER = "logs"
DOCKER_IP_ADDRESS = "127.0.0.1"
def call_draft(args: List[str], cwd: str = None, namespace: str = None, logs_size: int = 0) \
-> (str, int, str):
config_path = Config().config_path
full_command = [os.path.join(config_path, DRAFT_BIN)]
full_command.extend(args)
if get_verbosity_level() == logging.DEBUG:
full_command.append('--debug')
import logging
import os
import sys
import util.dbconnect as db
import util.logger as utl
import util.argparser as arg
import util.readFiles as rfiles
import os.path
# Initialize log object
logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)
utl.initialize_logger(os.getcwd(), logger)
def main():
tmpColumns = [
'cell_type_category', 'cell_type', 'cell_type_track_name',
'cell_type_short', 'assay_category', 'assay', 'assay_track_name',
'assay_short,donor', 'time_point', 'view', 'track_name', 'track_type',
'track_density', 'provider_institution', 'source_server',
'source_path_to_file', 'server,path_to_file', 'new_file_name'
]
args = arg.callArgparser()
conn = db.connect_db(args.db, logger)
#1
if args.command == "createdb":