def aggregate_pred_results_by_ts(sci_pred_results, science_model_ids_and_probs, token): """Map model-wise prediction results to TS-wise results. Parameters ---------- sci_pred_results : dict Dictionary whose keys are cesium_web model IDs, and values are dictionaries containing prediction results for that model. science_model_ids_and_probs : dict Dictionary whose primary keys are TS names, and values are dicts with associated model IDs and probabilities as keys and values, respectively, e.g. {'ts_1': {mdl_id_1: mdl_id_1_prob, mdl_id_2: mdl_id_2_prob}, ...} token : str Valid authentication token to be used with all `cesium_web` requests. Returns ------- pred_results_by_ts : dict Dictionary whose keys are TS names and values are dictionaries containing both weighted prediction results dicts (class names as keys, combined probabilities as values) accessed by the 'combined' key, and model-wise results ('by_model'). See below for example structure. E.g. {'ts_1': {'by_model': {'model_1': {'class_1': 0.6, 'class_2': 0.1, ...}, 'model_2': {'class_1': 0.3, 'class_2': 0.4, ...}, ...}, 'combined': {'class_1': 0.5, 'class_2': 0.2, ...}}, ...} """ cfg = load_config() model_id_to_name = {model['id']: model['name'] for model in requests.get('{}/models'.format(cfg['cesium_app']['url']), headers={'Authorization': f'token {token}'}) .json()['data'] if model['project_id'] == cfg['cesium_app']['survey_classifier_project_id']} ts_names = set([ts_name for model_id in sci_pred_results for ts_name in sci_pred_results[model_id]]) pred_results_by_ts = defaultdict(lambda: defaultdict(lambda: defaultdict(float))) for model_id, results_dict in sci_pred_results.items(): for ts_name, ts_results in results_dict.items(): pred_results_by_ts[ts_name]['by_model'][ model_id_to_name[model_id]] = ts_results['prediction'] for sci_class, prob in ts_results['prediction'].items(): pred_results_by_ts[ts_name]['combined'][sci_class] += ( science_model_ids_and_probs[ts_name][model_id] * prob) return pred_results_by_ts
def determine_model_ids(prediction_results, token): """Parse results and group model IDs and probabilities by time series name. Parameters ---------- prediction_results : dict Dictionary whose keys are time series names, and values are dictionaries containing the results as returned by `cesium_web`. token : str Valid authentication token to be used with all `cesium_web` requests. Returns ------- ts_name_model_ids_and_probs : dict Dictionary whose keys are TS names and values are dictionaries of model IDs and their associated probabilities, respectively, e.g. {'ts_1': {mdl_id_1: mdl_id_1_prob, ...}, ...}. """ cfg = load_config() model_name_to_id = {model['name']: model['id'] for model in requests.get('{}/models'.format(cfg['cesium_app']['url']), headers={'Authorization': f'token {token}'}) .json()['data'] if model['project_id'] == cfg['cesium_app']['survey_classifier_project_id']} ts_name_model_ids_and_probs = {} for ts_name in prediction_results: ts_name_model_ids_and_probs[ts_name] = { model_name_to_id[model_name]: prob for model_name, prob in prediction_results[ts_name]['prediction'].items() if prob >= 0.05} # Normalize probabilities for ts_name in ts_name_model_ids_and_probs: old_sum = sum(ts_name_model_ids_and_probs[ts_name].values()) ts_name_model_ids_and_probs[ts_name] = { model_id: old_prob / old_sum for model_id, old_prob in ts_name_model_ids_and_probs[ts_name].items()} return ts_name_model_ids_and_probs
init_db, FollowupRequest, Allocation, Invitation, SourceNotification, UserNotification, ) import tdtax TMP_DIR = mkdtemp() env, cfg = load_env() print("Loading test configuration from _test_config.yaml") basedir = pathlib.Path(os.path.dirname(__file__)) cfg = load_config([(basedir / "../../test_config.yaml").absolute()]) set_server_url(f'http://localhost:{cfg["ports.app"]}') print("Setting test database to:", cfg["database"]) init_db(**cfg["database"]) def is_already_deleted(instance, table): """ Helper function to check if a given ORM instance has already been deleted previously, either by earlier teardown functions or by a test itself through the API. """ # If the instance is marked detached, that means it was deleted earlier in the # current transaction. if instance in DBSession() and inspect(instance).detached: return True
import os from os.path import join as pjoin import time import uuid from selenium.webdriver.common.by import By from selenium.webdriver.common.action_chains import ActionChains from baselayer.app.config import load_config cfg = load_config() def test_public_source_page(driver, user, public_source): driver.get(f"/become_user/{user.id}") # TODO decorator/context manager? driver.get(f"/source/{public_source.id}") driver.wait_for_xpath(f'//div[text()="{public_source.id}"]') driver.wait_for_xpath( '//label[contains(text(), "band")]') # TODO how to check plot? driver.wait_for_xpath('//label[contains(text(), "Fe III")]') def test_comments(driver, user, public_source): driver.get(f"/become_user/{user.id}") # TODO decorator/context manager? driver.get(f"/source/{public_source.id}") driver.wait_for_xpath(f'//div[text()="{public_source.id}"]') comment_box = driver.find_element_by_css_selector('[name=comment]') comment_text = str(uuid.uuid4()) comment_box.send_keys(comment_text) driver.scroll_to_element_and_click( driver.find_element_by_css_selector('[type=submit]')) driver.wait_for_xpath(f'//div[text()="{comment_text}"]')
parser.add_argument( '--xml', action='store_true', help='Save JUnit xml output to `test-results/junit.xml`') parser.add_argument('--headless', action='store_true', help='Run browser headlessly') args = parser.parse_args() # Initialize the test database connection log('Initializing test database') from baselayer.app.models import init_db from baselayer.app.config import load_config basedir = pathlib.Path(os.path.dirname(__file__)) / '..' / '..' cfg = load_config([basedir / TEST_CONFIG]) app_name = cfg['app.factory'].split('.')[0] init_db(**cfg['database']) if args.test_spec is not None: test_spec = args.test_spec else: test_spec = basedir / app_name / 'tests' if args.xml: test_outdir = basedir / 'test-results' if not test_outdir.exists(): test_outdir.mkdir() xml = f'--junitxml={test_outdir}/junit.xml' else: xml = ''
'''Test fixture configuration.''' import pytest import os import pathlib import distutils.spawn import types import shutil from selenium import webdriver from selenium.webdriver.common.by import By from selenium.webdriver.support.ui import WebDriverWait from selenium.webdriver.support import expected_conditions from selenium.common.exceptions import TimeoutException from seleniumrequests.request import RequestMixin from baselayer.app.config import load_config from baselayer.app.test_util import (driver, MyCustomWebDriver, reset_state, set_server_url) print('Loading test configuration from test_config.yaml') basedir = pathlib.Path(os.path.dirname(__file__))/'../..' cfg = load_config([basedir/'test_config.yaml']) set_server_url(f'http://localhost:{cfg["ports:app"]}')
#!/usr/bin/env python import argparse import uuid import os from baselayer.app.config import load_config from baselayer.app.models import init_db from cesium_app import model_util cfg_paths = [os.path.join(os.path.dirname(__file__), '../config.yaml.defaults'), os.path.join(os.path.dirname(__file__), '../cesium_web/config.yaml.defaults')] cfg = load_config(cfg_paths) conn = init_db(**cfg['database']) if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument('bot_name') args = parser.parse_args() token = model_util.create_token( [], 1, args.bot_name) token_path = os.path.abspath(os.path.join( cfg['paths']['cesium_web_login_token_folder'], 'cesium_web_token')) if not os.path.exists(os.path.dirname(token_path)):
from logging.config import fileConfig from alembic import context from baselayer.app.config import load_config from baselayer.app.models import init_db # These imports need to happen for their side-effects of registering models from baselayer.app import models as baselayer_models # noqa from baselayer.app import psa # noqa from skyportal import models # noqa config_arg = context.get_x_argument(as_dictionary=True).get('config') skyportal_configs = config_arg.split(':') if config_arg else [] cfg = load_config(config_files=skyportal_configs) # this is the Alembic Config object, which provides # access to the values within the .ini file in use. config = context.config # Interpret the config file for Python logging. # This line sets up loggers basically. fileConfig(config.config_file_name) # add your model's MetaData object here # for 'autogenerate' support target_metadata = baselayer_models.Base.metadata # other values from the config, defined by the needs of env.py, # can be acquired: # my_important_option = config.get_main_option("my_important_option")
from selenium.webdriver.support.ui import Select import uuid import time import os from os.path import join as pjoin import numpy as np import numpy.testing as npt import pandas as pd import json import subprocess import glob from cesium_app.model_util import create_token_user from baselayer.app.config import load_config cfg = load_config() def _add_prediction(proj_id, driver): driver.refresh() proj_select = Select(driver.wait_for_xpath('//select[@name="project"]')) proj_select.select_by_value(str(proj_id)) driver.find_element_by_id('react-tabs-8').click() driver.find_element_by_partial_link_text('Predict Targets').click() driver.find_element_by_class_name('btn-primary').click() driver.wait_for_xpath("//div[contains(text(),'Model predictions begun')]") try:
from logging.config import fileConfig from alembic import context from baselayer.app.config import load_config from baselayer.app.models import init_db # These imports need to happen for their side-effects of registering models from baselayer.app import models as BaselayerModels # noqa from baselayer.app import psa # noqa from skyportal import models # noqa skyportal_config = context.get_x_argument(as_dictionary=True).get('config') cfg = load_config(config_files=[skyportal_config] if skyportal_config else []) # this is the Alembic Config object, which provides # access to the values within the .ini file in use. config = context.config # Interpret the config file for Python logging. # This line sets up loggers basically. fileConfig(config.config_file_name) # add your model's MetaData object here # for 'autogenerate' support target_metadata = models.Base.metadata # other values from the config, defined by the needs of env.py, # can be acquired: # my_important_option = config.get_main_option("my_important_option") # ... etc.
import os import pathlib import shutil from pytest_factoryboy import register, LazyFixture from baselayer.app.config import load_config from baselayer.app.test_util import (driver, MyCustomWebDriver, reset_state, set_server_url) from cesium_app import models from cesium_app.tests.fixtures import (TMP_DIR, ProjectFactory, DatasetFactory, FeaturesetFactory, ModelFactory, PredictionFactory) print('Loading test configuration from test_config.yaml') basedir = pathlib.Path(os.path.dirname(__file__))/'../..' cfg = load_config([basedir/'test_config.yaml']) set_server_url(f'http://*****:*****@pytest.fixture(scope='session', autouse=True) def delete_temporary_files(request): def teardown(): shutil.rmtree(TMP_DIR, ignore_errors=True) request.addfinalizer(teardown) register(ProjectFactory) register(DatasetFactory) register(DatasetFactory, "unlabeled_dataset", name="unlabeled")