def make_app(cfg, baselayer_handlers, baselayer_settings): """Create and return a `tornado.web.Application` object with specified handlers and settings. Parameters ---------- cfg : Config Loaded configuration. Can be specified with '--config' (multiple uses allowed). baselayer_handlers : list Tornado handlers needed for baselayer to function. baselayer_settings : cfg Settings needed for baselayer to function. """ if cfg['cookie_secret'] == 'abc01234': print('!' * 80) print(' Your server is insecure. Please update the secret string ') print(' in the configuration file!') print('!' * 80) handlers = baselayer_handlers + [ # (r'/some_url(/.*)?', MyTornadoHandler), (r'/example_compute', ExampleComputationHandler) ] settings = baselayer_settings settings.update({}) # Specify any additional Tornado settings here app = tornado.web.Application(handlers, **settings) models.init_db(**cfg['database']) model_util.create_tables() app.cfg = cfg return app
def make_app(cfg, baselayer_handlers, baselayer_settings, process=None, env=None): """Create and return a `tornado.web.Application` object with specified handlers and settings. Parameters ---------- cfg : Config Loaded configuration. Can be specified with '--config' (multiple uses allowed). baselayer_handlers : list Tornado handlers needed for baselayer to function. baselayer_settings : cfg Settings needed for baselayer to function. process : int When launching multiple app servers, which number is this? env : dict Environment in which the app was launched. Currently only has one key, 'debug'---true if launched with `--debug`. """ if cfg['cookie_secret'] == 'abc01234': print('!' * 80) print(' Your server is insecure. Please update the secret string ') print(' in the configuration file!') print('!' * 80) handlers = baselayer_handlers + [ # (r'/some_url(/.*)?', MyTornadoHandler), (r'/example_compute', ExampleComputationHandler), (r'/push_notification', PushNotificationHandler) ] settings = baselayer_settings settings.update({}) # Specify any additional Tornado settings here app = tornado.web.Application(handlers, **settings) models.init_db(**cfg['database']) if process == 0: model_util.create_tables(add=env.debug) app.cfg = cfg return app
def run_migrations_online(): """Run migrations in 'online' mode. In this scenario we need to create an Engine and associate a connection with the context. """ connectable = init_db(**cfg['database']) with connectable.connect() as connection: context.configure(connection=connection, target_metadata=target_metadata) with context.begin_transaction(): context.run_migrations()
import json import requests import sqlalchemy as sa from baselayer.app.models import init_db from baselayer.app.env import load_env from skyportal.models import ( DBSession, FollowupRequest, FacilityTransactionRequest, ) env, cfg = load_env() init_db(**cfg['database']) request_session = requests.Session() request_session.trust_env = ( False # Otherwise pre-existing netrc config will override auth headers ) WAIT_TIME_BETWEEN_QUERIES = timedelta(seconds=120) class FacilityQueue(asyncio.Queue): async def load_from_db(self): # Load items from database into queue with DBSession() as session: requests = (
Source, Candidate, Role, User, Allocation, FollowupRequest, ) import astroplan print("Loading test configuration from _test_config.yaml") basedir = pathlib.Path(os.path.dirname(__file__)) cfg = load_config([(basedir / "../../test_config.yaml").absolute()]) set_server_url(f'http://localhost:{cfg["ports.app"]}') print("Setting test database to:", cfg["database"]) models.init_db(**cfg["database"]) # Add a "test factory" User so that all factory-generated comments have a # proper author, if it doesn't already exist (the user may already be in # there if running the test server and running tests individually) if not DBSession.query(User).filter(User.username == "test factory").scalar(): DBSession.add(User(username="******")) DBSession.commit() def pytest_runtest_setup(item): # Print timestamp when running each test print(datetime.now().strftime('[%H:%M:%S] '), end='') # set up a hook to be able to check if a test has failed
import argparse import uuid import os from baselayer.app.config import load_config from baselayer.app.models import init_db from cesium_app import model_util cfg_paths = [os.path.join(os.path.dirname(__file__), '../config.yaml.defaults'), os.path.join(os.path.dirname(__file__), '../cesium_web/config.yaml.defaults')] cfg = load_config(cfg_paths) conn = init_db(**cfg['database']) if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument('bot_name') args = parser.parse_args() token = model_util.create_token( [], 1, args.bot_name) token_path = os.path.abspath(os.path.join( cfg['paths']['cesium_web_login_token_folder'], 'cesium_web_token')) if not os.path.exists(os.path.dirname(token_path)): print(f'{os.path.dirname(token_path)} does not exist - creating it now')
def setup_survey_db(): # if os.getcwd().endswith('survey_app'): # os.chdir('./cesium_web') env, cfg = load_env() for data_dir in cfg['paths'].values(): if not os.path.exists(data_dir): os.makedirs(data_dir) db_session = init_db(**baselayer.app.config.load_config()['database']) # Drop & create tables with status('Dropping and re-creating tables'): drop_tables() create_tables() # Add testuser with status('Adding testuser'): u = models.User(username='******') models.DBSession().add(u) models.DBSession().commit() # Add project with status('Adding project'): p = models.Project(name='Survey Classifier', users=[u]) models.DBSession().add(p) models.DBSession().commit() # Add datasets with status('Adding datasets'): for dataset_name, ts_data_dir in [ ['Survey Light Curve Data', 'survey_classifier_data/data/lightcurves'], ['ASAS', 'survey_classifier_data/data/ASAS_lcs'], ['Noisified to CoRoT', 'survey_classifier_data/data/noisified_CoRoT_lcs'], ['Noisified to HATNet', 'survey_classifier_data/data/noisified_HATNet_lcs'], ['Noisified to Hipparcos', 'survey_classifier_data/data/noisified_Hipparcos_lcs'], ['Noisified to KELT', 'survey_classifier_data/data/noisified_KELT_lcs'], ['Noisified to Kepler', 'survey_classifier_data/data/noisified_Kepler_lcs'], ['Noisified to LINEAR', 'survey_classifier_data/data/noisified_LINEAR_lcs'], ['Noisified to OGLE-III', 'survey_classifier_data/data/noisified_OGLE-III_lcs'], ['Noisified to SuperWASP', 'survey_classifier_data/data/noisified_SuperWASP_lcs'], ['Noisified to TrES', 'survey_classifier_data/data/noisified_TrES_lcs']]: ts_paths = [] # As these are only ever accessed to determine meta features, only # copy first ten (arbitrary) TS for src in glob.glob(os.path.join(os.path.abspath(ts_data_dir), '*.npz'))[:10]: # Add the path to the copied file in cesium data directory ts_paths.append(os.path.abspath(shutil.copy( os.path.abspath(src), cfg['paths']['ts_data_folder']))) try: meta_features = list(load_ts(ts_paths[0]) .meta_features.keys()) except IndexError: # No TS data on disk meta_features = None files = [models.DatasetFile(uri=ts_path) for ts_path in ts_paths] dataset = models.Dataset(name=dataset_name, project=p, files=files, meta_features=meta_features) models.DBSession().add_all(files + [dataset]) models.DBSession().commit() print(f'Added dataset {dataset.id}') # Add featuresets fset_dict = {} for fset_name, orig_fset_path, features_list in [ ['Survey LC Cadence/Error Features', './survey_classifier_data/data/survey_lc_features.npz', CADENCE_FEATS], ['ASAS', './survey_classifier_data/data/ASAS_features.npz', GENERAL_FEATS + LOMB_SCARGLE_FEATS], ['CoRoT', './survey_classifier_data/data/noisified_CoRoT_features_100.npz', GENERAL_FEATS + LOMB_SCARGLE_FEATS], ['HATNet', './survey_classifier_data/data/noisified_HATNet_features_100.npz', GENERAL_FEATS + LOMB_SCARGLE_FEATS], ['Hipparcos', './survey_classifier_data/data/noisified_Hipparcos_features_100.npz', GENERAL_FEATS + LOMB_SCARGLE_FEATS], ['KELT', './survey_classifier_data/data/noisified_KELT_features_100.npz', GENERAL_FEATS + LOMB_SCARGLE_FEATS], ['Kepler', './survey_classifier_data/data/noisified_Kepler_features_100.npz', GENERAL_FEATS + LOMB_SCARGLE_FEATS], ['LINEAR', './survey_classifier_data/data/noisified_LINEAR_features_100.npz', GENERAL_FEATS + LOMB_SCARGLE_FEATS], ['OGLE-III', './survey_classifier_data/data/noisified_OGLE-III_features_100.npz', GENERAL_FEATS + LOMB_SCARGLE_FEATS], ['SuperWASP', './survey_classifier_data/data/noisified_SuperWASP_features_100.npz', GENERAL_FEATS + LOMB_SCARGLE_FEATS], ['TrES', './survey_classifier_data/data/noisified_TrES_features_100.npz', GENERAL_FEATS + LOMB_SCARGLE_FEATS]]: fset_path = os.path.abspath( shutil.copy(os.path.abspath(orig_fset_path), cfg['paths']['features_folder'])) fset = models.Featureset(name=fset_name, file_uri=fset_path, project=p, features_list=features_list, task_id=None, finished=datetime.datetime.now()) models.DBSession().add(fset) models.DBSession().commit() # fset.task_id = None # fset.finished = datetime.datetime.now() # fset.save() fset_dict[fset_name] = fset print(f'Added featureset {fset.id}') # Add models # TODO: Add actual model params for model_name, orig_model_path, model_type, params, fset_name in [ ['Survey LCs RFC', os.path.abspath('./survey_classifier_data/data/survey_classifier.pkl'), 'RandomForestClassifier', {}, 'Survey LC Cadence/Error Features'], ['ASAS', os.path.abspath('./survey_classifier_data/data/ASAS_model_compressed.pkl'), 'RandomForestClassifier', {}, 'ASAS'], ['CoRoT', os.path.abspath('./survey_classifier_data/data/noisified_CoRoT_model_compressed.pkl'), 'RandomForestClassifier', {}, 'CoRoT'], ['HATNet', os.path.abspath('./survey_classifier_data/data/noisified_HATNet_model_compressed.pkl'), 'RandomForestClassifier', {}, 'HATNet'], ['Hipparcos', os.path.abspath('./survey_classifier_data/data/noisified_Hipparcos_model_compressed.pkl'), 'RandomForestClassifier', {}, 'Hipparcos'], ['KELT', os.path.abspath('./survey_classifier_data/data/noisified_KELT_model_compressed.pkl'), 'RandomForestClassifier', {}, 'KELT'], ['Kepler', os.path.abspath('./survey_classifier_data/data/noisified_Kepler_model_compressed.pkl'), 'RandomForestClassifier', {}, 'Kepler'], ['LINEAR', os.path.abspath('./survey_classifier_data/data/noisified_LINEAR_model_compressed.pkl'), 'RandomForestClassifier', {}, 'LINEAR'], ['OGLE-III', os.path.abspath('./survey_classifier_data/data/noisified_OGLE-III_model_compressed.pkl'), 'RandomForestClassifier', {}, 'OGLE-III'], ['SuperWASP', os.path.abspath('./survey_classifier_data/data/noisified_SuperWASP_model_compressed.pkl'), 'RandomForestClassifier', {}, 'SuperWASP'], ['TrES', os.path.abspath('./survey_classifier_data/data/noisified_TrES_model_compressed.pkl'), 'RandomForestClassifier', {}, 'TrES']]: model_path = os.path.abspath( shutil.copy(orig_model_path, cfg['paths']['models_folder'])) model = models.Model(name=model_name, file_uri=model_path, featureset_id=fset_dict[fset_name].id, project=p, project_id=p.id, params=params, type=model_type, task_id=None, finished=datetime.datetime.now()) models.DBSession().add(model) models.DBSession().commit() # model.task_id = None # model.finished = datetime.datetime.now() # model.save() print(f'Added model {model.id}') print(cfg)
import os import pathlib from psycopg2 import OperationalError from baselayer.app import models from baselayer.app.config import load_config from baselayer.app.test_util import (driver, MyCustomWebDriver, set_server_url, reset_state) from skyportal.tests.fixtures import TMP_DIR, SourceFactory, GroupFactory, UserFactory from skyportal.model_util import create_token print('Loading test configuration from _test_config.yaml') basedir = pathlib.Path(os.path.dirname(__file__)) cfg = load_config([(basedir / '../../test_config.yaml').absolute()]) set_server_url(f'http://localhost:{cfg["ports:app"]}') print('Setting test database to:', cfg['database']) models.init_db(**cfg['database']) @pytest.fixture() def public_group(): return GroupFactory() @pytest.fixture() def public_source(public_group): return SourceFactory(groups=[public_group]) @pytest.fixture() def public_sources_205(public_group): return [SourceFactory(groups=[public_group]) for _ in range(205)]
import time from datetime import datetime import yaml from dateutil.parser import parse as parse_time from baselayer.app.env import load_env from baselayer.app.models import CronJobRun, DBSession, init_db from baselayer.log import make_log log = make_log("cron") env, cfg = load_env() jobs = cfg.get("cron", []) init_db(**cfg["database"]) timestamp_file = ".jobs_timestamps.yaml" class TimeCache: def __init__(self): if os.path.exists(timestamp_file): with open(timestamp_file) as f: timestamps = yaml.full_load(f)["timestamps"] else: timestamps = {} self.ts = timestamps def should_run(self, key, interval, limit=None):
help="Save JUnit xml output to `test-results/junit.xml`", ) parser.add_argument("--headless", action="store_true", help="Run browser headlessly") args = parser.parse_args() # Initialize the test database connection log("Connecting to test database") from baselayer.app.config import load_config from baselayer.app.models import init_db basedir = pathlib.Path(os.path.dirname(__file__)) / ".." / ".." cfg = load_config([basedir / TEST_CONFIG]) app_name = cfg["app.factory"].split(".")[0] engine = init_db(**cfg["database"]) engine.connect() if args.test_spec is not None: test_spec = args.test_spec else: test_spec = basedir / app_name / "tests" if args.xml: test_outdir = basedir / "test-results" if not test_outdir.exists(): test_outdir.mkdir() xml = f"--junitxml={test_outdir}/junit.xml" else: xml = ""