def _connect(self):
env, cfg = load_env()
self.basedir = Path(os.path.dirname(__file__)) / '..'
(self.basedir / 'static/thumbnails').mkdir(parents=True, exist_ok=True)
with status(f"Connecting to database {cfg['database']['database']}"):
init_db(**cfg['database'])
def api(method,
endpoint,
data=None,
params=None,
host=None,
token=None,
raw_response=False):
"""Make a SkyPortal API call.
Parameters
----------
method : {'GET', 'POST', 'PUT', ...}
HTTP method.
endpoint : string
Relative API endpoint. E.g., `sources` means
`http://localhost:port/api/sources`.
data : dict
Data to post
params : dict
URL parameters, in GET
host : str
Defaults to http://localhost on the port specified in the
SkyPortal configuration file.
token : str
A token, for when authentication is needed. This is placed in the
`Authorization` header.
raw_response : bool
Return the response object, instead of the status code and parsed json.
Returns
-------
code : str
HTTP status code, if `raw_response` is False.
json : dict
Response JSON, if `raw_response` is False.
"""
if host is None:
env, cfg = load_env()
host = f'http://localhost:{cfg["ports.app"]}'
url = urllib.parse.urljoin(host, f'/api/{endpoint}')
headers = {'Authorization': f'token {token}'} if token else None
response = requests.request(method,
url,
json=data,
params=params,
headers=headers)
if raw_response:
return response
else:
if response.status_code in (200, 400):
if method == "HEAD":
return response.status_code, None
else:
return response.status_code, response.json()
else:
return response.status_code, None
def provision_public_group():
"""If public group name is set in the config file, create it."""
env, cfg = load_env()
public_group_name = cfg['misc.public_group_name']
pg = Group.query.filter(Group.name == public_group_name).first()
if pg is None:
DBSession().add(Group(name=public_group_name))
DBSession().commit()
def fill_config_file_values(template_paths):
env, cfg = load_env()
for template_path in template_paths:
with open(template_path) as f:
data = f.read()
data_compiled = re.sub('\$\$\{\{([a-zA-Z_]+)?\}\}', '{\\1}',
data.replace('{', '{{').replace('}', '}}'))
with open(os.path.splitext(template_path)[0], 'w') as f:
f.write(data_compiled.format(**cfg['ports']))
def fill_config_file_values(template_paths):
log('Compiling configuration templates')
env, cfg = load_env()
for template_path in template_paths:
with status(template_path):
with open(template_path) as f:
data = f.read()
template = jinja2.Template(data)
rendered = template.render(cfg)
with open(os.path.splitext(template_path)[0], 'w') as f:
f.write(rendered)
def fill_config_file_values(template_paths):
log('Compiling configuration templates')
env, cfg = load_env()
for template_path in template_paths:
with status(template_path):
tpath, tfile = os.path.split(template_path)
env = jinja2.Environment(loader=jinja2.FileSystemLoader(tpath), )
env.filters['md5sum'] = md5sum
template = env.get_template(tfile)
rendered = template.render(cfg)
with open(os.path.splitext(template_path)[0], 'w') as f:
f.write(rendered)
f.write('\n')
def copy_supervisor_configs():
env, cfg = load_env()
services = {}
for path in cfg["services.paths"]:
if os.path.exists(path):
path_services = [
d for d in os.listdir(path) if os.path.isdir(pjoin(path, d))
]
services.update({s: pjoin(path, s) for s in path_services})
duplicates = [k for k, v in Counter(services.keys()).items() if v > 1]
if duplicates:
raise RuntimeError(
f"Duplicate service definitions found for {duplicates}")
log(f"Discovered {len(services)} services")
disabled = cfg["services.disabled"] or []
enabled = cfg["services.enabled"] or []
both = set().union(disabled).intersection(enabled)
if both:
raise RuntimeError(
f"Invalid service specification: {both} in both enabled and disabled"
)
if disabled == "*":
disabled = services.keys()
if enabled == "*":
enabled = []
services_to_run = set(services.keys()).difference(disabled).union(enabled)
log(f"Enabling {len(services_to_run)} services")
supervisor_configs = []
for service in services_to_run:
path = services[service]
supervisor_conf = pjoin(path, "supervisor.conf")
if os.path.exists(supervisor_conf):
with open(supervisor_conf) as f:
supervisor_configs.append(f.read())
with open("baselayer/conf/supervisor/supervisor.conf", "a") as f:
f.write("\n\n".join(supervisor_configs))
def fill_config_file_values(template_paths):
log("Compiling configuration templates")
env, cfg = load_env()
for template_path in template_paths:
with status(template_path):
tpath, tfile = os.path.split(template_path)
jenv = jinja2.Environment(
loader=jinja2.FileSystemLoader(tpath),
)
jenv.filters.update(custom_filters)
template = jenv.get_template(tfile)
cfg['env'] = env
rendered = template.render(cfg)
with open(os.path.splitext(template_path)[0], "w") as f:
f.write(rendered)
f.write("\n")
'Super admin':
all_acl_ids,
'Group admin': [
'Annotate',
'Comment',
'Manage sources',
'Upload data',
'Post taxonomy',
'Manage users',
'Classify',
],
'Full user': ['Annotate', 'Comment', 'Upload data', 'Classify'],
'View only': [],
}
env, cfg = load_env()
def add_user(username, roles=[], auth=False, first_name=None, last_name=None):
user = User.query.filter(User.username == username).first()
if user is None:
user = User(username=username,
first_name=first_name,
last_name=last_name)
if auth:
TornadoStorage.user.create_social_auth(user, user.username,
'google-oauth2')
for rolename in roles:
role = Role.query.get(rolename)
if role not in user.roles:
def setup_survey_db():
# if os.getcwd().endswith('survey_app'):
# os.chdir('./cesium_web')
env, cfg = load_env()
for data_dir in cfg['paths'].values():
if not os.path.exists(data_dir):
os.makedirs(data_dir)
db_session = init_db(**baselayer.app.config.load_config()['database'])
# Drop & create tables
with status('Dropping and re-creating tables'):
drop_tables()
create_tables()
# Add testuser
with status('Adding testuser'):
u = models.User(username='******')
models.DBSession().add(u)
models.DBSession().commit()
# Add project
with status('Adding project'):
p = models.Project(name='Survey Classifier', users=[u])
models.DBSession().add(p)
models.DBSession().commit()
# Add datasets
with status('Adding datasets'):
for dataset_name, ts_data_dir in [
['Survey Light Curve Data',
'survey_classifier_data/data/lightcurves'],
['ASAS',
'survey_classifier_data/data/ASAS_lcs'],
['Noisified to CoRoT',
'survey_classifier_data/data/noisified_CoRoT_lcs'],
['Noisified to HATNet',
'survey_classifier_data/data/noisified_HATNet_lcs'],
['Noisified to Hipparcos',
'survey_classifier_data/data/noisified_Hipparcos_lcs'],
['Noisified to KELT',
'survey_classifier_data/data/noisified_KELT_lcs'],
['Noisified to Kepler',
'survey_classifier_data/data/noisified_Kepler_lcs'],
['Noisified to LINEAR',
'survey_classifier_data/data/noisified_LINEAR_lcs'],
['Noisified to OGLE-III',
'survey_classifier_data/data/noisified_OGLE-III_lcs'],
['Noisified to SuperWASP',
'survey_classifier_data/data/noisified_SuperWASP_lcs'],
['Noisified to TrES',
'survey_classifier_data/data/noisified_TrES_lcs']]:
ts_paths = []
# As these are only ever accessed to determine meta features, only
# copy first ten (arbitrary) TS
for src in glob.glob(os.path.join(os.path.abspath(ts_data_dir),
'*.npz'))[:10]:
# Add the path to the copied file in cesium data directory
ts_paths.append(os.path.abspath(shutil.copy(
os.path.abspath(src), cfg['paths']['ts_data_folder'])))
try:
meta_features = list(load_ts(ts_paths[0])
.meta_features.keys())
except IndexError: # No TS data on disk
meta_features = None
files = [models.DatasetFile(uri=ts_path) for ts_path in ts_paths]
dataset = models.Dataset(name=dataset_name, project=p, files=files,
meta_features=meta_features)
models.DBSession().add_all(files + [dataset])
models.DBSession().commit()
print(f'Added dataset {dataset.id}')
# Add featuresets
fset_dict = {}
for fset_name, orig_fset_path, features_list in [
['Survey LC Cadence/Error Features',
'./survey_classifier_data/data/survey_lc_features.npz',
CADENCE_FEATS],
['ASAS',
'./survey_classifier_data/data/ASAS_features.npz',
GENERAL_FEATS + LOMB_SCARGLE_FEATS],
['CoRoT',
'./survey_classifier_data/data/noisified_CoRoT_features_100.npz',
GENERAL_FEATS + LOMB_SCARGLE_FEATS],
['HATNet',
'./survey_classifier_data/data/noisified_HATNet_features_100.npz',
GENERAL_FEATS + LOMB_SCARGLE_FEATS],
['Hipparcos',
'./survey_classifier_data/data/noisified_Hipparcos_features_100.npz',
GENERAL_FEATS + LOMB_SCARGLE_FEATS],
['KELT',
'./survey_classifier_data/data/noisified_KELT_features_100.npz',
GENERAL_FEATS + LOMB_SCARGLE_FEATS],
['Kepler',
'./survey_classifier_data/data/noisified_Kepler_features_100.npz',
GENERAL_FEATS + LOMB_SCARGLE_FEATS],
['LINEAR',
'./survey_classifier_data/data/noisified_LINEAR_features_100.npz',
GENERAL_FEATS + LOMB_SCARGLE_FEATS],
['OGLE-III',
'./survey_classifier_data/data/noisified_OGLE-III_features_100.npz',
GENERAL_FEATS + LOMB_SCARGLE_FEATS],
['SuperWASP',
'./survey_classifier_data/data/noisified_SuperWASP_features_100.npz',
GENERAL_FEATS + LOMB_SCARGLE_FEATS],
['TrES',
'./survey_classifier_data/data/noisified_TrES_features_100.npz',
GENERAL_FEATS + LOMB_SCARGLE_FEATS]]:
fset_path = os.path.abspath(
shutil.copy(os.path.abspath(orig_fset_path),
cfg['paths']['features_folder']))
fset = models.Featureset(name=fset_name, file_uri=fset_path,
project=p, features_list=features_list,
task_id=None, finished=datetime.datetime.now())
models.DBSession().add(fset)
models.DBSession().commit()
# fset.task_id = None
# fset.finished = datetime.datetime.now()
# fset.save()
fset_dict[fset_name] = fset
print(f'Added featureset {fset.id}')
# Add models
# TODO: Add actual model params
for model_name, orig_model_path, model_type, params, fset_name in [
['Survey LCs RFC',
os.path.abspath('./survey_classifier_data/data/survey_classifier.pkl'),
'RandomForestClassifier', {}, 'Survey LC Cadence/Error Features'],
['ASAS',
os.path.abspath('./survey_classifier_data/data/ASAS_model_compressed.pkl'),
'RandomForestClassifier', {}, 'ASAS'],
['CoRoT',
os.path.abspath('./survey_classifier_data/data/noisified_CoRoT_model_compressed.pkl'),
'RandomForestClassifier', {}, 'CoRoT'],
['HATNet',
os.path.abspath('./survey_classifier_data/data/noisified_HATNet_model_compressed.pkl'),
'RandomForestClassifier', {}, 'HATNet'],
['Hipparcos',
os.path.abspath('./survey_classifier_data/data/noisified_Hipparcos_model_compressed.pkl'),
'RandomForestClassifier', {}, 'Hipparcos'],
['KELT',
os.path.abspath('./survey_classifier_data/data/noisified_KELT_model_compressed.pkl'),
'RandomForestClassifier', {}, 'KELT'],
['Kepler',
os.path.abspath('./survey_classifier_data/data/noisified_Kepler_model_compressed.pkl'),
'RandomForestClassifier', {}, 'Kepler'],
['LINEAR',
os.path.abspath('./survey_classifier_data/data/noisified_LINEAR_model_compressed.pkl'),
'RandomForestClassifier', {}, 'LINEAR'],
['OGLE-III',
os.path.abspath('./survey_classifier_data/data/noisified_OGLE-III_model_compressed.pkl'),
'RandomForestClassifier', {}, 'OGLE-III'],
['SuperWASP',
os.path.abspath('./survey_classifier_data/data/noisified_SuperWASP_model_compressed.pkl'),
'RandomForestClassifier', {}, 'SuperWASP'],
['TrES',
os.path.abspath('./survey_classifier_data/data/noisified_TrES_model_compressed.pkl'),
'RandomForestClassifier', {}, 'TrES']]:
model_path = os.path.abspath(
shutil.copy(orig_model_path, cfg['paths']['models_folder']))
model = models.Model(name=model_name, file_uri=model_path,
featureset_id=fset_dict[fset_name].id, project=p,
project_id=p.id,
params=params, type=model_type, task_id=None,
finished=datetime.datetime.now())
models.DBSession().add(model)
models.DBSession().commit()
# model.task_id = None
# model.finished = datetime.datetime.now()
# model.save()
print(f'Added model {model.id}')
print(cfg)
with status("Inserting dummy model... "):
m = models.Model(project=p, featureset=f, name='test model',
params={'n_estimators': 10}, type='RFC',
file_uri='/tmp/blah.pkl')
models.DBSession().add(m)
models.DBSession().commit()
with status("Inserting dummy prediction... "):
pr = models.Prediction(project=p, model=m, file_uri='/tmp/blergh.pkl', dataset=d)
models.DBSession().add(pr)
models.DBSession().commit()
def create_token_user(bot_name, project_ids):
u = models.User(username=bot_name)
p = models.Project.query.filter(models.Project.id.in_(project_ids)).all()
u.projects.extend(p)
t = models.Token(user=u)
models.DBSession().add_all([u, t])
models.DBSession().commit()
return t.id
if __name__ == "__main__":
env, cfg = load_env()
with status(f"Connecting to database {cfg['database']['database']}"):
models.init_db(**cfg['database'])
insert_test_data()
