def ssda_configuration() -> DatabaseConfiguration:
ssda_db_config = dsnparse.parse_environ("SSDA_DSN")
return DatabaseConfiguration(
username=ssda_db_config.user,
password=ssda_db_config.secret,
host=ssda_db_config.host,
port=ssda_db_config.port,
database=ssda_db_config.database,
)
def sdb_configuration() -> DatabaseConfiguration:
sdb_db_config = dsnparse.parse_environ("SDB_DSN")
return DatabaseConfiguration(
username=sdb_db_config.user,
password=sdb_db_config.secret,
host=sdb_db_config.host,
port=3306,
database=sdb_db_config.database,
)
def sdb_database_connection():
sdb_db_config = dsnparse.parse_environ("SDB_DSN")
sdb_db_config = DatabaseConfiguration(
username=sdb_db_config.user,
password=sdb_db_config.secret,
host=sdb_db_config.host,
port=3306,
database=sdb_db_config.database,
)
sdb_connection = pymysql.connect(database=sdb_db_config.database(),
host=sdb_db_config.host(),
user=sdb_db_config.username(),
passwd=sdb_db_config.password())
return sdb_connection
def delete_in_ssda(fits: Optional[str], start: Optional[str], end: Optional[str], out: Optional[str]):
validate_options(fits=fits, start=start, end=end, out=out)
# database access
ssda_db_config = dsnparse.parse_environ("SSDA_DSN")
ssda_db_config = types.DatabaseConfiguration(
username=ssda_db_config.user,
password=ssda_db_config.secret,
host=ssda_db_config.host,
port=ssda_db_config.port,
database=ssda_db_config.database,
)
ssda_database_service = SSDADatabaseService(ssda_db_config)
ssda_database_service.begin_transaction()
try:
if fits:
observation_id = ssda_database_service.find_observation_id(artifact_path=fits)
if observation_id is None:
raise ValueError(f"No observation id found for {fits}")
ssda_database_service.delete_observation(
observation_id=observation_id
)
ssda_database_service.commit_transaction()
logging.info(msg=f"\nSuccessfully deleted observation for {fits}")
ssda_database_service.delete_observation(observation_id=observation_id)
else:
if start is None:
raise ValueError("The start date must not be None.")
if end is None:
raise ValueError("The end date must not be None.")
now = datetime.now
observation_paths = ssda_database_service.find_file_paths(
nights=DateRange(parse_date(start, now), parse_date(end, now))
)
with open(out, "w") as f:
for obs_path in observation_paths:
f.write("ssda delete -fits " + obs_path + "\n")
logging.info(msg=f""
f"The delete command does not support deleting files for "
f"a date range, as there might be cases where this leads "
f"to unexpected results. Please see the file {out} for "
f"the list of commands to execute for removing all "
f"observations in the date range.")
except BaseException as e:
ssda_database_service.rollback_transaction()
logging.error(msg="Failed to delete data.", exc_info=True)
logging.error(e)
def ssda_database_connection():
ssda_db_config = dsnparse.parse_environ("SSDA_DSN")
ssda_db_config = DatabaseConfiguration(
username=ssda_db_config.user,
password=ssda_db_config.secret,
host=ssda_db_config.host,
port=ssda_db_config.port,
database=ssda_db_config.database,
)
ssda_connection = psycopg2.connect(
database=ssda_db_config.database(),
host=ssda_db_config.host(),
user=ssda_db_config.username(),
password=ssda_db_config.password(),
)
return ssda_connection
def fetch_objectstore_credentials(swift_conn_id="swift_default"):
env_varname = f"AIRFLOW_CONN_{swift_conn_id.upper()}"
from . import env
result = {
"VERSION": "2.0",
"AUTHURL": env("OS_AUTH_URL"),
"REGION_NAME": "NL",
}
if env_varname not in os.environ and env_varname == "AIRFLOW_CONN_SWIFT_DEFAULT":
result.update({
"TENANT_NAME": env('OS_TENANT_NAME'),
"TENANT_ID": env('OS_TENANT_NAME'),
"USER": env('OS_USERNAME'),
"PASSWORD": env('OS_PASSWORD'),
})
else:
airflow_conn_info = dsnparse.parse_environ(env_varname)
result.update({
"TENANT_NAME": airflow_conn_info.host,
"TENANT_ID": airflow_conn_info.host,
"USER": airflow_conn_info.username,
"PASSWORD": airflow_conn_info.password,
})
return result
def test_parse_environ(self):
os.environ['ENVIRON_DSN'] = 'scheme://*****:*****@host:1234/foo'
r = dsnparse.parse_environ('ENVIRON_DSN')
self.assertEqual(os.environ['ENVIRON_DSN'], r.geturl())
def test_parse_environ(self):
os.environ['ENVIRON_DSN'] = 'scheme://*****:*****@host:1234/foo'
r = dsnparse.parse_environ('ENVIRON_DSN')
self.assertEqual(os.environ['ENVIRON_DSN'], r.geturl())
def populate_ssda(
start: Optional[str],
end: Optional[str],
instruments: Tuple[str],
file: Optional[str],
fits_base_dir: Optional[str],
skip_errors: bool,
verbosity: Optional[str],
) -> int:
logging.basicConfig(level=logging.INFO)
logging.error("SALT is always assumed to be the telescope.")
verbosity_level = 2 if not verbosity else int(verbosity)
if not os.environ.get("SENTRY_DSN"):
logging.warning(
"Environment variable SENTRY_DSN for logging with Sentry not "
"set.")
# convert options as required and validate them
now = datetime.now
start_date = parse_date(start, now) if start else None
end_date = parse_date(end, now) if end else None
if len(instruments):
instruments_set = set(
Instrument.for_name(instrument) for instrument in instruments)
else:
instruments_set = set(instrument for instrument in Instrument)
validate_options(start=start_date,
end=end_date,
fits_base_dir=fits_base_dir)
# Off we go!
start_msg = "Inserting data "
if start_date and end_date:
start_msg += (
f"for the nights from {start_date.isoformat()} to {end_date.isoformat()}"
)
if file:
start_msg += f"from {file}"
if verbosity_level >= 1:
logging.info(start_msg)
# store the base directory
if fits_base_dir:
set_fits_base_dir(fits_base_dir)
# get the FITS file paths
if start_date and end_date and fits_base_dir:
paths = fits_file_paths(
nights=DateRange(start_date, end_date),
instruments=instruments_set,
base_dir=fits_base_dir,
)
elif file:
paths = iter([file])
else:
raise click.UsageError(
"The command line options do not allow the FITS file paths to be found."
)
# database access
ssda_db_config = dsnparse.parse_environ("SSDA_DSN")
ssda_db_config = types.DatabaseConfiguration(
username=ssda_db_config.user,
password=ssda_db_config.secret,
host=ssda_db_config.host,
port=ssda_db_config.port,
database=ssda_db_config.database,
)
sdb_db_config = dsnparse.parse_environ("SDB_DSN")
sdb_db_config = types.DatabaseConfiguration(
username=sdb_db_config.user,
password=sdb_db_config.secret,
host=sdb_db_config.host,
port=3306,
database=sdb_db_config.database,
)
ssda_database_service = SSDADatabaseService(ssda_db_config)
sdb_database_service = SaltDatabaseService(sdb_db_config)
database_services = DatabaseServices(ssda=ssda_database_service,
sdb=sdb_database_service)
ssda_connection = database_services.ssda.connection()
daytime_errors: List[str] = list()
nighttime_errors: List[str] = list()
warnings: List[str] = list()
night_date = ""
# execute the requested task
for path in paths:
try:
if verbosity_level >= 1 and night_date != get_night_date(path):
night_date = get_night_date(path)
click.echo(f"Mapping files for {get_night_date(path)}")
clear_warnings()
execute_database_insert(
fits_path=path,
database_services=database_services,
)
if get_warnings():
handle_exception(
e=get_warnings()[0],
daytime_errors=daytime_errors,
nighttime_errors=nighttime_errors,
warnings=warnings,
verbosity_level=verbosity_level,
path=path,
)
except BaseException as e:
handle_exception(
e=e,
daytime_errors=daytime_errors,
nighttime_errors=nighttime_errors,
warnings=warnings,
verbosity_level=verbosity_level,
path=path,
)
if not skip_errors:
ssda_connection.close()
return -1
ssda_connection.close()
if verbosity_level >= 1:
msg = ""
msg += "NIGHTTIME ERRORS:\n"
msg += "-----------------\n"
if verbosity_level == 1:
msg += "\n"
if nighttime_errors:
for error in nighttime_errors:
if verbosity_level > 1:
msg += "\n"
msg += f"{error}\n"
else:
if verbosity_level > 1:
msg += "\n"
msg += "There are no nighttime errors.\n\n"
msg += "\n"
msg += "DAYTIME ERRORS:\n"
msg += "---------------\n"
if verbosity_level == 1:
msg += "\n"
if daytime_errors:
if verbosity_level > 1:
msg += "\n"
for error in daytime_errors:
msg += f"{error}\n"
else:
if verbosity_level > 1:
msg += "\n"
msg += "There are no daytime errors.\n\n"
msg += "\n"
msg += "WARNINGS:\n"
msg += "---------\n"
if verbosity_level == 1:
msg += "\n"
if warnings:
for warning in warnings:
if verbosity_level > 1:
msg += "\n"
msg += f"{warning}\n"
else:
if verbosity_level > 1:
msg += "\n"
msg += "There are no warnings.\n\n"
msg = f"""
Total number of nighttime errors: {len(nighttime_errors)}
Total number of daytime errors: {len(daytime_errors)}
Total number of warnings: {len(warnings)}
"""
logging.info(msg)
# Success!
return 0
from os import environ
from flask_login import UserMixin
from dsnparse import parse_environ
from peewee import (SqliteDatabase, PostgresqlDatabase, Model, CharField,
TextField, DateTimeField, ForeignKeyField, DecimalField,
IntegerField,)
if(environ.get('DATABASE_URL')):
url = parse_environ('DATABASE_URL')
db = PostgresqlDatabase(url.paths[0],
user=url.username,
password=url.password,
host=url.host,
port=url.port)
else:
db = SqliteDatabase('groovin.db')
class Promoter(Model, UserMixin):
name = CharField()
email = CharField(unique=True)
bio = TextField()
password = CharField()
def to_map(self):
data = self.__dict__['__data__']
data.pop('password')
return data
class Meta:
