def ssda_configuration() -> DatabaseConfiguration: ssda_db_config = dsnparse.parse_environ("SSDA_DSN") return DatabaseConfiguration( username=ssda_db_config.user, password=ssda_db_config.secret, host=ssda_db_config.host, port=ssda_db_config.port, database=ssda_db_config.database, )
def sdb_configuration() -> DatabaseConfiguration: sdb_db_config = dsnparse.parse_environ("SDB_DSN") return DatabaseConfiguration( username=sdb_db_config.user, password=sdb_db_config.secret, host=sdb_db_config.host, port=3306, database=sdb_db_config.database, )
def sdb_database_connection(): sdb_db_config = dsnparse.parse_environ("SDB_DSN") sdb_db_config = DatabaseConfiguration( username=sdb_db_config.user, password=sdb_db_config.secret, host=sdb_db_config.host, port=3306, database=sdb_db_config.database, ) sdb_connection = pymysql.connect(database=sdb_db_config.database(), host=sdb_db_config.host(), user=sdb_db_config.username(), passwd=sdb_db_config.password()) return sdb_connection
def delete_in_ssda(fits: Optional[str], start: Optional[str], end: Optional[str], out: Optional[str]): validate_options(fits=fits, start=start, end=end, out=out) # database access ssda_db_config = dsnparse.parse_environ("SSDA_DSN") ssda_db_config = types.DatabaseConfiguration( username=ssda_db_config.user, password=ssda_db_config.secret, host=ssda_db_config.host, port=ssda_db_config.port, database=ssda_db_config.database, ) ssda_database_service = SSDADatabaseService(ssda_db_config) ssda_database_service.begin_transaction() try: if fits: observation_id = ssda_database_service.find_observation_id(artifact_path=fits) if observation_id is None: raise ValueError(f"No observation id found for {fits}") ssda_database_service.delete_observation( observation_id=observation_id ) ssda_database_service.commit_transaction() logging.info(msg=f"\nSuccessfully deleted observation for {fits}") ssda_database_service.delete_observation(observation_id=observation_id) else: if start is None: raise ValueError("The start date must not be None.") if end is None: raise ValueError("The end date must not be None.") now = datetime.now observation_paths = ssda_database_service.find_file_paths( nights=DateRange(parse_date(start, now), parse_date(end, now)) ) with open(out, "w") as f: for obs_path in observation_paths: f.write("ssda delete -fits " + obs_path + "\n") logging.info(msg=f"" f"The delete command does not support deleting files for " f"a date range, as there might be cases where this leads " f"to unexpected results. Please see the file {out} for " f"the list of commands to execute for removing all " f"observations in the date range.") except BaseException as e: ssda_database_service.rollback_transaction() logging.error(msg="Failed to delete data.", exc_info=True) logging.error(e)
def ssda_database_connection(): ssda_db_config = dsnparse.parse_environ("SSDA_DSN") ssda_db_config = DatabaseConfiguration( username=ssda_db_config.user, password=ssda_db_config.secret, host=ssda_db_config.host, port=ssda_db_config.port, database=ssda_db_config.database, ) ssda_connection = psycopg2.connect( database=ssda_db_config.database(), host=ssda_db_config.host(), user=ssda_db_config.username(), password=ssda_db_config.password(), ) return ssda_connection
def fetch_objectstore_credentials(swift_conn_id="swift_default"): env_varname = f"AIRFLOW_CONN_{swift_conn_id.upper()}" from . import env result = { "VERSION": "2.0", "AUTHURL": env("OS_AUTH_URL"), "REGION_NAME": "NL", } if env_varname not in os.environ and env_varname == "AIRFLOW_CONN_SWIFT_DEFAULT": result.update({ "TENANT_NAME": env('OS_TENANT_NAME'), "TENANT_ID": env('OS_TENANT_NAME'), "USER": env('OS_USERNAME'), "PASSWORD": env('OS_PASSWORD'), }) else: airflow_conn_info = dsnparse.parse_environ(env_varname) result.update({ "TENANT_NAME": airflow_conn_info.host, "TENANT_ID": airflow_conn_info.host, "USER": airflow_conn_info.username, "PASSWORD": airflow_conn_info.password, }) return result
def test_parse_environ(self): os.environ['ENVIRON_DSN'] = 'scheme://*****:*****@host:1234/foo' r = dsnparse.parse_environ('ENVIRON_DSN') self.assertEqual(os.environ['ENVIRON_DSN'], r.geturl())
def test_parse_environ(self): os.environ['ENVIRON_DSN'] = 'scheme://*****:*****@host:1234/foo' r = dsnparse.parse_environ('ENVIRON_DSN') self.assertEqual(os.environ['ENVIRON_DSN'], r.geturl())
def populate_ssda( start: Optional[str], end: Optional[str], instruments: Tuple[str], file: Optional[str], fits_base_dir: Optional[str], skip_errors: bool, verbosity: Optional[str], ) -> int: logging.basicConfig(level=logging.INFO) logging.error("SALT is always assumed to be the telescope.") verbosity_level = 2 if not verbosity else int(verbosity) if not os.environ.get("SENTRY_DSN"): logging.warning( "Environment variable SENTRY_DSN for logging with Sentry not " "set.") # convert options as required and validate them now = datetime.now start_date = parse_date(start, now) if start else None end_date = parse_date(end, now) if end else None if len(instruments): instruments_set = set( Instrument.for_name(instrument) for instrument in instruments) else: instruments_set = set(instrument for instrument in Instrument) validate_options(start=start_date, end=end_date, fits_base_dir=fits_base_dir) # Off we go! start_msg = "Inserting data " if start_date and end_date: start_msg += ( f"for the nights from {start_date.isoformat()} to {end_date.isoformat()}" ) if file: start_msg += f"from {file}" if verbosity_level >= 1: logging.info(start_msg) # store the base directory if fits_base_dir: set_fits_base_dir(fits_base_dir) # get the FITS file paths if start_date and end_date and fits_base_dir: paths = fits_file_paths( nights=DateRange(start_date, end_date), instruments=instruments_set, base_dir=fits_base_dir, ) elif file: paths = iter([file]) else: raise click.UsageError( "The command line options do not allow the FITS file paths to be found." ) # database access ssda_db_config = dsnparse.parse_environ("SSDA_DSN") ssda_db_config = types.DatabaseConfiguration( username=ssda_db_config.user, password=ssda_db_config.secret, host=ssda_db_config.host, port=ssda_db_config.port, database=ssda_db_config.database, ) sdb_db_config = dsnparse.parse_environ("SDB_DSN") sdb_db_config = types.DatabaseConfiguration( username=sdb_db_config.user, password=sdb_db_config.secret, host=sdb_db_config.host, port=3306, database=sdb_db_config.database, ) ssda_database_service = SSDADatabaseService(ssda_db_config) sdb_database_service = SaltDatabaseService(sdb_db_config) database_services = DatabaseServices(ssda=ssda_database_service, sdb=sdb_database_service) ssda_connection = database_services.ssda.connection() daytime_errors: List[str] = list() nighttime_errors: List[str] = list() warnings: List[str] = list() night_date = "" # execute the requested task for path in paths: try: if verbosity_level >= 1 and night_date != get_night_date(path): night_date = get_night_date(path) click.echo(f"Mapping files for {get_night_date(path)}") clear_warnings() execute_database_insert( fits_path=path, database_services=database_services, ) if get_warnings(): handle_exception( e=get_warnings()[0], daytime_errors=daytime_errors, nighttime_errors=nighttime_errors, warnings=warnings, verbosity_level=verbosity_level, path=path, ) except BaseException as e: handle_exception( e=e, daytime_errors=daytime_errors, nighttime_errors=nighttime_errors, warnings=warnings, verbosity_level=verbosity_level, path=path, ) if not skip_errors: ssda_connection.close() return -1 ssda_connection.close() if verbosity_level >= 1: msg = "" msg += "NIGHTTIME ERRORS:\n" msg += "-----------------\n" if verbosity_level == 1: msg += "\n" if nighttime_errors: for error in nighttime_errors: if verbosity_level > 1: msg += "\n" msg += f"{error}\n" else: if verbosity_level > 1: msg += "\n" msg += "There are no nighttime errors.\n\n" msg += "\n" msg += "DAYTIME ERRORS:\n" msg += "---------------\n" if verbosity_level == 1: msg += "\n" if daytime_errors: if verbosity_level > 1: msg += "\n" for error in daytime_errors: msg += f"{error}\n" else: if verbosity_level > 1: msg += "\n" msg += "There are no daytime errors.\n\n" msg += "\n" msg += "WARNINGS:\n" msg += "---------\n" if verbosity_level == 1: msg += "\n" if warnings: for warning in warnings: if verbosity_level > 1: msg += "\n" msg += f"{warning}\n" else: if verbosity_level > 1: msg += "\n" msg += "There are no warnings.\n\n" msg = f""" Total number of nighttime errors: {len(nighttime_errors)} Total number of daytime errors: {len(daytime_errors)} Total number of warnings: {len(warnings)} """ logging.info(msg) # Success! return 0
from os import environ from flask_login import UserMixin from dsnparse import parse_environ from peewee import (SqliteDatabase, PostgresqlDatabase, Model, CharField, TextField, DateTimeField, ForeignKeyField, DecimalField, IntegerField,) if(environ.get('DATABASE_URL')): url = parse_environ('DATABASE_URL') db = PostgresqlDatabase(url.paths[0], user=url.username, password=url.password, host=url.host, port=url.port) else: db = SqliteDatabase('groovin.db') class Promoter(Model, UserMixin): name = CharField() email = CharField(unique=True) bio = TextField() password = CharField() def to_map(self): data = self.__dict__['__data__'] data.pop('password') return data class Meta: