def check_database_environment_variable():
try:
database_config.get_database_config()
return True
except Exception as e:
logging.error(
"You should provide a DATABASE_URL by adding a database service "
"to this application, it can be either MySQL or Postgres "
"If this is the first push of a new app, set up a database service "
"and push again afterwards: %s",
e,
)
return False
def _set_up_postgres():
# TODO: set up a way to disable this, on shared database (mxapps.io) we
# don't want to allow this.
if not buildpackutil.i_am_primary_instance():
return
dbconfig = database_config.get_database_config()
for k in (
"DatabaseType",
"DatabaseUserName",
"DatabasePassword",
"DatabaseHost",
):
if k not in dbconfig:
logger.warn(
"Skipping database configuration for DataDog because "
"configuration is not found. See database_config.py "
"for details"
)
return
if dbconfig["DatabaseType"] != "PostgreSQL":
return
with open(".local/datadog/conf.d/postgres.yaml", "w") as fh:
config = {
"init_config": {},
"instances": [
{
"host": dbconfig["DatabaseHost"].split(":")[0],
"port": int(dbconfig["DatabaseHost"].split(":")[1]),
"username": dbconfig["DatabaseUserName"],
"password": dbconfig["DatabasePassword"],
"dbname": dbconfig["DatabaseName"],
}
],
}
fh.write(yaml.safe_dump(config))
def _get_db_conn(self):
if self.db and self.db.closed != 0:
self.db.close()
self.db = None
if not self.db:
db_config = database_config.get_database_config()
if db_config["DatabaseType"] != "PostgreSQL":
raise Exception("Metrics only supports postgresql, not %s" %
db_config["DatabaseType"])
host_and_port = db_config["DatabaseHost"].split(":")
host = host_and_port[0]
if len(host_and_port) > 1:
port = int(host_and_port[1])
else:
port = 5432
self.db = psycopg2.connect(
"options='-c statement_timeout=60s'",
database=db_config["DatabaseName"],
user=db_config["DatabaseUserName"],
password=db_config["DatabasePassword"],
host=host,
port=port,
connect_timeout=3,
)
self.db.set_isolation_level(
psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT)
return self.db
def _get_db_conn(self):
if self.db and self.db.closed != 0:
self.db.close()
self.db = None
if not self.db:
# get_database config may return None or empty
db_config = database_config.get_database_config()
if not db_config or "DatabaseType" not in db_config:
raise ValueError(
"Database not set as VCAP or DATABASE_URL. Check "
"documentation to see supported configuration options.")
if db_config["DatabaseType"] != "PostgreSQL":
raise Exception("Metrics only supports postgresql, not %s" %
db_config["DatabaseType"])
host_and_port = db_config["DatabaseHost"].split(":")
host = host_and_port[0]
if len(host_and_port) > 1:
port = int(host_and_port[1])
else:
port = 5432
self.db = psycopg2.connect(
"options='-c statement_timeout=60s'",
database=db_config["DatabaseName"],
user=db_config["DatabaseUserName"],
password=db_config["DatabasePassword"],
host=host,
port=port,
connect_timeout=3,
)
self.db.set_isolation_level(
psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT)
return self.db
def _get_database_table_size(self):
conn = self._get_db_conn()
db_config = database_config.get_database_config()
with conn.cursor() as cursor:
cursor.execute("SELECT pg_database_size('%s');" %
(db_config["DatabaseName"], ))
rows = cursor.fetchall()
return int_or_default(rows[0][0])
def test_database_url(self):
self.clean_env()
os.environ[
"DATABASE_URL"] = "jdbc:postgres://*****:*****@host/database"
config = get_database_config()
assert config
assert config["DatabaseType"] == "PostgreSQL"
def test_mx_runtime_db_config(self):
"""
Test is MXRUNTIME variables are set up no database configuration is returned
based on DATABASE_URL or VCAP_SERVICES
"""
self.clean_env()
os.environ["MXRUNTIME_DatabaseType"] = "PostgreSQL"
os.environ[
"MXRUNTIME_DatabaseJdbcUrl"] = "postgres://*****:*****@rdsbroker-testfree-nonprod-1-eu-west-1.asdbjasdg.eu-west-1.rds.amazonaws.com:5432/testdatabase" # noqa E501
config = get_database_config()
assert not config
def set_runtime_config(metadata, mxruntime_config, vcap_data, m2ee):
scheduled_event_execution, my_scheduled_events = get_scheduled_events(
metadata
)
app_config = {
"ApplicationRootUrl": "https://%s" % vcap_data["application_uris"][0],
"MicroflowConstants": get_constants(metadata),
"ScheduledEventExecution": scheduled_event_execution,
}
if my_scheduled_events is not None:
app_config["MyScheduledEvents"] = my_scheduled_events
if is_development_mode():
logger.warning(
"Runtime is being started in Development Mode. Set "
'DEVELOPMENT_MODE to "false" (currently "true") to '
"set it to production."
)
app_config["DTAPMode"] = "D"
if m2ee.config.get_runtime_version() >= 7 and not i_am_primary_instance():
app_config["com.mendix.core.isClusterSlave"] = "true"
elif (
m2ee.config.get_runtime_version() >= 5.15
and os.getenv("ENABLE_STICKY_SESSIONS", "false").lower() == "true"
):
logger.info("Enabling sticky sessions")
app_config["com.mendix.core.SessionIdCookieName"] = "JSESSIONID"
buildpackutil.mkdir_p(os.path.join(os.getcwd(), "model", "resources"))
mxruntime_config.update(app_config)
# db configuration might be None, database should then be set up with
# MXRUNTIME_Database... custom runtime settings.
runtime_db_config = database_config.get_database_config(
development_mode=is_development_mode()
)
if runtime_db_config:
mxruntime_config.update(runtime_db_config)
mxruntime_config.update(get_filestore_config(m2ee))
mxruntime_config.update(get_certificate_authorities())
mxruntime_config.update(get_client_certificates())
mxruntime_config.update(get_custom_settings(metadata, mxruntime_config))
mxruntime_config.update(get_license_subscription())
mxruntime_config.update(get_custom_runtime_settings())
def _get_database_mutations(self):
conn = self._get_db_conn()
db_config = database_config.get_database_config()
with conn.cursor() as cursor:
cursor.execute("SELECT xact_commit, "
" xact_rollback, "
" tup_inserted, "
" tup_updated, "
" tup_deleted "
"FROM pg_stat_database "
"WHERE datname = '%s';" %
(db_config["DatabaseName"], ))
rows = cursor.fetchall()
return {
"xact_commit": int_or_default(rows[0][0]),
"xact_rollback": int_or_default(rows[0][1]),
"tup_inserted": int_or_default(rows[0][2]),
"tup_updated": int_or_default(rows[0][3]),
"tup_deleted": int_or_default(rows[0][4]),
}
return None
def test_vcap(self):
self.clean_env()
os.environ[
"VCAP_SERVICES"
] = """
{
"rds-testfree": [
{
"binding_name": null,
"credentials": {
"db_name": "dbuajsdhkasdhaks",
"host": "rdsbroker-testfree-nonprod-1-eu-west-1.asdbjasdg.eu-west-1.rds.amazonaws.com",
"password": "******",
"uri": "postgres://*****:*****@rdsbroker-testfree-nonprod-1-eu-west-1.asdbjasdg.eu-west-1.rds.amazonaws.com:5432/dbuajsdhkasdhaks",
"username": "******"
},
"instance_name": "ops-432a659e.test.foo.io-database",
"label": "rds-testfree",
"name": "ops-432a659e.test.foo.io-database",
"plan": "shared-psql-testfree",
"provider": null,
"syslog_drain_url": null,
"tags": [
"database",
"RDS",
"postgresql"
],
"volume_mounts": []
}
]
}
""" # noqa
config = get_database_config()
assert config
assert config["DatabaseType"] == "PostgreSQL"
def service_backups():
vcap_services = buildpackutil.get_vcap_services_data()
schnapps = None
amazon_s3 = None
for key in vcap_services:
if key.startswith("amazon-s3"):
amazon_s3 = key
if key.startswith("schnapps"):
schnapps = key
if not vcap_services or schnapps not in vcap_services:
logger.debug("No backup service detected")
return
backup_service = {}
if amazon_s3 in vcap_services:
s3_credentials = vcap_services[amazon_s3][0]["credentials"]
backup_service["filesCredentials"] = {
"accessKey": s3_credentials["access_key_id"],
"secretKey": s3_credentials["secret_access_key"],
"bucketName": s3_credentials["bucket"],
}
if "key_suffix" in s3_credentials: # Not all s3 plans have this field
backup_service["filesCredentials"]["keySuffix"] = s3_credentials[
"key_suffix"
]
try:
db_config = database_config.get_database_config()
if db_config["DatabaseType"] != "PostgreSQL":
raise Exception(
"Schnapps only supports postgresql, not %s"
% db_config["DatabaseType"]
)
host_and_port = db_config["DatabaseHost"].split(":")
backup_service["databaseCredentials"] = {
"host": host_and_port[0],
"username": db_config["DatabaseUserName"],
"password": db_config["DatabasePassword"],
"dbname": db_config["DatabaseName"],
"port": int(host_and_port[1]) if len(host_and_port) > 1 else 5432,
}
except Exception as e:
logger.exception(
"Schnapps will not be activated because error occurred with "
"parsing the database credentials"
)
return
schnapps_url = vcap_services[schnapps][0]["credentials"]["url"]
schnapps_api_key = vcap_services[schnapps][0]["credentials"]["apiKey"]
try:
result = requests.put(
schnapps_url,
headers={
"Content-Type": "application/json",
"apiKey": schnapps_api_key,
},
data=json.dumps(backup_service),
)
except requests.exceptions.SSLError as e:
logger.warning("Failed to contact backup service. SSLError: " + str(e))
return
except Exception as e:
logger.warning("Failed to contact backup service: ", exc_info=True)
return
if result.status_code == 200:
logger.info("Successfully updated backup service")
else:
logger.warning("Failed to update backup service: " + result.text)
def test_no_setup(self):
self.clean_env()
with self.assertRaises(RuntimeError):
get_database_config()
