def get_java_version(mx_version):
if mx_version >= MXVersion("8.0.0"):
java_version = {
"version": os.getenv("JAVA_VERSION", "11.0.3"),
"vendor": "AdoptOpenJDK",
}
elif mx_version >= MXVersion("7.23.1"):
java_version = {
"version": os.getenv("JAVA_VERSION", "8u202"),
"vendor": "AdoptOpenJDK",
}
elif mx_version >= MXVersion("6.6"):
java_version = {
"version": os.getenv("JAVA_VERSION", "8u202"),
"vendor": "oracle",
}
elif mx_version >= MXVersion("6.0"):
java_version = {
"version": os.getenv("JAVA_VERSION", "8u51"),
"vendor": "oracle",
}
else:
java_version = {
"version": os.getenv("JAVA_VERSION", "7u80"),
"vendor": "oracle",
}
return java_version
def check_deprecation(version):
if version >= MXVersion("5.0.0") and version < MXVersion("6.0.0"):
logging.error("Mendix Runtime 5.x is no longer supported.")
logging.error("You can version pin on v3.8.0.")
return False
return True
def _runtime_sts_support(version):
if (version >= MXVersion("9.6.1")
or (version.major == 8 and version >= MXVersion("8.18.11"))
or (version.major == 7 and version >= MXVersion("7.23.30"))):
return True
# Only enable STS support for these versions when CERTIFICATE_AUTHORITIES
# is not set or STS will break.
elif (version >= 9.2 or
(version.major == 8 and version >= MXVersion("8.18.7")) or
(version.major == 7 and version >= MXVersion("7.23.22"))
) and not os.getenv("CERTIFICATE_AUTHORITIES", None):
return True
else:
return False
def get_version(build_path):
file_name = os.path.join(build_path, "model", "metadata.json")
try:
with open(file_name) as file_handle:
data = json.loads(file_handle.read())
return MXVersion(data["RuntimeVersion"])
except IOError:
mpr = util.get_mpr_file_from_dir(build_path)
if not mpr:
raise Exception("No model/metadata.json or .mpr found in archive")
cursor = sqlite3.connect(mpr).cursor()
cursor.execute("SELECT _ProductVersion FROM _MetaData LIMIT 1")
record = cursor.fetchone()
return MXVersion(record[0])
def configure(m2ee):
samesite_cookie_workaround_enabled = _is_samesite_cookie_workaround_enabled(
MXVersion(str(m2ee.config.get_runtime_version())))
if samesite_cookie_workaround_enabled:
logging.info("SameSite cookie workaround is enabled")
output_path = os.path.abspath(CONFIG_FILE)
template_path = os.path.abspath("{}.j2".format(CONFIG_FILE))
with open(template_path, "r") as file_:
template = Template(file_.read(), trim_blocks=True, lstrip_blocks=True)
rendered = template.render(
instadeploy_enabled=instadeploy.use_instadeploy(
m2ee.config.get_runtime_version()),
samesite_cookie_workaround_enabled=samesite_cookie_workaround_enabled,
locations=get_access_restriction_locations(),
default_headers=get_http_headers(),
nginx_port=str(util.get_nginx_port()),
runtime_port=str(util.get_runtime_port()),
admin_port=str(util.get_admin_port()),
deploy_port=str(util.get_deploy_port()),
root=os.getcwd(),
)
logging.debug("Writing nginx configuration file...")
with open(output_path, "w") as file_:
file_.write(rendered)
logging.debug("nginx configuration file written")
generate_password_file({"MxAdmin": security.get_m2ee_password()})
generate_password_file({"deploy": os.getenv("DEPLOY_PASSWORD")},
file_name_suffix="-mxbuild")
def set_up_files(m2ee):
lines = ""
if instadeploy.use_instadeploy(m2ee.config.get_runtime_version()):
mxbuild_upstream = "proxy_pass http://mendix_mxbuild"
else:
mxbuild_upstream = "return 501"
with open("nginx/conf/nginx.conf") as fh:
lines = "".join(fh.readlines())
samesite_cookie_workaround_enabled = _is_samesite_cookie_workaround_enabled(
MXVersion(str(m2ee.config.get_runtime_version())))
if samesite_cookie_workaround_enabled:
logging.info("SameSite cookie workaround is enabled")
http_headers = parse_headers(samesite_cookie_workaround_enabled)
lines = (lines.replace(
"CONFIG", get_path_config(samesite_cookie_workaround_enabled)).replace(
"NGINX_PORT", str(util.get_nginx_port())).replace(
"RUNTIME_PORT", str(util.get_runtime_port())).replace(
"ADMIN_PORT", str(util.get_admin_port())).replace(
"DEPLOY_PORT", str(util.get_deploy_port())).replace(
"ROOT", os.getcwd()).replace(
"HTTP_HEADERS",
http_headers).replace("MXBUILD_UPSTREAM",
mxbuild_upstream))
with open("nginx/conf/nginx.conf", "w") as fh:
fh.write(lines)
gen_htpasswd({"MxAdmin": security.get_m2ee_password()})
gen_htpasswd({"deploy": os.getenv("DEPLOY_PASSWORD")},
file_name_suffix="-mxbuild")
class TestCaseStorageOverride(TestCase):
@mock.patch.dict(
os.environ,
{"VCAP_SERVICES": S3_STORAGE_VCAP_EXAMPLE},
clear=True,
)
@mock.patch(
"buildpack.core.runtime.get_runtime_version",
mock.MagicMock(return_value=MXVersion(7.23)),
)
def _assert_storage_service_override(self, outcome, user_setting=True):
m2ee = M2EEMock()
if user_setting:
_upsert_core_storage_setting(m2ee, value=outcome)
storage.update_config(m2ee)
assert (
util.get_custom_runtime_setting(
m2ee, storage.STORAGE_CORE_CUSTOM_RUNTIME_SETTINGS_KEY
)
== outcome
)
def test_override_settings(self):
self._assert_storage_service_override("bar", user_setting=True)
def test_do_not_override_settings(self):
self._assert_storage_service_override(
"com.mendix.storage.s3", user_setting=False
)
def _get_scheduled_events(metadata):
scheduled_events = os.getenv("SCHEDULED_EVENTS", None)
# Scheduled events need to be enabled on every instance >= 9.12
if (get_runtime_version() < MXVersion(9.12)
and not util.is_cluster_leader()):
logging.debug(
"This instance is not a cluster leader, disabling scheduled events..."
)
return ("NONE", None)
elif scheduled_events is None or scheduled_events == "ALL":
logging.debug("Enabling all scheduled events")
return ("ALL", None)
elif scheduled_events == "NONE":
logging.debug("Disabling all scheduled events")
return ("NONE", None)
else:
parsed_scheduled_events = scheduled_events.split(",")
metadata_scheduled_events = [
scheduled_event["Name"]
for scheduled_event in metadata["ScheduledEvents"]
]
result = []
for scheduled_event in parsed_scheduled_events:
if scheduled_event not in metadata_scheduled_events:
logging.warning(
"Scheduled event defined but not detected in model: [%s]",
scheduled_event,
)
else:
result.append(scheduled_event)
logging.debug("Enabling scheduled events [%s]...", ",".join(result))
return ("SPECIFIED", result)
class TestNegativeMemoryMetricsThrowError(TestCase):
def test_validating_bad_metrics(self):
m2ee_stats = {"memory": {"javaheap": -12345}}
with self.assertRaises(RuntimeError):
PaidAppsMetricsEmitterThread._sanity_check_m2ee_stats(m2ee_stats)
def test_no_memorypools_good_metrics(self):
m2ee_stats = {"memory": {"javaheap": 12345}}
self.assertIsNone(
PaidAppsMetricsEmitterThread._sanity_check_m2ee_stats(m2ee_stats)
)
def test_non_ints_dont_cause_problems(self):
m2ee_stats = {
"memory": {
"javaheap": 123,
"memorypools": {"blah": "stuff"},
"foo": "bar",
}
}
self.assertIsNone(
PaidAppsMetricsEmitterThread._sanity_check_m2ee_stats(m2ee_stats)
)
def test_non_ints_dont_cause_problems_when_raising(self):
m2ee_stats = {
"memory": {
"javaheap": -123,
"memorypools": {"blah": "stuff"},
"foo": "bar",
}
}
with self.assertRaises(RuntimeError):
PaidAppsMetricsEmitterThread._sanity_check_m2ee_stats(m2ee_stats)
@patch(
"buildpack.core.runtime.get_runtime_version",
MagicMock(return_value=MXVersion(7.23)),
)
def test_underlying_log_message_propagates_upwards(self):
m2ee = Mock()
m2ee_stats = {
"memory": {
"javaheap": -123,
"memorypools": {"blah": "stuff"},
"foo": "bar",
}
}
interval = 1
metrics_emitter = PaidAppsMetricsEmitterThread(interval, m2ee)
with self.assertRaises(RuntimeError):
# ensure we log the error, before we raise the exception
with self.assertLogs(level="ERROR") as cm:
metrics_emitter._sanity_check_m2ee_stats(m2ee_stats)
# check the output logs contain the following message
self.assertIn("Memory stats with non-logical values", cm.output[-1])
def preflight_check():
logging.debug("pre-flight-check")
if not check_database_environment():
raise Exception("Missing environment variables")
mx_version_str = runtime.get_version(BUILD_DIR)
logging.info("Preflight check on version %s", mx_version_str)
mx_version = MXVersion(str(mx_version_str))
if not runtime.check_deprecation(mx_version):
raise Exception("Version {} is deprecated.".format(mx_version_str))
def test_guess_future_mendix_versions_doesnt_error(self):
m2ee_about = M2EEResponse(action="about",
json={
"feedback": {},
"result": 0
})
runtime_version = MXVersion("6.1.0")
stats = {}
guessed_version = _guess_java_version(m2ee_about, runtime_version,
stats)
self.assertEqual(8, guessed_version)
def _is_samesite_cookie_workaround_enabled(mx_version):
try:
return distutils.util.strtobool(
os.environ.get(
SAMESITE_COOKIE_WORKAROUND_ENV_KEY,
str(SAMESITE_COOKIE_WORKAROUND_DEFAULT),
)) and mx_version < MXVersion(
SAMESITE_COOKIE_WORKAROUND_LESS_MX_VERSION)
except (ValueError, AttributeError):
logging.warning(
"Invalid value for [%s], disabling SameSite cookie workaround",
SAMESITE_COOKIE_WORKAROUND_ENV_KEY,
)
return False
def get_runtime_version(build_path=BASE_PATH):
result = get_metadata_value("RuntimeVersion", build_path)
if result == None:
logging.debug(
"Cannot retrieve runtime version %s from metadata file, falling back to project file"
)
mpr = util.get_mpr_file_from_dir(build_path)
if not mpr:
raise Exception("No model/metadata.json or .mpr found in archive")
cursor = sqlite3.connect(mpr).cursor()
cursor.execute("SELECT _ProductVersion FROM _MetaData LIMIT 1")
record = cursor.fetchone()
result = record[0]
return MXVersion(result)
def test_guess_java11(self):
m2ee_about = M2EEResponse(
action="about",
json={
"feedback": {
"java_version": "11.0.3"
},
"result": 0
},
)
runtime_version = MXVersion("8.7.0.1476")
stats = {}
guessed_version = _guess_java_version(m2ee_about, runtime_version,
stats)
self.assertEqual(11, guessed_version)
def test_guess_java8(self):
m2ee_about = M2EEResponse(
action="about",
json={
"feedback": {
"java_version": "1.8.0_202"
},
"result": 0
},
)
runtime_version = MXVersion("7.2.3.7.55882")
stats = {}
guessed_version = _guess_java_version(m2ee_about, runtime_version,
stats)
self.assertEqual(8, guessed_version)
def test_guess_mendix_6_with_missing_java_version(self):
"""All Mendix 6 versions are supposed to use Java 8.
This is reachable in CloudV4 for Mendix runtimes <= 6.5.0, as the
runtime there does not expose the Java version from the about response.
"""
m2ee_about = M2EEResponse(action="about",
json={
"feedback": {},
"result": 0
})
runtime_version = MXVersion("6.1.0")
stats = {}
guessed_version = _guess_java_version(m2ee_about, runtime_version,
stats)
self.assertEqual(8, guessed_version)
class TestCaseAzureBlobStoreDryRun(TestCase):
azure_storage_vcap_example = """
{
"objectstore": [
{
"binding_name": null,
"credentials": {
"account_name": "sapcp4f4f4f4f4f4f4f4f4f4f",
"container_name": "sapcp-osaas-2d2d2d2d-cccc-4444-8888-4ed76dca688e",
"container_uri": "https://sapcp4f81hh2hps11tx7iuc7.blob.core.windows.net/sapcp-osaas-2d2d2d2d-cccc-4444-8888-4ed76dca688e",
"region": "westeurope",
"sas_token": "sig=JC1hALu1%2FOFA%1FzyuCzuZKivlb%1IIYktBYxHKPF2OJz3U%3D\u0026sv=2017-01-17\u0026spr=https\u0026si=77777777-ffff-4444-bbbb-bdeafdd87d00\u0026sr=c"
},
"instance_name": "test-azure-nl_Test_XgXgX",
"label": "objectstore",
"name": "test-azure-nl_Test_XgXgX",
"plan": "azure-standard",
"provider": null,
"syslog_drain_url": null,
"tags": [
"blobStore",
"objectStore"
],
"volume_mounts": []
}
]
}
""" # noqa
@mock.patch(
"buildpack.core.runtime.get_runtime_version",
mock.MagicMock(return_value=MXVersion(7.13)),
)
def test_azure_blob_store(self):
vcap = json.loads(self.azure_storage_vcap_example)
os.environ["MENDIX_BLOBSTORE_TYPE"] = "azure"
config = storage._get_azure_storage_specific_config(vcap)
assert (config["com.mendix.storage.azure.Container"] ==
"sapcp-osaas-2d2d2d2d-cccc-4444-8888-4ed76dca688e" # noqa
)
assert (
config["com.mendix.storage.azure.SharedAccessSignature"] ==
"sig=JC1hALu1%2FOFA%1FzyuCzuZKivlb%1IIYktBYxHKPF2OJz3U%3D\u0026sv=2017-01-17\u0026spr=https\u0026si=77777777-ffff-4444-bbbb-bdeafdd87d00\u0026sr=c" # noqa
)
def _inject_storage_stats(self, stats):
storage_stats = {}
runtime_version = runtime.get_runtime_version()
try:
storage_stats["get_number_of_files"] = self._get_number_of_files()
except Exception as e:
logging.warn("Metrics: Failed to retrieve number of files, " +
str(e))
raise
if runtime_version >= MXVersion("7.4.0"):
try:
storage_stats["get_size_of_files"] = self._get_size_of_files()
except Exception as e:
logging.warn("Metrics: Failed to retrieve size of files, " +
str(e))
raise
stats["storage"] = storage_stats
return stats
def get_client_certificates(mx_version):
config = {}
client_certificates_json = os.getenv("CLIENT_CERTIFICATES", "[]")
"""
[
{
'pfx': 'base64...', # required
'password': '',
'pin_to': ['Module.WS1', 'Module2.WS2'] # optional
},
{...}
]
"""
client_certificates = json.loads(client_certificates_json)
num = 0
files = []
passwords = []
pins = {}
for client_certificate in client_certificates:
pfx = base64.b64decode(client_certificate["pfx"])
location = os.path.abspath(".local/client_certificate.%d.crt" % num)
with open(location, "wb") as f:
f.write(pfx)
passwords.append(client_certificate["password"])
files.append(location)
if "pin_to" in client_certificate:
for ws in client_certificate["pin_to"]:
pins[ws] = location
num += 1
if len(files) > 0:
config["ClientCertificates"] = ",".join(files)
config["ClientCertificatePasswords"] = ",".join(passwords)
if mx_version < MXVersion("7.20"):
logging.debug(
"Runtime version < 7.20, using WebServiceClientCertificates..."
)
config["WebServiceClientCertificates"] = pins
else:
# Deprecated in 7.20
logging.debug(
"Runtime version >= 7.20, using ClientCertificateUsages..."
)
config["ClientCertificateUsages"] = pins
return config
def test_guess_mendix_5_java_7_with_missing_java_version(self):
"""For some Mendix 5 versions, Java version is not exposed, so
you are supposed to infer the Java information from some exposed
memory statistics.
This code is deprecated, but I am including unit tests against the
theoretical implementation to be on the safe side.
This may be overly defensive.
"""
m2ee_about = M2EEResponse(action="about",
json={
"feedback": {},
"result": 0
})
runtime_version = MXVersion("5.21.0")
stats = {"memory": {"used_nonheap": 2, "code": 1, "permanent": 1}}
guessed_version = _guess_java_version(m2ee_about, runtime_version,
stats)
self.assertEqual(7, guessed_version)
def preflight_check():
if not check_database_environment():
raise ValueError("Missing environment variables")
mx_version_str = runtime.get_version(BUILD_DIR)
stack = os.getenv("CF_STACK")
logging.info(
"Preflight check on Mendix runtime version [%s] and stack [%s]...",
mx_version_str,
stack,
)
mx_version = MXVersion(str(mx_version_str))
if not stack in SUPPORTED_STACKS:
raise NotImplementedError("Stack [{}] is not supported".format(stack))
if not runtime.check_deprecation(mx_version):
raise NotImplementedError(
"Mendix runtime version [{}] is not supported".format(
mx_version_str))
logging.info("Preflight check completed")
def test_mx6_not_supported(self):
assert not runtime.is_version_supported(MXVersion("6.2"))
def check_deprecation(version):
if version >= MXVersion("5.0.0") and version < MXVersion("6.0.0"):
return False
return True
def test_mx9_maintained(self):
assert runtime.is_version_maintained(MXVersion("9.6.9"))
assert runtime.is_version_maintained(MXVersion("9.12.1"))
assert runtime.is_version_maintained(MXVersion("9.13.0"))
def test_mx9_not_maintained(self):
assert not runtime.is_version_maintained(MXVersion("9.7"))
def test_mx8_maintained(self):
assert runtime.is_version_maintained(MXVersion("8.18.1"))
def test_mx7_not_maintained(self):
assert not runtime.is_version_maintained(MXVersion("7.16"))
def test_mx7_maintained(self):
assert runtime.is_version_maintained(MXVersion("7.23.1"))
def test_mx7_supported(self):
assert runtime.is_version_supported(MXVersion("7.16"))
def test_mx6_implemented(self):
assert runtime.is_version_implemented(MXVersion("6.7.5"))
