def get_java_version(mx_version): if mx_version >= MXVersion("8.0.0"): java_version = { "version": os.getenv("JAVA_VERSION", "11.0.3"), "vendor": "AdoptOpenJDK", } elif mx_version >= MXVersion("7.23.1"): java_version = { "version": os.getenv("JAVA_VERSION", "8u202"), "vendor": "AdoptOpenJDK", } elif mx_version >= MXVersion("6.6"): java_version = { "version": os.getenv("JAVA_VERSION", "8u202"), "vendor": "oracle", } elif mx_version >= MXVersion("6.0"): java_version = { "version": os.getenv("JAVA_VERSION", "8u51"), "vendor": "oracle", } else: java_version = { "version": os.getenv("JAVA_VERSION", "7u80"), "vendor": "oracle", } return java_version
def check_deprecation(version): if version >= MXVersion("5.0.0") and version < MXVersion("6.0.0"): logging.error("Mendix Runtime 5.x is no longer supported.") logging.error("You can version pin on v3.8.0.") return False return True
def _runtime_sts_support(version): if (version >= MXVersion("9.6.1") or (version.major == 8 and version >= MXVersion("8.18.11")) or (version.major == 7 and version >= MXVersion("7.23.30"))): return True # Only enable STS support for these versions when CERTIFICATE_AUTHORITIES # is not set or STS will break. elif (version >= 9.2 or (version.major == 8 and version >= MXVersion("8.18.7")) or (version.major == 7 and version >= MXVersion("7.23.22")) ) and not os.getenv("CERTIFICATE_AUTHORITIES", None): return True else: return False
def get_version(build_path): file_name = os.path.join(build_path, "model", "metadata.json") try: with open(file_name) as file_handle: data = json.loads(file_handle.read()) return MXVersion(data["RuntimeVersion"]) except IOError: mpr = util.get_mpr_file_from_dir(build_path) if not mpr: raise Exception("No model/metadata.json or .mpr found in archive") cursor = sqlite3.connect(mpr).cursor() cursor.execute("SELECT _ProductVersion FROM _MetaData LIMIT 1") record = cursor.fetchone() return MXVersion(record[0])
def configure(m2ee): samesite_cookie_workaround_enabled = _is_samesite_cookie_workaround_enabled( MXVersion(str(m2ee.config.get_runtime_version()))) if samesite_cookie_workaround_enabled: logging.info("SameSite cookie workaround is enabled") output_path = os.path.abspath(CONFIG_FILE) template_path = os.path.abspath("{}.j2".format(CONFIG_FILE)) with open(template_path, "r") as file_: template = Template(file_.read(), trim_blocks=True, lstrip_blocks=True) rendered = template.render( instadeploy_enabled=instadeploy.use_instadeploy( m2ee.config.get_runtime_version()), samesite_cookie_workaround_enabled=samesite_cookie_workaround_enabled, locations=get_access_restriction_locations(), default_headers=get_http_headers(), nginx_port=str(util.get_nginx_port()), runtime_port=str(util.get_runtime_port()), admin_port=str(util.get_admin_port()), deploy_port=str(util.get_deploy_port()), root=os.getcwd(), ) logging.debug("Writing nginx configuration file...") with open(output_path, "w") as file_: file_.write(rendered) logging.debug("nginx configuration file written") generate_password_file({"MxAdmin": security.get_m2ee_password()}) generate_password_file({"deploy": os.getenv("DEPLOY_PASSWORD")}, file_name_suffix="-mxbuild")
def set_up_files(m2ee): lines = "" if instadeploy.use_instadeploy(m2ee.config.get_runtime_version()): mxbuild_upstream = "proxy_pass http://mendix_mxbuild" else: mxbuild_upstream = "return 501" with open("nginx/conf/nginx.conf") as fh: lines = "".join(fh.readlines()) samesite_cookie_workaround_enabled = _is_samesite_cookie_workaround_enabled( MXVersion(str(m2ee.config.get_runtime_version()))) if samesite_cookie_workaround_enabled: logging.info("SameSite cookie workaround is enabled") http_headers = parse_headers(samesite_cookie_workaround_enabled) lines = (lines.replace( "CONFIG", get_path_config(samesite_cookie_workaround_enabled)).replace( "NGINX_PORT", str(util.get_nginx_port())).replace( "RUNTIME_PORT", str(util.get_runtime_port())).replace( "ADMIN_PORT", str(util.get_admin_port())).replace( "DEPLOY_PORT", str(util.get_deploy_port())).replace( "ROOT", os.getcwd()).replace( "HTTP_HEADERS", http_headers).replace("MXBUILD_UPSTREAM", mxbuild_upstream)) with open("nginx/conf/nginx.conf", "w") as fh: fh.write(lines) gen_htpasswd({"MxAdmin": security.get_m2ee_password()}) gen_htpasswd({"deploy": os.getenv("DEPLOY_PASSWORD")}, file_name_suffix="-mxbuild")
class TestCaseStorageOverride(TestCase): @mock.patch.dict( os.environ, {"VCAP_SERVICES": S3_STORAGE_VCAP_EXAMPLE}, clear=True, ) @mock.patch( "buildpack.core.runtime.get_runtime_version", mock.MagicMock(return_value=MXVersion(7.23)), ) def _assert_storage_service_override(self, outcome, user_setting=True): m2ee = M2EEMock() if user_setting: _upsert_core_storage_setting(m2ee, value=outcome) storage.update_config(m2ee) assert ( util.get_custom_runtime_setting( m2ee, storage.STORAGE_CORE_CUSTOM_RUNTIME_SETTINGS_KEY ) == outcome ) def test_override_settings(self): self._assert_storage_service_override("bar", user_setting=True) def test_do_not_override_settings(self): self._assert_storage_service_override( "com.mendix.storage.s3", user_setting=False )
def _get_scheduled_events(metadata): scheduled_events = os.getenv("SCHEDULED_EVENTS", None) # Scheduled events need to be enabled on every instance >= 9.12 if (get_runtime_version() < MXVersion(9.12) and not util.is_cluster_leader()): logging.debug( "This instance is not a cluster leader, disabling scheduled events..." ) return ("NONE", None) elif scheduled_events is None or scheduled_events == "ALL": logging.debug("Enabling all scheduled events") return ("ALL", None) elif scheduled_events == "NONE": logging.debug("Disabling all scheduled events") return ("NONE", None) else: parsed_scheduled_events = scheduled_events.split(",") metadata_scheduled_events = [ scheduled_event["Name"] for scheduled_event in metadata["ScheduledEvents"] ] result = [] for scheduled_event in parsed_scheduled_events: if scheduled_event not in metadata_scheduled_events: logging.warning( "Scheduled event defined but not detected in model: [%s]", scheduled_event, ) else: result.append(scheduled_event) logging.debug("Enabling scheduled events [%s]...", ",".join(result)) return ("SPECIFIED", result)
class TestNegativeMemoryMetricsThrowError(TestCase): def test_validating_bad_metrics(self): m2ee_stats = {"memory": {"javaheap": -12345}} with self.assertRaises(RuntimeError): PaidAppsMetricsEmitterThread._sanity_check_m2ee_stats(m2ee_stats) def test_no_memorypools_good_metrics(self): m2ee_stats = {"memory": {"javaheap": 12345}} self.assertIsNone( PaidAppsMetricsEmitterThread._sanity_check_m2ee_stats(m2ee_stats) ) def test_non_ints_dont_cause_problems(self): m2ee_stats = { "memory": { "javaheap": 123, "memorypools": {"blah": "stuff"}, "foo": "bar", } } self.assertIsNone( PaidAppsMetricsEmitterThread._sanity_check_m2ee_stats(m2ee_stats) ) def test_non_ints_dont_cause_problems_when_raising(self): m2ee_stats = { "memory": { "javaheap": -123, "memorypools": {"blah": "stuff"}, "foo": "bar", } } with self.assertRaises(RuntimeError): PaidAppsMetricsEmitterThread._sanity_check_m2ee_stats(m2ee_stats) @patch( "buildpack.core.runtime.get_runtime_version", MagicMock(return_value=MXVersion(7.23)), ) def test_underlying_log_message_propagates_upwards(self): m2ee = Mock() m2ee_stats = { "memory": { "javaheap": -123, "memorypools": {"blah": "stuff"}, "foo": "bar", } } interval = 1 metrics_emitter = PaidAppsMetricsEmitterThread(interval, m2ee) with self.assertRaises(RuntimeError): # ensure we log the error, before we raise the exception with self.assertLogs(level="ERROR") as cm: metrics_emitter._sanity_check_m2ee_stats(m2ee_stats) # check the output logs contain the following message self.assertIn("Memory stats with non-logical values", cm.output[-1])
def preflight_check(): logging.debug("pre-flight-check") if not check_database_environment(): raise Exception("Missing environment variables") mx_version_str = runtime.get_version(BUILD_DIR) logging.info("Preflight check on version %s", mx_version_str) mx_version = MXVersion(str(mx_version_str)) if not runtime.check_deprecation(mx_version): raise Exception("Version {} is deprecated.".format(mx_version_str))
def test_guess_future_mendix_versions_doesnt_error(self): m2ee_about = M2EEResponse(action="about", json={ "feedback": {}, "result": 0 }) runtime_version = MXVersion("6.1.0") stats = {} guessed_version = _guess_java_version(m2ee_about, runtime_version, stats) self.assertEqual(8, guessed_version)
def _is_samesite_cookie_workaround_enabled(mx_version): try: return distutils.util.strtobool( os.environ.get( SAMESITE_COOKIE_WORKAROUND_ENV_KEY, str(SAMESITE_COOKIE_WORKAROUND_DEFAULT), )) and mx_version < MXVersion( SAMESITE_COOKIE_WORKAROUND_LESS_MX_VERSION) except (ValueError, AttributeError): logging.warning( "Invalid value for [%s], disabling SameSite cookie workaround", SAMESITE_COOKIE_WORKAROUND_ENV_KEY, ) return False
def get_runtime_version(build_path=BASE_PATH): result = get_metadata_value("RuntimeVersion", build_path) if result == None: logging.debug( "Cannot retrieve runtime version %s from metadata file, falling back to project file" ) mpr = util.get_mpr_file_from_dir(build_path) if not mpr: raise Exception("No model/metadata.json or .mpr found in archive") cursor = sqlite3.connect(mpr).cursor() cursor.execute("SELECT _ProductVersion FROM _MetaData LIMIT 1") record = cursor.fetchone() result = record[0] return MXVersion(result)
def test_guess_java11(self): m2ee_about = M2EEResponse( action="about", json={ "feedback": { "java_version": "11.0.3" }, "result": 0 }, ) runtime_version = MXVersion("8.7.0.1476") stats = {} guessed_version = _guess_java_version(m2ee_about, runtime_version, stats) self.assertEqual(11, guessed_version)
def test_guess_java8(self): m2ee_about = M2EEResponse( action="about", json={ "feedback": { "java_version": "1.8.0_202" }, "result": 0 }, ) runtime_version = MXVersion("7.2.3.7.55882") stats = {} guessed_version = _guess_java_version(m2ee_about, runtime_version, stats) self.assertEqual(8, guessed_version)
def test_guess_mendix_6_with_missing_java_version(self): """All Mendix 6 versions are supposed to use Java 8. This is reachable in CloudV4 for Mendix runtimes <= 6.5.0, as the runtime there does not expose the Java version from the about response. """ m2ee_about = M2EEResponse(action="about", json={ "feedback": {}, "result": 0 }) runtime_version = MXVersion("6.1.0") stats = {} guessed_version = _guess_java_version(m2ee_about, runtime_version, stats) self.assertEqual(8, guessed_version)
class TestCaseAzureBlobStoreDryRun(TestCase): azure_storage_vcap_example = """ { "objectstore": [ { "binding_name": null, "credentials": { "account_name": "sapcp4f4f4f4f4f4f4f4f4f4f", "container_name": "sapcp-osaas-2d2d2d2d-cccc-4444-8888-4ed76dca688e", "container_uri": "https://sapcp4f81hh2hps11tx7iuc7.blob.core.windows.net/sapcp-osaas-2d2d2d2d-cccc-4444-8888-4ed76dca688e", "region": "westeurope", "sas_token": "sig=JC1hALu1%2FOFA%1FzyuCzuZKivlb%1IIYktBYxHKPF2OJz3U%3D\u0026sv=2017-01-17\u0026spr=https\u0026si=77777777-ffff-4444-bbbb-bdeafdd87d00\u0026sr=c" }, "instance_name": "test-azure-nl_Test_XgXgX", "label": "objectstore", "name": "test-azure-nl_Test_XgXgX", "plan": "azure-standard", "provider": null, "syslog_drain_url": null, "tags": [ "blobStore", "objectStore" ], "volume_mounts": [] } ] } """ # noqa @mock.patch( "buildpack.core.runtime.get_runtime_version", mock.MagicMock(return_value=MXVersion(7.13)), ) def test_azure_blob_store(self): vcap = json.loads(self.azure_storage_vcap_example) os.environ["MENDIX_BLOBSTORE_TYPE"] = "azure" config = storage._get_azure_storage_specific_config(vcap) assert (config["com.mendix.storage.azure.Container"] == "sapcp-osaas-2d2d2d2d-cccc-4444-8888-4ed76dca688e" # noqa ) assert ( config["com.mendix.storage.azure.SharedAccessSignature"] == "sig=JC1hALu1%2FOFA%1FzyuCzuZKivlb%1IIYktBYxHKPF2OJz3U%3D\u0026sv=2017-01-17\u0026spr=https\u0026si=77777777-ffff-4444-bbbb-bdeafdd87d00\u0026sr=c" # noqa )
def _inject_storage_stats(self, stats): storage_stats = {} runtime_version = runtime.get_runtime_version() try: storage_stats["get_number_of_files"] = self._get_number_of_files() except Exception as e: logging.warn("Metrics: Failed to retrieve number of files, " + str(e)) raise if runtime_version >= MXVersion("7.4.0"): try: storage_stats["get_size_of_files"] = self._get_size_of_files() except Exception as e: logging.warn("Metrics: Failed to retrieve size of files, " + str(e)) raise stats["storage"] = storage_stats return stats
def get_client_certificates(mx_version): config = {} client_certificates_json = os.getenv("CLIENT_CERTIFICATES", "[]") """ [ { 'pfx': 'base64...', # required 'password': '', 'pin_to': ['Module.WS1', 'Module2.WS2'] # optional }, {...} ] """ client_certificates = json.loads(client_certificates_json) num = 0 files = [] passwords = [] pins = {} for client_certificate in client_certificates: pfx = base64.b64decode(client_certificate["pfx"]) location = os.path.abspath(".local/client_certificate.%d.crt" % num) with open(location, "wb") as f: f.write(pfx) passwords.append(client_certificate["password"]) files.append(location) if "pin_to" in client_certificate: for ws in client_certificate["pin_to"]: pins[ws] = location num += 1 if len(files) > 0: config["ClientCertificates"] = ",".join(files) config["ClientCertificatePasswords"] = ",".join(passwords) if mx_version < MXVersion("7.20"): logging.debug( "Runtime version < 7.20, using WebServiceClientCertificates..." ) config["WebServiceClientCertificates"] = pins else: # Deprecated in 7.20 logging.debug( "Runtime version >= 7.20, using ClientCertificateUsages..." ) config["ClientCertificateUsages"] = pins return config
def test_guess_mendix_5_java_7_with_missing_java_version(self): """For some Mendix 5 versions, Java version is not exposed, so you are supposed to infer the Java information from some exposed memory statistics. This code is deprecated, but I am including unit tests against the theoretical implementation to be on the safe side. This may be overly defensive. """ m2ee_about = M2EEResponse(action="about", json={ "feedback": {}, "result": 0 }) runtime_version = MXVersion("5.21.0") stats = {"memory": {"used_nonheap": 2, "code": 1, "permanent": 1}} guessed_version = _guess_java_version(m2ee_about, runtime_version, stats) self.assertEqual(7, guessed_version)
def preflight_check(): if not check_database_environment(): raise ValueError("Missing environment variables") mx_version_str = runtime.get_version(BUILD_DIR) stack = os.getenv("CF_STACK") logging.info( "Preflight check on Mendix runtime version [%s] and stack [%s]...", mx_version_str, stack, ) mx_version = MXVersion(str(mx_version_str)) if not stack in SUPPORTED_STACKS: raise NotImplementedError("Stack [{}] is not supported".format(stack)) if not runtime.check_deprecation(mx_version): raise NotImplementedError( "Mendix runtime version [{}] is not supported".format( mx_version_str)) logging.info("Preflight check completed")
def test_mx6_not_supported(self): assert not runtime.is_version_supported(MXVersion("6.2"))
def check_deprecation(version): if version >= MXVersion("5.0.0") and version < MXVersion("6.0.0"): return False return True
def test_mx9_maintained(self): assert runtime.is_version_maintained(MXVersion("9.6.9")) assert runtime.is_version_maintained(MXVersion("9.12.1")) assert runtime.is_version_maintained(MXVersion("9.13.0"))
def test_mx9_not_maintained(self): assert not runtime.is_version_maintained(MXVersion("9.7"))
def test_mx8_maintained(self): assert runtime.is_version_maintained(MXVersion("8.18.1"))
def test_mx7_not_maintained(self): assert not runtime.is_version_maintained(MXVersion("7.16"))
def test_mx7_maintained(self): assert runtime.is_version_maintained(MXVersion("7.23.1"))
def test_mx7_supported(self): assert runtime.is_version_supported(MXVersion("7.16"))
def test_mx6_implemented(self): assert runtime.is_version_implemented(MXVersion("6.7.5"))