def stage(buildpack_dir, build_path, cache_path):
logging.debug("Staging required components for Mendix runtime...")
database.stage(buildpack_dir, build_path)
logs.stage(buildpack_dir, build_path)
logging.debug("Staging the Mendix runtime...")
shutil.copy(
os.path.join(buildpack_dir, "etc", "m2ee", "m2ee.yaml"),
os.path.join(build_path, ".local", "m2ee.yaml"),
)
git_repo_found = os.path.isdir("/usr/local/share/mendix-runtimes.git")
if git_repo_found and not os.environ.get("FORCED_MXRUNTIME_URL"):
logging.debug(
"Root FS with built-in Mendix runtimes detected, skipping Mendix runtime download"
)
return
url = os.environ.get("FORCED_MXRUNTIME_URL")
if url is not None:
cache_dir = "/tmp/downloads"
else:
cache_dir = cache_path
url = util.get_blobstore_url("/runtime/mendix-%s.tar.gz" %
str(get_version(build_path)))
logging.debug("Downloading Mendix runtime...")
util.download_and_unpack(url,
os.path.join(build_path, "runtimes"),
cache_dir=cache_dir)
def ensure_and_get_jvm(java_version,
cache_dir,
dot_local_location,
package="jdk"):
jdk = determine_jdk(java_version, package)
rootfs_java_path = "/{}".format(compose_jvm_target_dir(jdk))
if not os.path.isdir(rootfs_java_path):
logging.debug(
"Downloading and installing Java {} if required...".format(
package.upper()))
util.download_and_unpack(
util.get_blobstore_url(_compose_jre_url_path(jdk)),
os.path.join(dot_local_location, compose_jvm_target_dir(jdk)),
cache_dir,
)
logging.debug("Java {} installed".format(package.upper()))
else:
logging.debug("Root FS with Java SDK detected, not installing Java")
return util.get_existing_directory_or_raise(
[
"/" + compose_jvm_target_dir(jdk),
os.path.join(dot_local_location, compose_jvm_target_dir(jdk)),
],
"Java not found",
)
def ensure_and_get_jvm(java_version,
cache_dir,
dot_local_location,
package="jdk"):
logging.debug("Begin download and install java %s" % package)
jdk = determine_jdk(java_version, package)
rootfs_java_path = "/{}".format(compose_jvm_target_dir(jdk))
if not os.path.isdir(rootfs_java_path):
logging.debug("rootfs without java sdk detected")
util.download_and_unpack(
util.get_blobstore_url(_compose_jre_url_path(jdk)),
os.path.join(dot_local_location, compose_jvm_target_dir(jdk)),
cache_dir,
)
else:
logging.debug("rootfs with java sdk detected")
logging.debug("end download and install java %s" % package)
return util.get_existing_directory_or_raise(
[
"/" + compose_jvm_target_dir(jdk),
os.path.join(dot_local_location, compose_jvm_target_dir(jdk)),
],
"Java not found",
)
def stage(buildpack_path, cache_path, local_path, java_version):
logging.debug("Staging Java...")
# Download Java
util.mkdir_p(os.path.join(local_path, "bin"))
jvm_location = ensure_and_get_jvm(java_version,
cache_path,
local_path,
package="jre")
# Create a symlink in .local/bin/java
os.symlink(
# Use .. when JDK is in .local because absolute path
# is different at staging time
os.path.join(jvm_location.replace(local_path, ".."), "bin", "java"),
os.path.join(local_path, "bin", "java"),
)
# Import Mozilla CA certificates
# This is done by a third-party tool (keyutil),
# using the Python certifi certificate bundle
#
# While recent versions of AdoptOpenJDK have these on board,
# we still also have to deal with Oracle JREs / JDKs for now.
# When we retire support for Mendix 6,
# we should reconsider importing these certificates ourselves.
util.download_and_unpack(
util.get_blobstore_url(
"/mx-buildpack/java-keyutil/{}".format(KEYUTIL_JAR)),
None,
cache_path,
)
_update_java_cacert(cache_path, jvm_location)
logging.debug("Staging Java finished")
def _download(build_path, cache_dir):
util.download_and_unpack(
util.get_blobstore_url("{}/{}".format(SIDECAR_URL_ROOT,
SIDECAR_ARCHIVE)),
os.path.join(build_path, NAMESPACE),
cache_dir=cache_dir,
)
def download(install_path, cache_dir):
util.download_and_unpack(
util.get_blobstore_url(
"/mx-buildpack/experimental/{}".format(DD_SIDECAR)),
os.path.join(install_path, "datadog"),
cache_dir=cache_dir,
)
def stage(build_path, cache_path):
util.download_and_unpack(
util.get_blobstore_url(
"/mx-buildpack/nginx_1.19.1_linux_x64_cflinuxfs3_b5af01b0.tgz"),
os.path.join(build_path, "nginx"),
cache_dir=cache_path,
)
def compile(build_path, cache_path):
util.download_and_unpack(
util.get_blobstore_url(
"/mx-buildpack/nginx-1.15.10-linux-x64-cflinuxfs2-6247377a.tgz"
),
build_path,
cache_dir=cache_path,
)
def stage(install_dir, cache_dir, runtime_version):
if is_enabled(runtime_version):
util.download_and_unpack(
util.get_blobstore_url("/mx-buildpack/{}/{}".format(
NAMESPACE, ARTIFACT)),
_get_destination_dir(install_dir),
cache_dir=cache_dir,
unpack=False,
)
def stage(destination_path, cache_path):
if appdynamics_used():
util.download_and_unpack(
util.get_blobstore_url(
"/mx-buildpack/appdynamics/appdynamics-agent-{}.zip".format(
APPDYNAMICS_VERSION)),
destination_path, # DOT_LOCAL_LOCATION,
cache_path, # CACHE_DIR,
)
def stage(install_path, cache_path):
if get_new_relic_license_key():
util.download_and_unpack(
util.get_blobstore_url(
"/mx-buildpack/newrelic/newrelic-java-{}.zip".format(
AGENT_VERSION
)
),
_get_destination_dir(install_path),
cache_path,
)
def compile(buildpack_path, build_path):
if is_enabled():
agent_url = "{url}/e/{environment}/api/v1/deployment/installer/agent/unix/paas/latest?include=java&bitness=64&Api-Token={token}".format(
url=os.environ.get("DT_SAAS_URL"),
environment=os.environ.get("DT_TENANT"),
token=os.environ.get("DT_PAAS_TOKEN"),
)
util.download_and_unpack(
agent_url,
buildpack_path, # DOT_LOCAL_LOCATION,
build_path, # CACHE_DIR,
)
def stage(buildpack_path, build_path, cache_path):
logging.debug("Staging nginx...")
shutil.copytree(
os.path.join(buildpack_path, "etc/nginx"),
os.path.join(build_path, "nginx"),
)
util.download_and_unpack(
util.get_blobstore_url(
"/mx-buildpack/nginx_1.19.1_linux_x64_cflinuxfs3_b5af01b0.tgz"),
os.path.join(build_path, "nginx"),
cache_dir=cache_path,
)
def _download_pkgs(install_path, cache_dir):
# Download producer streams artifact
PDR_STREAMS_DOWNLOAD_URL = "{}{}-{}.{}".format(
BASE_URL,
PDR_STREAMS_FILENAME,
PDR_STREAMS_VERSION,
TAR_EXT,
)
util.download_and_unpack(
util.get_blobstore_url(PDR_STREAMS_DOWNLOAD_URL),
os.path.join(install_path, PDR_STREAMS_DIR),
cache_dir=cache_dir,
)
def stage(install_path, cache_dir):
if not is_enabled():
return
#
# Add Telegraf to the container which can forward metrics to a custom
# AppMetrics target
util.download_and_unpack(
util.get_blobstore_url(
"/mx-buildpack/telegraf/telegraf-1.7.1_linux_amd64.tar.gz"
),
install_path,
cache_dir=cache_dir,
)
def _download(build_path, cache_dir):
util.download_and_unpack(
util.get_blobstore_url("{}/{}".format(SIDECAR_URL_ROOT,
SIDECAR_ARTIFACT_NAME)),
os.path.join(build_path, NAMESPACE),
cache_dir=cache_dir,
alias="cf-datadog-sidecar", # Removes the old sidecar if present
)
util.download_and_unpack(
util.get_blobstore_url("{}/{}".format(JAVA_AGENT_URL_ROOT,
JAVA_AGENT_ARTIFACT_NAME)),
os.path.join(build_path, NAMESPACE),
cache_dir=cache_dir,
unpack=False,
)
def stage(buildpack_path, build_path):
if is_enabled():
agent_url = "{url}/e/{environment}/api/v1/deployment/installer/agent/unix/paas/latest?include=java&bitness=64&Api-Token={token}".format(
url=os.environ.get("DT_SAAS_URL"),
environment=os.environ.get("DT_TENANT"),
token=os.environ.get("DT_PAAS_TOKEN"),
)
try:
util.download_and_unpack(
agent_url,
buildpack_path, # DOT_LOCAL_LOCATION,
cache_dir=build_path, # CACHE_DIR,
unpack=True,
)
except Exception as e:
logging.warning("Dynatrace agent download and unpack failed",
exc_info=True)
def ensure_and_get_mono(mx_version, cache_dir):
logging.debug("Ensuring mono for Mendix %s", mx_version)
mono_version = _detect_mono_version(mx_version)
fallback_location = "/tmp/opt"
if (
mono_version == "mono-3.10.0"
and platform.linux_distribution()[2].lower() == "bionic"
):
util.download_and_unpack(
util.get_blobstore_url(_compose_mono_url_path(mono_version)),
os.path.join(fallback_location, "store"),
cache_dir,
)
mono_subpath = glob.glob("/tmp/opt/store/*-mono-env-3.10.0")
mono_location = "/tmp/opt/mono-3.10.0"
os.symlink(mono_subpath[0], mono_location)
logging.debug(
"Using {mono_location}".format(mono_location=mono_location)
)
logging.warning(
"The staging phase is likely going to fail when the default "
+ "settings are used. As a workaround, more disk space needs to be "
+ "allocated for the cache. Consult "
+ "https://docs.cloudfoundry.org/devguide/deploy-apps/large-app-deploy.html "
+ "for more information."
)
return mono_location
else:
try:
mono_location = _get_mono_path("/tmp/opt", mono_version)
except NotFoundException:
logging.debug("Mono not found in default locations")
util.download_and_unpack(
util.get_blobstore_url(_compose_mono_url_path(mono_version)),
os.path.join(fallback_location, mono_version),
cache_dir,
)
mono_location = _get_mono_path(fallback_location, mono_version)
logging.debug(
"Using {mono_location}".format(mono_location=mono_location)
)
return mono_location
def _download_pkgs(install_path, cache_dir):
# Download kafka connect and debezium
KAFKA_CONNECT_DOWNLOAD_URL = "{}{}-{}.{}".format(
BASE_URL, KAFKA_CONNECT_FILENAME, KAFKA_CONNECT_VERSION, PKG_FILE_EXT,
)
util.download_and_unpack(
util.get_blobstore_url(KAFKA_CONNECT_DOWNLOAD_URL),
os.path.join(install_path, BASE_DIR, KAFKA_CONNECT_DIR),
cache_dir=cache_dir,
)
DBZ_DOWNLOAD_URL = "{}{}-{}.{}".format(
BASE_URL, DBZ_FILENAME, DBZ_VERSION, PKG_FILE_EXT
)
util.download_and_unpack(
util.get_blobstore_url(DBZ_DOWNLOAD_URL),
os.path.join(install_path, BASE_DIR, DBZ_DIR),
cache_dir=cache_dir,
)
def stage(buildpack_path, build_path, cache_path):
logging.debug("Staging nginx...")
shutil.copytree(
os.path.join(buildpack_path, "etc/nginx"),
os.path.join(build_path, "nginx"),
)
if not is_custom_nginx():
logging.debug("Downloading nginx...")
util.download_and_unpack(
util.get_blobstore_url(
"/mx-buildpack/nginx_1.19.1_linux_x64_cflinuxfs3_b5af01b0.tgz"
),
os.path.join(build_path, "nginx"),
cache_dir=cache_path,
)
else:
logging.debug(
"Custom nginx path provided, nginx will not be downloaded")
def ensure_mxbuild_in_directory(directory, mx_version, cache_dir):
if os.path.isdir(os.path.join(directory, "modeler")):
return
util.mkdir_p(directory)
url = os.environ.get("FORCED_MXBUILD_URL")
if url:
# don"t ever cache with a FORCED_MXBUILD_URL
util.download_and_unpack(url, directory, cache_dir="/tmp/downloads")
else:
try:
_checkout_from_git_rootfs(directory, mx_version)
except NotFoundException as e:
logging.debug(str(e))
util.download_and_unpack(
util.get_blobstore_url("/runtime/mxbuild-%s.tar.gz" %
str(mx_version)),
directory,
cache_dir=cache_dir,
)
def compile(build_path, cache_path):
logging.debug("downloading mendix version")
git_repo_found = os.path.isdir("/usr/local/share/mendix-runtimes.git")
if git_repo_found and not os.environ.get("FORCED_MXRUNTIME_URL"):
logging.debug("rootfs with mendix runtime detected, skipping download")
return
url = os.environ.get("FORCED_MXRUNTIME_URL")
if url is not None:
cache_dir = "/tmp/downloads"
else:
cache_dir = cache_path
url = util.get_blobstore_url("/runtime/mendix-%s.tar.gz" %
str(get_version(build_path)))
logging.debug("rootfs without mendix runtimes detected, "
"downloading and unpacking mendix runtime now")
util.download_and_unpack(url,
os.path.join(build_path, "runtimes"),
cache_dir=cache_dir)
def stage(buildpack_dir, build_path, cache_path):
logging.debug("Staging required components for Mendix runtime...")
database.stage(buildpack_dir, build_path)
logs.stage(buildpack_dir, build_path)
logging.debug("Staging the Mendix runtime...")
shutil.copy(
os.path.join(buildpack_dir, "etc", "m2ee", "m2ee.yaml"),
os.path.join(build_path, ".local", "m2ee.yaml"),
)
git_repo_found = os.path.isdir("/usr/local/share/mendix-runtimes.git")
if git_repo_found and not os.environ.get("FORCED_MXRUNTIME_URL"):
logging.debug(
"Root FS with built-in Mendix runtimes detected, skipping Mendix runtime download"
)
return
forced_url = os.environ.get("FORCED_MXRUNTIME_URL")
url = None
if forced_url is not None:
cache_dir = "/tmp/downloads"
if not forced_url.endswith(".tar.gz"):
# Assume that the forced URL points to a blobstore root
url = _get_runtime_url(forced_url, build_path)
else:
url = forced_url
else:
cache_dir = cache_path
url = _get_runtime_url(util.get_blobstore(), build_path)
logging.debug("Downloading Mendix runtime...")
util.download_and_unpack(
url, os.path.join(build_path, "runtimes"), cache_dir=cache_dir
)
def stage(buildpack_path, build_path, cache_dir):
if not is_enabled():
return
logging.debug("Staging the Telegraf metrics agent...")
util.download_and_unpack(
util.get_blobstore_url(
"/mx-buildpack/telegraf/telegraf-{}_linux_amd64.tar.gz".format(
VERSION)),
os.path.join(build_path, NAMESPACE),
cache_dir=cache_dir,
)
# Copy the configuration template
shutil.copy(
os.path.join(buildpack_path, "etc", "telegraf", TEMPLATE_FILENAME),
os.path.join(
build_path,
NAMESPACE,
"telegraf-{}".format(VERSION),
"etc",
"telegraf",
),
)
def set_up_m2ee_client(vcap_data):
client = m2ee_class(
yamlfiles=[".local/m2ee.yaml"],
load_default_files=False,
config={
"m2ee": {
# this is named admin_pass, but it's the verification http header
# to communicate with the internal management port of the runtime
"admin_pass": security.get_m2ee_password()
}
},
)
version = client.config.get_runtime_version()
mendix_runtimes_path = "/usr/local/share/mendix-runtimes.git"
mendix_runtime_version_path = os.path.join(os.getcwd(), "runtimes",
str(version))
if os.path.isdir(mendix_runtimes_path
) and not os.path.isdir(mendix_runtime_version_path):
util.mkdir_p(mendix_runtime_version_path)
env = dict(os.environ)
env["GIT_WORK_TREE"] = mendix_runtime_version_path
# checkout the runtime version
process = subprocess.Popen(
["git", "checkout", str(version), "-f"],
cwd=mendix_runtimes_path,
env=env,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
process.communicate()
if process.returncode != 0:
logging.info("Mendix %s is not available in the rootfs", version)
logging.info("Fallback (1): trying to fetch Mendix %s using git",
version)
process = subprocess.Popen(
[
"git",
"fetch",
"origin",
"refs/tags/{0}:refs/tags/{0}".format(str(version)),
"&&",
"git",
"checkout",
str(version),
"-f",
],
cwd=mendix_runtimes_path,
env=env,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
process.communicate()
if process.returncode != 0:
logging.info(
"Unable to fetch Mendix {} using git".format(version))
url = util.get_blobstore_url("/runtime/mendix-%s.tar.gz" %
str(version))
logging.info(
"Fallback (2): downloading Mendix {} from {}".format(
version, url))
util.download_and_unpack(url,
os.path.join(os.getcwd(), "runtimes"))
client.reload_config()
runtime.set_runtime_config(
client.config._model_metadata,
client.config._conf["mxruntime"],
vcap_data,
client,
)
java_version = runtime.get_java_version(
client.config.get_runtime_version())["version"]
java.update_config(client.config._conf["m2ee"], vcap_data, java_version)
runtime.set_jetty_config(client)
newrelic.update_config(client, vcap_data["application_name"])
appdynamics.update_config(client, vcap_data["application_name"])
runtime.set_application_name(client, vcap_data["application_name"])
telegraf.update_config(client, vcap_data["application_name"])
datadog.update_config(client)
return client
