def _download(build_path, cache_dir): util.download_and_unpack( util.get_blobstore_url("{}/{}".format(SIDECAR_URL_ROOT, SIDECAR_ARCHIVE)), os.path.join(build_path, NAMESPACE), cache_dir=cache_dir, ) util.download_and_unpack( util.get_blobstore_url("{}/{}".format(JAVA_AGENT_URL_ROOT, JAVA_AGENT_JAR)), os.path.join(build_path, NAMESPACE), cache_dir=cache_dir, unpack=False, )
def _download(build_path, cache_dir): util.download_and_unpack( util.get_blobstore_url("{}/{}".format(SIDECAR_URL_ROOT, SIDECAR_ARTIFACT_NAME)), os.path.join(build_path, NAMESPACE), cache_dir=cache_dir, alias="cf-datadog-sidecar", # Removes the old sidecar if present ) util.download_and_unpack( util.get_blobstore_url("{}/{}".format(JAVA_AGENT_URL_ROOT, JAVA_AGENT_ARTIFACT_NAME)), os.path.join(build_path, NAMESPACE), cache_dir=cache_dir, unpack=False, )
def ensure_and_get_jvm(java_version, cache_dir, dot_local_location, package="jdk"): jdk = determine_jdk(java_version, package) rootfs_java_path = "/{}".format(compose_jvm_target_dir(jdk)) if not os.path.isdir(rootfs_java_path): logging.debug( "Downloading and installing Java {} if required...".format( package.upper())) util.download_and_unpack( util.get_blobstore_url(_compose_jre_url_path(jdk)), os.path.join(dot_local_location, compose_jvm_target_dir(jdk)), cache_dir, ) logging.debug("Java {} installed".format(package.upper())) else: logging.debug("Root FS with Java SDK detected, not installing Java") return util.get_existing_directory_or_raise( [ "/" + compose_jvm_target_dir(jdk), os.path.join(dot_local_location, compose_jvm_target_dir(jdk)), ], "Java not found", )
def stage(buildpack_path, cache_path, local_path, java_version): logging.debug("Staging Java...") # Download Java util.mkdir_p(os.path.join(local_path, "bin")) jvm_location = ensure_and_get_jvm(java_version, cache_path, local_path, package="jre") # Create a symlink in .local/bin/java os.symlink( # Use .. when JDK is in .local because absolute path # is different at staging time os.path.join(jvm_location.replace(local_path, ".."), "bin", "java"), os.path.join(local_path, "bin", "java"), ) # Import Mozilla CA certificates # This is done by a third-party tool (keyutil), # using the Python certifi certificate bundle # # While recent versions of AdoptOpenJDK have these on board, # we still also have to deal with Oracle JREs / JDKs for now. # When we retire support for Mendix 6, # we should reconsider importing these certificates ourselves. util.download_and_unpack( util.get_blobstore_url( "/mx-buildpack/java-keyutil/{}".format(KEYUTIL_JAR)), None, cache_path, ) _update_java_cacert(cache_path, jvm_location) logging.debug("Staging Java finished")
def stage(build_path, cache_path): util.download_and_unpack( util.get_blobstore_url( "/mx-buildpack/nginx_1.19.1_linux_x64_cflinuxfs3_b5af01b0.tgz"), os.path.join(build_path, "nginx"), cache_dir=cache_path, )
def stage(buildpack_dir, build_path, cache_path): logging.debug("Staging required components for Mendix runtime...") database.stage(buildpack_dir, build_path) logs.stage(buildpack_dir, build_path) logging.debug("Staging the Mendix runtime...") shutil.copy( os.path.join(buildpack_dir, "etc", "m2ee", "m2ee.yaml"), os.path.join(build_path, ".local", "m2ee.yaml"), ) git_repo_found = os.path.isdir("/usr/local/share/mendix-runtimes.git") if git_repo_found and not os.environ.get("FORCED_MXRUNTIME_URL"): logging.debug( "Root FS with built-in Mendix runtimes detected, skipping Mendix runtime download" ) return url = os.environ.get("FORCED_MXRUNTIME_URL") if url is not None: cache_dir = "/tmp/downloads" else: cache_dir = cache_path url = util.get_blobstore_url("/runtime/mendix-%s.tar.gz" % str(get_version(build_path))) logging.debug("Downloading Mendix runtime...") util.download_and_unpack(url, os.path.join(build_path, "runtimes"), cache_dir=cache_dir)
def download(install_path, cache_dir): util.download_and_unpack( util.get_blobstore_url( "/mx-buildpack/experimental/{}".format(DD_SIDECAR)), os.path.join(install_path, "datadog"), cache_dir=cache_dir, )
def ensure_and_get_jvm(java_version, cache_dir, dot_local_location, package="jdk"): logging.debug("Begin download and install java %s" % package) jdk = determine_jdk(java_version, package) rootfs_java_path = "/{}".format(compose_jvm_target_dir(jdk)) if not os.path.isdir(rootfs_java_path): logging.debug("rootfs without java sdk detected") util.download_and_unpack( util.get_blobstore_url(_compose_jre_url_path(jdk)), os.path.join(dot_local_location, compose_jvm_target_dir(jdk)), cache_dir, ) else: logging.debug("rootfs with java sdk detected") logging.debug("end download and install java %s" % package) return util.get_existing_directory_or_raise( [ "/" + compose_jvm_target_dir(jdk), os.path.join(dot_local_location, compose_jvm_target_dir(jdk)), ], "Java not found", )
def compile(build_path, cache_path): util.download_and_unpack( util.get_blobstore_url( "/mx-buildpack/nginx-1.15.10-linux-x64-cflinuxfs2-6247377a.tgz" ), build_path, cache_dir=cache_path, )
def stage(destination_path, cache_path): if appdynamics_used(): util.download_and_unpack( util.get_blobstore_url( "/mx-buildpack/appdynamics/appdynamics-agent-{}.zip".format( APPDYNAMICS_VERSION)), destination_path, # DOT_LOCAL_LOCATION, cache_path, # CACHE_DIR, )
def stage(install_dir, cache_dir, runtime_version): if is_enabled(runtime_version): util.download_and_unpack( util.get_blobstore_url("/mx-buildpack/{}/{}".format( NAMESPACE, ARTIFACT)), _get_destination_dir(install_dir), cache_dir=cache_dir, unpack=False, )
def _download_pkgs(install_path, cache_dir): # Download kafka connect and debezium KAFKA_CONNECT_DOWNLOAD_URL = "{}{}-{}.{}".format( BASE_URL, KAFKA_CONNECT_FILENAME, KAFKA_CONNECT_VERSION, PKG_FILE_EXT, ) util.download_and_unpack( util.get_blobstore_url(KAFKA_CONNECT_DOWNLOAD_URL), os.path.join(install_path, BASE_DIR, KAFKA_CONNECT_DIR), cache_dir=cache_dir, ) DBZ_DOWNLOAD_URL = "{}{}-{}.{}".format( BASE_URL, DBZ_FILENAME, DBZ_VERSION, PKG_FILE_EXT ) util.download_and_unpack( util.get_blobstore_url(DBZ_DOWNLOAD_URL), os.path.join(install_path, BASE_DIR, DBZ_DIR), cache_dir=cache_dir, )
def ensure_and_get_mono(mx_version, cache_dir): logging.debug("Ensuring mono for Mendix %s", mx_version) mono_version = _detect_mono_version(mx_version) fallback_location = "/tmp/opt" if ( mono_version == "mono-3.10.0" and platform.linux_distribution()[2].lower() == "bionic" ): util.download_and_unpack( util.get_blobstore_url(_compose_mono_url_path(mono_version)), os.path.join(fallback_location, "store"), cache_dir, ) mono_subpath = glob.glob("/tmp/opt/store/*-mono-env-3.10.0") mono_location = "/tmp/opt/mono-3.10.0" os.symlink(mono_subpath[0], mono_location) logging.debug( "Using {mono_location}".format(mono_location=mono_location) ) logging.warning( "The staging phase is likely going to fail when the default " + "settings are used. As a workaround, more disk space needs to be " + "allocated for the cache. Consult " + "https://docs.cloudfoundry.org/devguide/deploy-apps/large-app-deploy.html " + "for more information." ) return mono_location else: try: mono_location = _get_mono_path("/tmp/opt", mono_version) except NotFoundException: logging.debug("Mono not found in default locations") util.download_and_unpack( util.get_blobstore_url(_compose_mono_url_path(mono_version)), os.path.join(fallback_location, mono_version), cache_dir, ) mono_location = _get_mono_path(fallback_location, mono_version) logging.debug( "Using {mono_location}".format(mono_location=mono_location) ) return mono_location
def stage(install_path, cache_path): if get_new_relic_license_key(): util.download_and_unpack( util.get_blobstore_url( "/mx-buildpack/newrelic/newrelic-java-{}.zip".format( AGENT_VERSION ) ), _get_destination_dir(install_path), cache_path, )
def _download_pkgs(install_path, cache_dir): # Download producer streams artifact PDR_STREAMS_DOWNLOAD_URL = "{}{}-{}.{}".format( BASE_URL, PDR_STREAMS_FILENAME, PDR_STREAMS_VERSION, TAR_EXT, ) util.download_and_unpack( util.get_blobstore_url(PDR_STREAMS_DOWNLOAD_URL), os.path.join(install_path, PDR_STREAMS_DIR), cache_dir=cache_dir, )
def stage(install_path, cache_dir): if not is_enabled(): return # # Add Telegraf to the container which can forward metrics to a custom # AppMetrics target util.download_and_unpack( util.get_blobstore_url( "/mx-buildpack/telegraf/telegraf-1.7.1_linux_amd64.tar.gz" ), install_path, cache_dir=cache_dir, )
def stage(buildpack_path, build_path, cache_path): logging.debug("Staging nginx...") shutil.copytree( os.path.join(buildpack_path, "etc/nginx"), os.path.join(build_path, "nginx"), ) util.download_and_unpack( util.get_blobstore_url( "/mx-buildpack/nginx_1.19.1_linux_x64_cflinuxfs3_b5af01b0.tgz"), os.path.join(build_path, "nginx"), cache_dir=cache_path, )
def stage(buildpack_path, build_path, cache_path): logging.debug("Staging nginx...") shutil.copytree( os.path.join(buildpack_path, "etc/nginx"), os.path.join(build_path, "nginx"), ) if not is_custom_nginx(): logging.debug("Downloading nginx...") util.download_and_unpack( util.get_blobstore_url( "/mx-buildpack/nginx_1.19.1_linux_x64_cflinuxfs3_b5af01b0.tgz" ), os.path.join(build_path, "nginx"), cache_dir=cache_path, ) else: logging.debug( "Custom nginx path provided, nginx will not be downloaded")
def ensure_mxbuild_in_directory(directory, mx_version, cache_dir): if os.path.isdir(os.path.join(directory, "modeler")): return util.mkdir_p(directory) url = os.environ.get("FORCED_MXBUILD_URL") if url: # don"t ever cache with a FORCED_MXBUILD_URL util.download_and_unpack(url, directory, cache_dir="/tmp/downloads") else: try: _checkout_from_git_rootfs(directory, mx_version) except NotFoundException as e: logging.debug(str(e)) util.download_and_unpack( util.get_blobstore_url("/runtime/mxbuild-%s.tar.gz" % str(mx_version)), directory, cache_dir=cache_dir, )
def compile(build_path, cache_path): logging.debug("downloading mendix version") git_repo_found = os.path.isdir("/usr/local/share/mendix-runtimes.git") if git_repo_found and not os.environ.get("FORCED_MXRUNTIME_URL"): logging.debug("rootfs with mendix runtime detected, skipping download") return url = os.environ.get("FORCED_MXRUNTIME_URL") if url is not None: cache_dir = "/tmp/downloads" else: cache_dir = cache_path url = util.get_blobstore_url("/runtime/mendix-%s.tar.gz" % str(get_version(build_path))) logging.debug("rootfs without mendix runtimes detected, " "downloading and unpacking mendix runtime now") util.download_and_unpack(url, os.path.join(build_path, "runtimes"), cache_dir=cache_dir)
def stage(buildpack_path, build_path, cache_dir): if not is_enabled(): return logging.debug("Staging the Telegraf metrics agent...") util.download_and_unpack( util.get_blobstore_url( "/mx-buildpack/telegraf/telegraf-{}_linux_amd64.tar.gz".format( VERSION)), os.path.join(build_path, NAMESPACE), cache_dir=cache_dir, ) # Copy the configuration template shutil.copy( os.path.join(buildpack_path, "etc", "telegraf", TEMPLATE_FILENAME), os.path.join( build_path, NAMESPACE, "telegraf-{}".format(VERSION), "etc", "telegraf", ), )
def set_up_m2ee_client(vcap_data): client = m2ee_class( yamlfiles=[".local/m2ee.yaml"], load_default_files=False, config={ "m2ee": { # this is named admin_pass, but it's the verification http header # to communicate with the internal management port of the runtime "admin_pass": security.get_m2ee_password() } }, ) version = client.config.get_runtime_version() mendix_runtimes_path = "/usr/local/share/mendix-runtimes.git" mendix_runtime_version_path = os.path.join(os.getcwd(), "runtimes", str(version)) if os.path.isdir(mendix_runtimes_path ) and not os.path.isdir(mendix_runtime_version_path): util.mkdir_p(mendix_runtime_version_path) env = dict(os.environ) env["GIT_WORK_TREE"] = mendix_runtime_version_path # checkout the runtime version process = subprocess.Popen( ["git", "checkout", str(version), "-f"], cwd=mendix_runtimes_path, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) process.communicate() if process.returncode != 0: logging.info("Mendix %s is not available in the rootfs", version) logging.info("Fallback (1): trying to fetch Mendix %s using git", version) process = subprocess.Popen( [ "git", "fetch", "origin", "refs/tags/{0}:refs/tags/{0}".format(str(version)), "&&", "git", "checkout", str(version), "-f", ], cwd=mendix_runtimes_path, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) process.communicate() if process.returncode != 0: logging.info( "Unable to fetch Mendix {} using git".format(version)) url = util.get_blobstore_url("/runtime/mendix-%s.tar.gz" % str(version)) logging.info( "Fallback (2): downloading Mendix {} from {}".format( version, url)) util.download_and_unpack(url, os.path.join(os.getcwd(), "runtimes")) client.reload_config() runtime.set_runtime_config( client.config._model_metadata, client.config._conf["mxruntime"], vcap_data, client, ) java_version = runtime.get_java_version( client.config.get_runtime_version())["version"] java.update_config(client.config._conf["m2ee"], vcap_data, java_version) runtime.set_jetty_config(client) newrelic.update_config(client, vcap_data["application_name"]) appdynamics.update_config(client, vcap_data["application_name"]) runtime.set_application_name(client, vcap_data["application_name"]) telegraf.update_config(client, vcap_data["application_name"]) datadog.update_config(client) return client
def _get_runtime_url(blobstore, build_path): return util.get_blobstore_url( "/runtime/mendix-{}.tar.gz".format(str(get_version(build_path))), blobstore=blobstore, )