def build():
mpr = os.path.abspath(buildpackutil.get_mpr_file_from_dir(PROJECT_DIR))
response = requests.post(
'http://localhost:6666/build',
data=json.dumps({
'target': 'Deploy',
'projectFilePath': mpr,
'forceFullDeployment': False
}),
headers={'Content-Type': 'application/json'},
timeout=120,
)
if response.status_code != requests.codes.ok:
raise MxBuildFailure("MxBuild failure", response.status_code, response.json())
result = response.json()
if result['status'] == 'Success':
try:
sync_project_files()
logger.info("Syncing project files ...")
except:
logger.warning("Syncing project files failed: %s", traceback.format_exc())
raise
else:
logger.warning("Not syncing project files. MxBuild result: %s", result)
return result
def activate_appdynamics(m2ee, app_name):
if not buildpackutil.appdynamics_used():
return
logger.info("Adding app dynamics")
m2ee.config._conf["m2ee"]["javaopts"].append(
"-javaagent:{path}".format(
path=os.path.abspath(".local/ver4.3.5.7/javaagent.jar")
)
)
m2ee.config._conf["m2ee"]["javaopts"].append(
"-Dappagent.install.dir={path}".format(
path=os.path.abspath(".local/ver4.3.5.7")
)
)
APPDYNAMICS_AGENT_NODE_NAME = "APPDYNAMICS_AGENT_NODE_NAME"
if os.getenv(APPDYNAMICS_AGENT_NODE_NAME):
m2ee.config._conf["m2ee"]["custom_environment"][
APPDYNAMICS_AGENT_NODE_NAME
] = (
"%s-%s"
% (
os.getenv(APPDYNAMICS_AGENT_NODE_NAME),
os.getenv("CF_INSTANCE_INDEX", "0"),
)
)
def create_admin_user(m2ee):
logger.info("Ensuring admin user credentials")
app_admin_password = get_admin_password()
if os.getenv("M2EE_PASSWORD"):
logger.debug(
"M2EE_PASSWORD is set so skipping creation of application admin password"
)
return
if not app_admin_password:
logger.warning(
"ADMIN_PASSWORD not set, so skipping creation of application admin password"
)
return
logger.debug("Creating admin user")
m2eeresponse = m2ee.client.create_admin_user(
{"password": app_admin_password}
)
if m2eeresponse.has_error():
m2eeresponse.display_error()
if not is_development_mode():
sys.exit(1)
logger.debug("Setting admin user password")
m2eeresponse = m2ee.client.create_admin_user(
{
"username": m2ee.config._model_metadata["AdminUser"],
"password": app_admin_password,
}
)
if m2eeresponse.has_error():
m2eeresponse.display_error()
if not is_development_mode():
sys.exit(1)
def do_enable_debugger(self, args):
if self._report_not_implemented(4.3) or self._report_not_running():
return
if not args:
debugger_password = raw_input(
"Please enter the password to be used for remote debugger "
"access from the modeler, or leave blank to auto-generate "
"a password: "******"The remote debugger is now enabled, the password to "
"use is %s" % debugger_password)
logger.info("You can use the remote debugger option in the Mendix "
"Business Modeler to connect to the /debugger/ sub "
"url on your application (e.g. "
"https://app.example.com/debugger/). ")
def create_admin_user(m2ee):
logger.info('Ensuring admin user credentials')
app_admin_password = get_admin_password()
if os.getenv('M2EE_PASSWORD'):
logger.debug(
'M2EE_PASSWORD is set so skipping creation of application admin password'
)
return
if not app_admin_password:
logger.warning(
'ADMIN_PASSWORD not set, so skipping creation of application admin password'
)
return
logger.debug('Creating admin user')
m2eeresponse = m2ee.client.create_admin_user(
{'password': app_admin_password})
if m2eeresponse.has_error():
m2eeresponse.display_error()
if not is_development_mode():
sys.exit(1)
logger.debug('Setting admin user password')
m2eeresponse = m2ee.client.create_admin_user({
'username':
m2ee.config._model_metadata['AdminUser'],
'password':
app_admin_password
})
if m2eeresponse.has_error():
m2eeresponse.display_error()
if not is_development_mode():
sys.exit(1)
def set_runtime_config(metadata, mxruntime_config, vcap_data, m2ee):
scheduled_event_execution, my_scheduled_events = get_scheduled_events(
metadata)
app_config = {
"ApplicationRootUrl": "https://%s" % vcap_data["application_uris"][0],
"MicroflowConstants": get_constants(metadata),
"ScheduledEventExecution": scheduled_event_execution,
}
if my_scheduled_events is not None:
app_config["MyScheduledEvents"] = my_scheduled_events
if is_development_mode():
logger.warning("Runtime is being started in Development Mode. Set "
'DEVELOPMENT_MODE to "false" (currently "true") to '
"set it to production.")
app_config["DTAPMode"] = "D"
if m2ee.config.get_runtime_version() >= 7 and not i_am_primary_instance():
app_config["com.mendix.core.isClusterSlave"] = "true"
elif (m2ee.config.get_runtime_version() >= 5.15
and os.getenv("ENABLE_STICKY_SESSIONS", "false").lower() == "true"):
logger.info("Enabling sticky sessions")
app_config["com.mendix.core.SessionIdCookieName"] = "JSESSIONID"
buildpackutil.mkdir_p(os.path.join(os.getcwd(), "model", "resources"))
mxruntime_config.update(app_config)
mxruntime_config.update(
buildpackutil.get_database_config(
development_mode=is_development_mode()))
mxruntime_config.update(get_filestore_config(m2ee))
mxruntime_config.update(get_certificate_authorities())
mxruntime_config.update(get_client_certificates())
mxruntime_config.update(get_custom_settings(metadata, mxruntime_config))
mxruntime_config.update(get_custom_runtime_settings())
def do_download_runtime(self, args):
if args:
try:
mxversion = m2ee.version.MXVersion(args)
except Exception:
logger.error("The provided runtime version string is not a "
"valid Mendix Runtime version number. Try using "
"the format x.y.z, e.g. 4.7.1.")
return
else:
mxversion = self.m2ee.config.get_runtime_version()
if not mxversion:
logger.info("You did not specify a Mendix Runtime version to "
"download, and no current unpacked application "
"model is available to determine the version from. "
"Specify a version number or use unpack first.")
return
version = str(mxversion)
if self.m2ee.config.lookup_in_mxjar_repo(version):
logger.info("The Mendix Runtime for version %s is already "
"installed. If you want to download another Runtime "
"version, specify the version number as argument to "
"download_runtime." % version)
return
if not self.m2ee.config.get_first_writable_mxjar_repo():
logger.error("None of the locations specified in the mxjar_repo "
"configuration option are writable by the current "
"user account.")
return
self.m2ee.download_and_unpack_runtime(version)
def run(self):
logger.debug(
"Starting metrics emitter with interval %d", self.interval
)
for line in itertools.cycle(HEARTBEAT_STRING_LIST):
logger.info("MENDIX-LOGGING-HEARTBEAT: %s", line)
time.sleep(self.interval)
def set_runtime_config(metadata, mxruntime_config, vcap_data, m2ee):
scheduled_event_execution, my_scheduled_events = (
get_scheduled_events(metadata))
app_config = {
'ApplicationRootUrl': 'https://%s' % vcap_data['application_uris'][0],
'MicroflowConstants': get_constants(metadata),
'ScheduledEventExecution': scheduled_event_execution,
}
if my_scheduled_events is not None:
app_config['MyScheduledEvents'] = my_scheduled_events
if is_development_mode():
logger.warning('Runtime is being started in Development Mode. Set '
'DEVELOPMENT_MODE to "false" (currently "true") to '
'set it to production.')
app_config['DTAPMode'] = 'D'
if (m2ee.config.get_runtime_version() >= 7
and not i_am_primary_instance()):
app_config['com.mendix.core.isClusterSlave'] = 'true'
elif (m2ee.config.get_runtime_version() >= 5.15
and os.getenv('ENABLE_STICKY_SESSIONS', 'false').lower() == 'true'):
logger.info('Enabling sticky sessions')
app_config['com.mendix.core.SessionIdCookieName'] = 'JSESSIONID'
mxruntime_config.update(app_config)
mxruntime_config.update(
buildpackutil.get_database_config(
development_mode=is_development_mode(), ))
mxruntime_config.update(get_filestore_config(m2ee))
mxruntime_config.update(get_certificate_authorities())
mxruntime_config.update(get_client_certificates())
mxruntime_config.update(get_custom_settings(metadata, mxruntime_config))
mxruntime_config.update(get_custom_runtime_settings())
def do_download_runtime(self, args):
if args:
try:
mxversion = m2ee.version.MXVersion(args)
except Exception:
logger.error("The provided runtime version string is not a "
"valid Mendix Runtime version number. Try using "
"the format x.y.z, e.g. 4.7.1.")
return
else:
mxversion = self.m2ee.config.get_runtime_version()
if not mxversion:
logger.info("You did not specify a Mendix Runtime version to "
"download, and no current unpacked application "
"model is available to determine the version from. "
"Specify a version number or use unpack first.")
return
version = str(mxversion)
if self.m2ee.config.lookup_in_mxjar_repo(version):
logger.info("The Mendix Runtime for version %s is already "
"installed. If you want to download another Runtime "
"version, specify the version number as argument to "
"download_runtime." % version)
return
if not self.m2ee.config.get_first_writable_mxjar_repo():
logger.error("None of the locations specified in the mxjar_repo "
"configuration option are writable by the current "
"user account.")
return
self.m2ee.download_and_unpack_runtime(version)
def do_enable_debugger(self, args):
if self._report_not_implemented(4.3) or self._report_not_running():
return
if not args:
debugger_password = raw_input(
"Please enter the password to be used for remote debugger "
"access from the modeler, or leave blank to auto-generate "
"a password: "******"The remote debugger is now enabled, the password to "
"use is %s" % debugger_password)
logger.info("You can use the remote debugger option in the Mendix "
"Business Modeler to connect to the /debugger/ sub "
"url on your application (e.g. "
"https://app.example.com/debugger/). ")
def display_java_version():
java_version = (subprocess.check_output(
[".local/bin/java", "-version"],
stderr=subprocess.STDOUT).decode("utf8").strip().split("\n"))
logger.info("Using Java version:")
for line in java_version:
logger.info(line)
def _set_log_level(self, subscriber, node, level):
level = level.upper()
response = self.m2ee.set_log_level(subscriber, node, level)
if response.has_error():
response.display_error()
print("Remember, all parameters are case sensitive")
else:
logger.info("Loglevel for %s set to %s" % (node, level))
def loop_until_process_dies(m2ee):
while True:
if app_is_restarting or m2ee.runner.check_pid():
time.sleep(10)
else:
break
logger.info('process died, stopping')
sys.exit(1)
def _set_log_level(self, subscriber, node, level):
level = level.upper()
response = self.m2ee.set_log_level(subscriber, node, level)
if response.has_error():
response.display_error()
print("Remember, all parameters are case sensitive")
else:
logger.info("Loglevel for %s set to %s" % (node, level))
def _start(self):
"""
This function deals with the start-up sequence of the Mendix Runtime.
Starting the Mendix Runtime can fail in both a temporary or permanent
way. See the client_errno for possible error codes.
"""
if not self.m2ee.config.get_runtime_path():
logger.error("It appears that the Mendix Runtime version which "
"has to be used for your application is not present "
"yet.")
logger.info("You can try downloading it using the "
"download_runtime command.")
return
if not self.m2ee.start_appcontainer():
return
if not self.m2ee.send_runtime_config():
self._stop()
return
abort = False
fully_started = False
params = {}
while not (fully_started or abort):
startresponse = self.m2ee.start_runtime(params)
result = startresponse.get_result()
if result == client_errno.SUCCESS:
fully_started = True
else:
startresponse.display_error()
if result == client_errno.start_NO_EXISTING_DB:
answer = self._ask_user_whether_to_create_db()
if answer == 'a':
abort = True
elif (self.m2ee.config.get_runtime_version() // 2.5 and
answer == 'c'):
params["autocreatedb"] = True
elif result == client_errno.start_INVALID_DB_STRUCTURE:
answer = self._handle_ddl_commands()
if answer == 'a':
abort = True
elif result == client_errno.start_MISSING_MF_CONSTANT:
logger.error("You'll have to add the constant definitions "
"to the configuration in the "
"MicroflowConstants section.")
abort = True
elif result == client_errno.start_ADMIN_1:
answer = self._handle_admin_1(
startresponse.get_feedback()['users'])
if answer == 'a':
abort = True
else:
abort = True
if abort:
self._stop()
def process_request(self):
try:
form = MyFieldStorage(
fp=self.rfile,
headers=self.headers,
environ={
"REQUEST_METHOD": "POST",
"CONTENT_TYPE": self.headers["Content-Type"],
},
)
if "file" in form:
with open(MPK_FILE, "wb") as output:
shutil.copyfileobj(form["file"].file, output)
update_project_dir()
mxbuild_response = build()
logger.debug(mxbuild_response)
if mxbuild_response["status"] == "Busy":
return (200, {"state": "BUSY"}, mxbuild_response)
if mxbuild_response["status"] != "Success":
# possible 'status': Success, Failure, Busy
logger.warning("Failed to build project, "
"keeping previous model running")
state = "FAILED"
elif mxbuild_response["restartRequired"] is True:
logger.info("Restarting app after MPK push")
self.server.restart_callback()
state = "STARTED"
else:
logger.info("Reloading model after MPK push")
self.server.reload_callback()
state = "STARTED"
return (200, {"state": state}, mxbuild_response)
else:
return (
401,
{
"state": "FAILED",
"errordetails": "No MPK found"
},
None,
)
except MxBuildFailure as mbf:
logger.warning(
"InstaDeploy terminating with MxBuildFailure: {}".format(
mbf.message))
return (200, {"state": "FAILED"}, mbf.mxbuild_response)
except Exception:
logger.warning("Instadeploy failed", exc_info=True)
return (
500,
{
"state": "FAILED",
"errordetails": traceback.format_exc()
},
None,
)
def service_backups():
vcap_services = buildpackutil.get_vcap_services_data()
if not vcap_services or 'schnapps' not in vcap_services:
logger.debug("No backup service detected")
return
backup_service = {}
if 'amazon-s3' in vcap_services:
s3_credentials = vcap_services['amazon-s3'][0]['credentials']
backup_service['filesCredentials'] = {
'accessKey': s3_credentials['access_key_id'],
'secretKey': s3_credentials['secret_access_key'],
'bucketName': s3_credentials['bucket'],
}
if 'key_suffix' in s3_credentials: # Not all s3 plans have this field
backup_service['filesCredentials']['keySuffix'] = s3_credentials['key_suffix']
try:
db_config = buildpackutil.get_database_config()
if db_config['DatabaseType'] != 'PostgreSQL':
raise Exception(
'Schnapps only supports postgresql, not %s'
% db_config['DatabaseType']
)
host_and_port = db_config['DatabaseHost'].split(':')
backup_service['databaseCredentials'] = {
'host': host_and_port[0],
'username': db_config['DatabaseUserName'],
'password': db_config['DatabasePassword'],
'dbname': db_config['DatabaseName'],
'port': int(host_and_port[1]) if len(host_and_port) > 1 else 5432,
}
except Exception as e:
logger.exception(
'Schnapps will not be activated because error occurred with '
'parsing the database credentials'
)
return
schnapps_url = vcap_services['schnapps'][0]['credentials']['url']
schnapps_api_key = vcap_services['schnapps'][0]['credentials']['apiKey']
try:
result = requests.put(
schnapps_url,
headers={
'Content-Type': 'application/json',
'apiKey': schnapps_api_key
},
data=json.dumps(backup_service),
)
except Exception as e:
logger.warning('Failed to contact backup service: ' + e)
return
if result.status_code == 200:
logger.info("Successfully updated backup service")
else:
logger.warning("Failed to update backup service: " + result.text)
def activate_appdynamics(m2ee, app_name):
if os.getenv('APPDYNAMICS', 'false').lower() != 'true':
return
logger.info('Adding app dynamics')
m2ee.config._conf['m2ee']['javaopts'].append(
'-javaagent:{path}'.format(
path=os.path.abspath('.local/ver4.2.0.2/javaagent.jar')
)
)
def _handle_admin_1_yolo(self, users):
for username in users:
newpasswd = self._generate_password()
logger.info("Changing password for user %s to %s" %
(username, newpasswd))
self.m2ee.client.update_admin_user({
"username": username,
"password": newpasswd,
})
def do_disable_debugger(self, args):
if self._report_not_implemented(4.3) or self._report_not_running():
return
m2eeresp = self.m2ee.client.disable_debugger()
if not m2eeresp.has_error():
logger.info("The remote debugger is now disabled.")
else:
m2eeresp.display_error()
def loop_until_process_dies(m2ee):
while True:
if app_is_restarting or m2ee.runner.check_pid():
time.sleep(10)
else:
break
emit(jvm={"crash": 1.0})
logger.info("process died, stopping")
sys.exit(1)
def _handle_admin_1_yolo(self, users):
for username in users:
newpasswd = self._generate_password()
logger.info("Changing password for user %s to %s" %
(username, newpasswd))
self.m2ee.client.update_admin_user({
"username": username,
"password": newpasswd,
})
def run(self):
logger.debug("Starting metrics emitter with interval %d",
self.interval)
counter = 1
while True:
logger.info("MENDIX-LOGGING-HEARTBEAT: Heartbeat number %s",
counter)
time.sleep(self.interval)
counter += 1
def do_disable_debugger(self, args):
if self._report_not_implemented(4.3) or self._report_not_running():
return
m2eeresp = self.m2ee.client.disable_debugger()
if not m2eeresp.has_error():
logger.info("The remote debugger is now disabled.")
else:
m2eeresp.display_error()
def do_show_critical_log_messages(self, args):
if self._report_not_running():
return
critlist = self.m2ee.client.get_critical_log_messages()
if len(critlist) == 0:
logger.info("No messages were logged to a critical loglevel since "
"starting the application.")
return
print("\n".join(critlist))
def do_show_critical_log_messages(self, args):
if self._report_not_running():
return
critlist = self.m2ee.client.get_critical_log_messages()
if len(critlist) == 0:
logger.info("No messages were logged to a critical loglevel since "
"starting the application.")
return
print("\n".join(critlist))
def start_metrics(m2ee):
metrics_interval = os.getenv("METRICS_INTERVAL")
profile = os.getenv("PROFILE")
if metrics_interval and profile != "free":
import metrics
thread = metrics.MetricsEmitterThread(int(metrics_interval), m2ee)
thread.setDaemon(True)
thread.start()
else:
logger.info("MENDIX-INTERNAL: Metrics are disabled.")
def __init__(self, yaml_files=None):
logger.debug('Using m2ee-tools version %s' % m2ee.__version__)
cmd.Cmd.__init__(self)
if yaml_files:
self.m2ee = M2EE(yamlfiles=yaml_files, load_default_files=False)
else:
self.m2ee = M2EE()
self.do_status(None)
username = pwd.getpwuid(os.getuid())[0]
self._default_prompt = "m2ee(%s): " % username
self.prompt = self._default_prompt
logger.info("Application Name: %s" % self.m2ee.config.get_app_name())
def __init__(self, interval, m2ee):
super(MetricsEmitterThread, self).__init__()
self.interval = interval
self.m2ee = m2ee
self.db = None
if buildpackutil.bypass_loggregator_logging():
logger.info("Metrics are logged direct to metrics server.")
self.emitter = MetricsServerEmitter(
metrics_url=buildpackutil.get_metrics_url())
else:
logger.info("Metrics are logged to stdout.")
self.emitter = LoggingEmitter()
def _who(self, limitint=None):
limit = {}
if limitint is not None:
limit = {"limit": limitint}
m2eeresp = self.m2ee.client.get_logged_in_user_names(limit)
m2eeresp.display_error()
if not m2eeresp.has_error():
feedback = m2eeresp.get_feedback()
logger.info("Logged in users: (%s) %s" %
(feedback['count'], feedback['users']))
return feedback['count']
return 0
def _who(self, limitint=None):
limit = {}
if limitint is not None:
limit = {"limit": limitint}
m2eeresp = self.m2ee.client.get_logged_in_user_names(limit)
m2eeresp.display_error()
if not m2eeresp.has_error():
feedback = m2eeresp.get_feedback()
logger.info("Logged in users: (%s) %s" %
(feedback['count'], feedback['users']))
return feedback['count']
return 0
def activate_appdynamics(m2ee, app_name):
if not buildpackutil.appdynamics_used():
return
logger.info('Adding app dynamics')
m2ee.config._conf['m2ee']['javaopts'].append('-javaagent:{path}'.format(
path=os.path.abspath('.local/ver4.3.5.7/javaagent.jar')))
APPDYNAMICS_AGENT_NODE_NAME = 'APPDYNAMICS_AGENT_NODE_NAME'
if os.getenv(APPDYNAMICS_AGENT_NODE_NAME):
m2ee.config._conf['m2ee']['custom_environment'][
APPDYNAMICS_AGENT_NODE_NAME] = ('%s-%s' % (
os.getenv(APPDYNAMICS_AGENT_NODE_NAME),
os.getenv('CF_INSTANCE_INDEX', '0'),
))
def do_show_current_runtime_requests(self, args):
if (self._report_not_implemented(('2.5.8', 3.1))
or self._report_not_running()):
return
m2eeresp = self.m2ee.client.get_current_runtime_requests()
m2eeresp.display_error()
if not m2eeresp.has_error():
feedback = m2eeresp.get_feedback()
if not feedback:
logger.info("There are no currently running runtime requests.")
else:
print("Current running Runtime Requests:")
print(yaml.safe_dump(feedback))
def do_show_current_runtime_requests(self, args):
if (self._report_not_implemented(('2.5.8', 3.1))
or self._report_not_running()):
return
m2eeresp = self.m2ee.client.get_current_runtime_requests()
m2eeresp.display_error()
if not m2eeresp.has_error():
feedback = m2eeresp.get_feedback()
if not feedback:
logger.info("There are no currently running runtime requests.")
else:
print("Current running Runtime Requests:")
print(yaml.safe_dump(feedback))
def __init__(self, yaml_files=None, yolo_mode=False):
logger.debug('Using m2ee-tools version %s' % m2ee.__version__)
cmd.Cmd.__init__(self)
if yaml_files:
self.m2ee = M2EE(yamlfiles=yaml_files, load_default_files=False)
else:
self.m2ee = M2EE()
self.yolo_mode = yolo_mode
self.do_status(None)
self.prompt_username = pwd.getpwuid(os.getuid())[0]
self._default_prompt = "m2ee(%s): " % self.prompt_username
self.prompt = self._default_prompt
logger.info("Application Name: %s" % self.m2ee.config.get_app_name())
def set_jvm_memory(m2ee_section, vcap, java_version):
max_memory = os.environ.get('MEMORY_LIMIT')
if max_memory:
match = re.search('([0-9]+)M', max_memory.upper())
limit = int(match.group(1))
else:
limit = int(vcap['limits']['mem'])
if limit >= 8192:
heap_size = limit - 2048
elif limit >= 4096:
heap_size = limit - 1536
elif limit >= 2048:
heap_size = limit - 1024
else:
heap_size = int(limit / 2)
heap_size = str(heap_size) + 'M'
env_heap_size = os.environ.get('HEAP_SIZE')
if env_heap_size:
if int(env_heap_size[:-1]) < limit:
heap_size = env_heap_size
else:
logger.warning(
'specified heap size {} is larger than max memory of the '
'container ({}), falling back to a heap size of {}'.format(
env_heap_size,
str(limit) + 'M',
heap_size,
)
)
javaopts = m2ee_section['javaopts']
javaopts.append('-Xmx%s' % heap_size)
javaopts.append('-Xms%s' % heap_size)
if java_version.startswith('7'):
javaopts.append('-XX:MaxPermSize=256M')
else:
javaopts.append('-XX:MaxMetaspaceSize=256M')
logger.debug('Java heap size set to %s' % heap_size)
if os.getenv('MALLOC_ARENA_MAX'):
logger.info('Using provided environment setting for MALLOC_ARENA_MAX')
else:
m2ee_section['custom_environment']['MALLOC_ARENA_MAX'] = str(
max(1, limit / 1024) * 2
)
def start_app(m2ee):
m2ee.start_appcontainer()
if not m2ee.send_runtime_config():
sys.exit(1)
logger.debug('Appcontainer has been started')
abort = False
success = False
while not (success or abort):
startresponse = m2ee.client.start({'autocreatedb': True})
result = startresponse.get_result()
if result == 0:
success = True
logger.info('The MxRuntime is fully started now.')
else:
startresponse.display_error()
if result == 2:
logger.warning('DB does not exists')
abort = True
elif result == 3:
if am_i_primary_instance():
m2eeresponse = m2ee.client.execute_ddl_commands()
m2eeresponse.display_error()
else:
logger.info(
'waiting 10 seconds before primary instance '
'synchronizes database'
)
time.sleep(10)
elif result == 4:
logger.warning('Not enough constants!')
abort = True
elif result == 5:
logger.warning('Unsafe password!')
abort = True
elif result == 6:
logger.warning('Invalid state!')
abort = True
elif result == 7 or result == 8 or result == 9:
logger.warning(
"You'll have to fix the configuration and run start "
"again... (or ask for help..)"
)
abort = True
else:
abort = True
if abort:
logger.warning('start failed, stopping')
sys.exit(1)
def do_POST(self):
try:
form = cgi.FieldStorage(
fp=self.rfile,
headers=self.headers,
environ={
'REQUEST_METHOD': 'POST',
'CONTENT_TYPE': self.headers['Content-Type'],
})
if 'file' in form:
with open(MPK_FILE, 'wb') as output:
shutil.copyfileobj(form['file'].file, output)
update_project_dir()
mxbuild_response = build()
logger.debug(mxbuild_response)
if mxbuild_response['status'] == 'Busy':
return self._terminate(200, {'state': 'BUSY'}, mxbuild_response)
if mxbuild_response['status'] != 'Success':
# possible 'status': Success, Failure, Busy
logger.warning(
'Failed to build project, '
'keeping previous model running'
)
state = 'FAILED'
elif mxbuild_response['restartRequired'] is True:
logger.info('Restarting app after MPK push')
self.server.restart_callback()
state = 'STARTED'
else:
logger.info('Reloading model after MPK push')
self.server.reload_callback()
state = 'STARTED'
return self._terminate(200, {
'state': state,
}, mxbuild_response)
else:
return self._terminate(401, {
'state': 'FAILED',
'errordetails': 'No MPK found',
})
except MxBuildFailure as mbf:
logger.warning('InstaDeploy terminating with MxBuildFailure: {}'.format(mbf.message))
return self._terminate(200, {'state': 'FAILED'}, mbf.mxbuild_response)
except Exception:
return self._terminate(500, {
'state': 'FAILED',
'errordetails': traceback.format_exc(),
})
def terminate_process():
logger.info('stopping app...')
if not m2ee.stop():
if not m2ee.terminate():
m2ee.kill()
try:
this_process = os.getpgid(0)
logger.debug(
'Terminating process group with pgid={}'.format(this_process))
os.killpg(this_process, signal.SIGTERM)
time.sleep(3)
os.killpg(this_process, signal.SIGKILL)
except Exception:
logger.exception('Failed to terminate all child processes')
def set_jvm_memory(m2ee_section, vcap, java_version):
max_memory = os.environ.get("MEMORY_LIMIT")
if max_memory:
match = re.search("([0-9]+)M", max_memory.upper())
limit = int(match.group(1))
else:
limit = int(vcap["limits"]["mem"])
if limit >= 8192:
heap_size = limit - 2048
elif limit >= 4096:
heap_size = limit - 1536
elif limit >= 2048:
heap_size = limit - 1024
else:
heap_size = int(limit / 2)
heap_size = str(heap_size) + "M"
env_heap_size = os.environ.get("HEAP_SIZE")
if env_heap_size:
if int(env_heap_size[:-1]) < limit:
heap_size = env_heap_size
else:
logger.warning(
"specified heap size {} is larger than max memory of the "
"container ({}), falling back to a heap size of {}".format(
env_heap_size, str(limit) + "M", heap_size
)
)
javaopts = m2ee_section["javaopts"]
javaopts.append("-Xmx%s" % heap_size)
javaopts.append("-Xms%s" % heap_size)
if java_version.startswith("7"):
javaopts.append("-XX:MaxPermSize=256M")
else:
javaopts.append("-XX:MaxMetaspaceSize=256M")
logger.debug("Java heap size set to %s" % heap_size)
if os.getenv("MALLOC_ARENA_MAX"):
logger.info("Using provided environment setting for MALLOC_ARENA_MAX")
else:
m2ee_section["custom_environment"]["MALLOC_ARENA_MAX"] = str(
max(1, limit / 1024) * 2
)
def start_metrics(m2ee):
metrics_interval = os.getenv("METRICS_INTERVAL")
if metrics_interval:
import metrics
if buildpackutil.is_free_app():
thread = metrics.FreeAppsMetricsEmitterThread(
int(metrics_interval), m2ee)
else:
thread = metrics.PaidAppsMetricsEmitterThread(
int(metrics_interval), m2ee)
thread.setDaemon(True)
thread.start()
else:
logger.info("MENDIX-INTERNAL: Metrics are disabled.")
def do_status(self, args):
if self._report_not_running():
return
feedback = self.m2ee.client.runtime_status().get_feedback()
logger.info("The application process is running, the MxRuntime has "
"status: %s" % feedback['status'])
critlist = self.m2ee.client.get_critical_log_messages()
if len(critlist) > 0:
logger.error("%d critical error(s) were logged. Use show_critical"
"_log_messages to view them." % len(critlist))
max_show_users = 10
total_users = self._who(max_show_users)
if total_users > max_show_users:
logger.info("Only showing %s logged in users. Use who to see a "
"complete list." % max_show_users)
def do_emptydb(self, args):
if not self.m2ee.config.is_using_postgresql():
logger.error("Only PostgreSQL databases are supported right now.")
return
(pid_alive, m2ee_alive) = self.m2ee.check_alive()
if pid_alive or m2ee_alive:
logger.warn("The application process is still running, refusing "
"to empty the database right now.")
return
logger.info("This command will drop all tables and sequences in "
"database %s." %
self.m2ee.config.get_pg_environment()['PGDATABASE'])
answer = raw_input("Continue? (y)es, (N)o? ")
if answer != 'y':
print("Aborting!")
return
pgutil.emptydb(self.m2ee.config)
def do_interrupt_request(self, args):
if (self._report_not_implemented(('2.5.8', 3.1))
or self._report_not_running()):
return
if args == "":
logger.error("This function needs a request id as parameter")
logger.error("Use show_current_runtime_requests to view currently "
"running requests")
return
m2eeresp = self.m2ee.client.interrupt_request({"request_id": args})
m2eeresp.display_error()
if not m2eeresp.has_error():
feedback = m2eeresp.get_feedback()
if feedback["result"] is False:
logger.error("A request with ID %s was not found" % args)
else:
logger.info("An attempt to cancel the running action was "
"made.")
def create_admin_user(m2ee):
logger.info('Ensuring admin user credentials')
logger.debug('Creating admin user')
m2eeresponse = m2ee.client.create_admin_user({
'password': os.environ.get('ADMIN_PASSWORD'),
})
if m2eeresponse.has_error():
m2eeresponse.display_error()
sys.exit(1)
logger.debug('Setting admin user password')
m2eeresponse = m2ee.client.create_admin_user({
'username': m2ee.config._model_metadata['AdminUser'],
'password': os.environ.get('ADMIN_PASSWORD'),
})
if m2eeresponse.has_error():
m2eeresponse.display_error()
sys.exit(1)
def do_unpack(self, args):
if not args:
logger.error("unpack needs the name of a model upload zipfile in "
"%s as argument" %
self.m2ee.config.get_model_upload_path())
return
(pid_alive, m2ee_alive) = self.m2ee.check_alive()
if pid_alive or m2ee_alive:
logger.error("The application process is still running, refusing "
"to unpack a new application model right now.")
return
logger.info("This command will replace the contents of the model/ and "
"web/ locations, using the files extracted from the "
"archive")
answer = raw_input("Continue? (y)es, (N)o? ")
if answer != 'y':
logger.info("Aborting!")
return
self.m2ee.unpack(args)
def activate_appdynamics(m2ee, app_name):
if not buildpackutil.appdynamics_used():
return
logger.info('Adding app dynamics')
m2ee.config._conf['m2ee']['javaopts'].append(
'-javaagent:{path}'.format(
path=os.path.abspath('.local/ver4.1.7.1/javaagent.jar')
)
)
APPDYNAMICS_AGENT_NODE_NAME = 'APPDYNAMICS_AGENT_NODE_NAME'
if os.getenv(APPDYNAMICS_AGENT_NODE_NAME):
m2ee.config._conf['m2ee']['custom_environment'][
APPDYNAMICS_AGENT_NODE_NAME
] = (
'%s-%s' % (
os.getenv(APPDYNAMICS_AGENT_NODE_NAME),
os.getenv('CF_INSTANCE_INDEX', '0'),
)
)
def service_backups():
vcap_services = buildpackutil.get_vcap_services_data()
if not vcap_services or 'schnapps' not in vcap_services:
logger.info("No backup service detected")
return
backup_service = {}
if 'amazon-s3' in vcap_services:
s3_credentials = vcap_services['amazon-s3'][0]['credentials']
backup_service['filesCredentials'] = {
'accessKey': s3_credentials['access_key_id'],
'secretKey': s3_credentials['secret_access_key'],
'bucketName': s3_credentials['bucket'],
}
if 'key_suffix' in s3_credentials: # Not all s3 plans have this field
backup_service['filesCredentials']['keySuffix'] = s3_credentials['key_suffix']
if 'PostgreSQL' in vcap_services:
db_config = buildpackutil.get_database_config()
host_and_port = db_config['DatabaseHost'].split(':')
backup_service['databaseCredentials'] = {
'host': host_and_port[0],
'username': db_config['DatabaseUserName'],
'password': db_config['DatabasePassword'],
'dbname': db_config['DatabaseName'],
'port': int(host_and_port[1]) if len(host_and_port) > 1 else 5432,
}
schnapps_url = vcap_services['schnapps'][0]['credentials']['url']
schnapps_api_key = vcap_services['schnapps'][0]['credentials']['apiKey']
result = requests.put(
schnapps_url,
headers={
'Content-Type': 'application/json',
'apiKey': schnapps_api_key
},
data=json.dumps(backup_service),
)
if result.status_code == 200:
logger.info("Successfully updated backup service")
else:
logger.warning("Failed to update backup service: " + result.text)
def _report_not_running(self):
"""
To be used by actions to see whether m2ee is available for executing
requests. Also prints a line when the application is not running.
if self._report_not_running():
return
do_things_that_communicate_using_m2ee_client()
returns True when m2ee is not available for requests, else False
"""
(pid_alive, m2ee_alive) = self.m2ee.check_alive()
if not pid_alive and not m2ee_alive:
logger.info("The application process is not running.")
return True
# if pid is alive, but m2ee does not respond, errors are already
# printed by check_alive
if pid_alive and not m2ee_alive:
return True
return False
def do_restoredb(self, args):
if not self.m2ee.config.is_using_postgresql():
logger.error("Only PostgreSQL databases are supported right now.")
return
if not args:
logger.error("restoredb needs the name of a dump file in %s as arg"
"ument" % self.m2ee.config.get_database_dump_path())
return
(pid_alive, m2ee_alive) = self.m2ee.check_alive()
if pid_alive or m2ee_alive:
logger.warn("The application is still running, refusing to "
"restore the database right now.")
return
database_name = self.m2ee.config.get_pg_environment()['PGDATABASE']
answer = raw_input("This command will restore this dump into database "
"%s. Continue? (y)es, (N)o? " % database_name)
if answer != 'y':
logger.info("Aborting!")
return
pgutil.restoredb(self.m2ee.config, args)
def set_up_m2ee_client(vcap_data):
m2ee = M2EE(yamlfiles=['.local/m2ee.yaml'], load_default_files=False)
version = m2ee.config.get_runtime_version()
mendix_runtimes_path = '/usr/local/share/mendix-runtimes.git'
mendix_runtime_version_path = os.path.join(os.getcwd(), 'runtimes', str(version))
if os.path.isdir(mendix_runtimes_path) and not os.path.isdir(mendix_runtime_version_path):
buildpackutil.mkdir_p(mendix_runtime_version_path)
env = dict(os.environ)
env['GIT_WORK_TREE'] = mendix_runtime_version_path
# checkout the runtime version
process = subprocess.Popen(['git', 'checkout', str(version), '-f'], cwd=mendix_runtimes_path, env=env,
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
process.communicate()
if process.returncode != 0:
# do a 'git fetch --tags' to refresh the bare repo, then retry to checkout the runtime version
logger.info('mendix runtime version {mx_version} is missing in this rootfs'.format(mx_version=version))
process = subprocess.Popen(['git', 'fetch', '--tags', '&&', 'git', 'checkout', str(version), '-f'],
cwd=mendix_runtimes_path, env=env, stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
process.communicate()
if process.returncode != 0:
# download the mendix runtime version from our blobstore
logger.info('unable to use rootfs for mendix runtime version {mx_version}'.format(mx_version=version))
url = buildpackutil.get_blobstore_url('/runtime/mendix-%s.tar.gz' % str(version))
buildpackutil.download_and_unpack(url, os.path.join(os.getcwd(), 'runtimes'))
m2ee.reload_config()
set_runtime_config(
m2ee.config._model_metadata,
m2ee.config._conf['mxruntime'],
vcap_data,
m2ee,
)
set_heap_size(m2ee.config._conf['m2ee']['javaopts'])
activate_new_relic(m2ee, vcap_data['application_name'])
activate_appdynamics(m2ee, vcap_data['application_name'])
set_application_name(m2ee, vcap_data['application_name'])
return m2ee
def set_runtime_config(metadata, mxruntime_config, vcap_data, m2ee):
scheduled_event_execution, my_scheduled_events = (
get_scheduled_events(metadata)
)
app_config = {
'ApplicationRootUrl': 'https://%s' % vcap_data['application_uris'][0],
'MicroflowConstants': get_constants(metadata),
'ScheduledEventExecution': scheduled_event_execution,
}
if my_scheduled_events is not None:
app_config['MyScheduledEvents'] = my_scheduled_events
if is_development_mode():
logger.warning(
'Runtime is being started in Development Mode. Set '
'DEVELOPMENT_MODE to "false" (currently "true") to '
'set it to production.'
)
app_config['DTAPMode'] = 'D'
if (m2ee.config.get_runtime_version() >= 5.15 and
os.getenv('ENABLE_STICKY_SESSIONS', 'false').lower() == 'true' and
not is_cluster_enabled()):
logger.info('Enabling sticky sessions')
app_config['com.mendix.core.SessionIdCookieName'] = 'JSESSIONID'
mxruntime_config.update(app_config)
mxruntime_config.update(buildpackutil.get_database_config(
development_mode=is_development_mode(),
))
mxruntime_config.update(get_filestore_config(m2ee))
mxruntime_config.update(get_cluster_config())
mxruntime_config.update(get_certificate_authorities())
mxruntime_config.update(get_custom_settings(metadata, mxruntime_config))
for k, v in os.environ.iteritems():
if k.startswith('MXRUNTIME_'):
mxruntime_config[
k.replace('MXRUNTIME_', '', 1).replace('_', '.')
] = v
def do_check_health(self, args):
if self._report_not_implemented('2.5.4') or self._report_not_running():
return
health_response = self.m2ee.client.check_health()
if not health_response.has_error():
feedback = health_response.get_feedback()
if feedback['health'] == 'healthy':
logger.info("Health check microflow says the application is "
"healthy.")
elif feedback['health'] == 'sick':
logger.warning("Health check microflow says the application "
"is sick: %s" % feedback['diagnosis'])
elif feedback['health'] == 'unknown':
logger.info("Health check microflow is not configured, no "
"health information available.")
else:
logger.error("Unexpected health check status: %s" %
feedback['health'])
else:
runtime_version = self.m2ee.config.get_runtime_version()
if (health_response.get_result() == 3
and runtime_version // ('2.5.4', '2.5.5')):
# Because of an incomplete implementation, in Mendix 2.5.4 or
# 2.5.5 this means that the runtime is health-check
# capable, but no health check microflow is defined.
logger.info("Health check microflow is probably not "
"configured, no health information available.")
else:
health_response.display_error()
def activate_new_relic(m2ee, app_name):
if buildpackutil.get_new_relic_license_key() is None:
logger.debug(
'Skipping New Relic setup, no license key found in environment'
)
return
logger.info('Adding new relic')
m2ee_section = m2ee.config._conf['m2ee']
if 'custom_environment' not in m2ee_section:
m2ee_section['custom_environment'] = {}
m2ee_section['custom_environment']['NEW_RELIC_LICENSE_KEY'] = (
buildpackutil.get_new_relic_license_key()
)
m2ee_section['custom_environment']['NEW_RELIC_APP_NAME'] = app_name
m2ee_section['custom_environment']['NEW_RELIC_LOG'] = (
os.path.abspath('newrelic/agent.log')
)
m2ee.config._conf['m2ee']['javaopts'].append(
'-javaagent:{path}'.format(
path=os.path.abspath('newrelic/newrelic.jar')
)
)
def configure_debugger(m2ee):
debugger_password = os.environ.get('DEBUGGER_PASSWORD')
if debugger_password is None:
logger.debug('Not configuring debugger, as environment variable '
'was not found')
return
response = m2ee.client.enable_debugger({
'password': debugger_password
})
response.display_error()
if not response.has_error():
logger.info(
'The remote debugger is now enabled, the password to '
'use is %s' % debugger_password
)
logger.info(
'You can use the remote debugger option in the Mendix '
'Business Modeler to connect to the /debugger/ sub '
'url on your application (e.g. '
'https://app.example.com/debugger/). '
)
def _stop(self):
(pid_alive, m2ee_alive) = self.m2ee.check_alive()
if not pid_alive and not m2ee_alive:
logger.info("Nothing to stop, the application is not running.")
return True
logger.debug("Trying to stop the application.")
stopped = False
stopped = self.m2ee.stop()
if stopped:
return True
answer = None
while answer not in ('y', 'n'):
answer = ('y' if self.yolo_mode
else raw_input("Do you want to try to signal the JVM "
"process to stop immediately? (y)es, (n)o? "))
if answer == 'y':
stopped = self.m2ee.terminate()
if stopped:
return True
elif answer == 'n':
logger.info("Doing nothing, use stop again to check if the "
"process finally disappeared...")
return False
else:
print("Unknown option %s" % answer)
answer = None
while answer not in ('y', 'n'):
answer = ('y' if self.yolo_mode
else raw_input("Do you want to kill the JVM process? "
"(y)es, (n)o? "))
if answer == 'y':
stopped = self.m2ee.kill()
if stopped:
return True
elif answer == 'n':
logger.info("Doing nothing, use stop again to check if the "
"process finally disappeared...")
return False
else:
print("Unknown option %s" % answer)
return False
def display_running_version(m2ee):
if m2ee.config.get_runtime_version() >= 4.4:
feedback = m2ee.client.about().get_feedback()
if 'model_version' in feedback:
logger.info('Model version: %s' % feedback['model_version'])
def loop_until_process_dies(m2ee):
while m2ee.runner.check_pid():
time.sleep(10)
logger.info('process died, stopping')
sys.exit(1)
def get_filestore_config(m2ee):
access_key = secret = bucket = encryption_keys = key_suffix = None
vcap_services = buildpackutil.get_vcap_services_data()
endpoint = None
v2_auth = ''
if vcap_services and 'amazon-s3' in vcap_services:
_conf = vcap_services['amazon-s3'][0]['credentials']
access_key = _conf['access_key_id']
secret = _conf['secret_access_key']
bucket = _conf['bucket']
if 'encryption_keys' in _conf:
encryption_keys = _conf['encryption_keys']
if 'key_suffix' in _conf:
key_suffix = _conf['key_suffix']
elif vcap_services and 'p-riakcs' in vcap_services:
_conf = vcap_services['p-riakcs'][0]['credentials']
access_key = _conf['access_key_id']
secret = _conf['secret_access_key']
pattern = r'https://(([^:]+):([^@]+)@)?([^/]+)/(.*)'
match = re.search(pattern, _conf['uri'])
endpoint = 'https://' + match.group(4)
bucket = match.group(5)
v2_auth = 'true'
access_key = os.getenv('S3_ACCESS_KEY_ID', access_key)
secret = os.getenv('S3_SECRET_ACCESS_KEY', secret)
bucket = os.getenv('S3_BUCKET_NAME', bucket)
if 'S3_ENCRYPTION_KEYS' in os.environ:
encryption_keys = json.loads(os.getenv('S3_ENCRYPTION_KEYS'))
perform_deletes = os.getenv('S3_PERFORM_DELETES', '').lower() == 'false'
key_suffix = os.getenv('S3_KEY_SUFFIX', key_suffix)
endpoint = os.getenv('S3_ENDPOINT', endpoint)
v2_auth = os.getenv('S3_USE_V2_AUTH', v2_auth).lower() == 'true'
sse = os.getenv('S3_USE_SSE', '').lower() == 'true'
if not (access_key and secret and bucket):
logger.warning(
'External file store not configured, uploaded files in the app '
'will not persist across restarts. See https://github.com/mendix/'
'cf-mendix-buildpack for file store configuration details.'
)
return {}
logger.info(
'S3 config detected, activating external file store'
)
config = {
'com.mendix.core.StorageService': 'com.mendix.storage.s3',
'com.mendix.storage.s3.AccessKeyId': access_key,
'com.mendix.storage.s3.SecretAccessKey': secret,
'com.mendix.storage.s3.BucketName': bucket,
}
if not perform_deletes:
config['com.mendix.storage.s3.PerformDeleteFromStorage'] = False
if key_suffix:
config['com.mendix.storage.s3.ResourceNameSuffix'] = key_suffix
if v2_auth:
config['com.mendix.storage.s3.UseV2Auth'] = v2_auth
if endpoint:
config['com.mendix.storage.s3.EndPoint'] = endpoint
if m2ee.config.get_runtime_version() >= 5.17 and encryption_keys:
config['com.mendix.storage.s3.EncryptionKeys'] = encryption_keys
if m2ee.config.get_runtime_version() >= 6 and sse:
config['com.mendix.storage.s3.UseSSE'] = sse
return config
