def launch_command(self, cmd, timeout=300):
self.logger.info("Launch command:{0}", cmd)
base_dir = self.get_base_dir()
try:
devnull = open(os.devnull, 'w')
child = subprocess.Popen(base_dir + "/" + cmd,
shell=True,
cwd=base_dir,
stdout=devnull)
except Exception as e:
#TODO do not catch all exception
raise ExtensionError("Failed to launch: {0}, {1}".format(cmd, e))
retry = timeout
while retry > 0 and child.poll() is None:
time.sleep(1)
retry -= 1
if retry == 0:
os.kill(child.pid, 9)
raise ExtensionError("Timeout({0}): {1}".format(timeout, cmd))
ret = child.wait()
if ret == None or ret != 0:
raise ExtensionError("Non-zero exit code: {0}, {1}".format(
ret, cmd))
self.report_event(message="Launch command succeeded: {0}".format(cmd))
def capture_from_process_modern(process, cmd, timeout):
try:
stdout, stderr = process.communicate(timeout=timeout)
except subprocess.TimeoutExpired:
# Just kill the process. The .communicate method will gather stdout/stderr, close those pipes, and reap
# the zombie process. That is, .communicate() does all the other stuff that _destroy_process does.
os.killpg(os.getpgid(process.pid), signal.SIGKILL)
stdout, stderr = process.communicate()
msg = format_stdout_stderr(sanitize(stdout), sanitize(stderr))
raise ExtensionError("Timeout({0}): {1}\n{2}".format(
timeout, cmd, msg))
except OSError as e:
_destroy_process(process, signal.SIGKILL)
raise ExtensionError("Error while running '{0}': {1}".format(
cmd, e.strerror))
except ValueError:
_destroy_process(process, signal.SIGKILL)
raise ExtensionError(
"Invalid timeout ({0}) specified for '{1}'".format(timeout, cmd))
except Exception as e:
_destroy_process(process, signal.SIGKILL)
raise ExtensionError("Exception while running '{0}': {1}".format(
cmd, e))
return stdout, stderr
def launch_command(self, cmd, timeout=300):
begin_utc = datetime.datetime.utcnow()
self.logger.verbose("Launch command: [{0}]", cmd)
base_dir = self.get_base_dir()
try:
# This should be .run(), but due to the wide variety
# of Python versions we must support we must use .communicate().
# Some extensions erroneously begin cmd with a slash; don't interpret those
# as root-relative. (Issue #1170)
full_path = os.path.join(base_dir, cmd.lstrip(os.sep))
process = subprocess.Popen(full_path,
shell=True,
cwd=base_dir,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env=os.environ,
preexec_fn=os.setsid)
except OSError as e:
raise ExtensionError("Failed to launch '{0}': {1}".format(
full_path, e.strerror))
msg = capture_from_process(process, cmd, timeout)
ret = process.poll()
if ret is None or ret != 0:
raise ExtensionError("Non-zero exit code: {0}, {1}\n{2}".format(
ret, cmd, msg))
duration = elapsed_milliseconds(begin_utc)
self.report_event(message="{0}\n{1}".format(cmd, msg),
duration=duration,
log_event=False)
def launch_command(self, cmd, timeout=300):
begin_utc = datetime.datetime.utcnow()
self.logger.verbose("Launch command: [{0}]", cmd)
base_dir = self.get_base_dir()
def sanitize(s):
return ustr(s, encoding='utf-8', errors='backslashreplace')
try:
# This should be .run(), but due to the wide variety
# of Python versions we must support we must use .communicate().
process = subprocess.Popen(os.path.join(base_dir, cmd),
shell=True,
cwd=base_dir,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env=os.environ)
stdout, stderr = process.communicate(timeout)
except subprocess.TimeoutExpired:
process.kill()
stdout, stderr = process.communicate()
msg = format_stdout_stderr(sanitize(stdout), sanitize(stderr))
raise ExtensionError("Timeout({0}): {1}\n{2}".format(timeout, cmd, msg))
except Exception as e:
process.kill()
process.wait()
raise ExtensionError("Failed to launch: {0}, {1}".format(cmd, e))
ret = process.poll()
if ret is None or ret != 0:
raise ExtensionError("Non-zero exit code: {0}, {1}".format(ret, cmd))
duration = elapsed_milliseconds(begin_utc)
msg = format_stdout_stderr(sanitize(stdout), sanitize(stderr))
self.report_event(message="{0}\n{1}".format(cmd, msg), duration=duration)
def launch_command(self, cmd, timeout=300):
begin_utc = datetime.datetime.utcnow()
self.logger.verbose("Launch command: [{0}]", cmd)
base_dir = self.get_base_dir()
try:
devnull = open(os.devnull, 'w')
child = subprocess.Popen(base_dir + "/" + cmd,
shell=True,
cwd=base_dir,
stdout=devnull,
env=os.environ)
except Exception as e:
#TODO do not catch all exception
raise ExtensionError("Failed to launch: {0}, {1}".format(cmd, e))
retry = timeout
while retry > 0 and child.poll() is None:
time.sleep(1)
retry -= 1
if retry == 0:
os.kill(child.pid, 9)
raise ExtensionError("Timeout({0}): {1}".format(timeout, cmd))
ret = child.wait()
if ret == None or ret != 0:
raise ExtensionError("Non-zero exit code: {0}, {1}".format(
ret, cmd))
duration = elapsed_milliseconds(begin_utc)
self.report_event(message="Launch command succeeded: {0}".format(cmd),
duration=duration)
def capture_from_process_poll(process, cmd, timeout, code):
"""
Capture output from the process if it does not fork, or forks
and completes quickly.
"""
retry = timeout
while retry > 0 and process.poll() is None:
time.sleep(1)
retry -= 1
# process did not fork, timeout expired
if retry == 0:
os.killpg(os.getpgid(process.pid), signal.SIGKILL)
stdout, stderr = process.communicate()
msg = format_stdout_stderr(sanitize(stdout), sanitize(stderr))
raise ExtensionError("Timeout({0}): {1}\n{2}".format(
timeout, cmd, msg),
code=code)
# process completed or forked
return_code = process.wait()
if return_code != 0:
raise ExtensionError("Non-zero exit code: {0}, {1}".format(
return_code, cmd),
code=code)
stderr = b''
stdout = b'cannot collect stdout'
# attempt non-blocking process communication to capture output
def proc_comm(_process, _return):
try:
_stdout, _stderr = _process.communicate()
_return[0] = _stdout
_return[1] = _stderr
except Exception:
pass
try:
mgr = multiprocessing.Manager()
ret_dict = mgr.dict()
cproc = Process(target=proc_comm, args=(process, ret_dict))
cproc.start()
# allow 1s to capture output
cproc.join(1)
if len(ret_dict) == 2:
stdout = ret_dict[0]
stderr = ret_dict[1]
except Exception:
pass
return stdout, stderr
def load_manifest(self):
man_file = self.get_manifest_file()
try:
data = json.loads(fileutil.read_file(man_file))
except (IOError, OSError) as e:
raise ExtensionError('Failed to load manifest file ({0}): {1}'.format(man_file, e.strerror))
except ValueError:
raise ExtensionError('Malformed manifest file ({0}).'.format(man_file))
return HandlerManifest(data[0])
def load_manifest(self):
man_file = self.get_manifest_file()
try:
data = json.loads(fileutil.read_file(man_file))
except IOError as e:
raise ExtensionError('Failed to load manifest file.')
except ValueError as e:
raise ExtensionError('Malformed manifest file.')
return HandlerManifest(data[0])
def launch_command(self, cmd, timeout=300, extension_error_code=1000, env=None):
begin_utc = datetime.datetime.utcnow()
self.logger.verbose("Launch command: [{0}]", cmd)
base_dir = self.get_base_dir()
if env is None:
env = {}
env.update(os.environ)
try:
# This should be .run(), but due to the wide variety
# of Python versions we must support we must use .communicate().
# Some extensions erroneously begin cmd with a slash; don't interpret those
# as root-relative. (Issue #1170)
full_path = os.path.join(base_dir, cmd.lstrip(os.path.sep))
def pre_exec_function():
"""
Change process state before the actual target process is started. Effectively, this runs between
the fork() and the exec() of sub-process creation.
:return:
"""
os.setsid()
CGroups.add_to_extension_cgroup(self.ext_handler.name)
process = subprocess.Popen(full_path,
shell=True,
cwd=base_dir,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env=env,
preexec_fn=pre_exec_function)
except OSError as e:
raise ExtensionError("Failed to launch '{0}': {1}".format(full_path, e.strerror),
code=extension_error_code)
cg = CGroups.for_extension(self.ext_handler.name)
CGroupsTelemetry.track_extension(self.ext_handler.name, cg)
msg = capture_from_process(process, cmd, timeout, extension_error_code)
ret = process.poll()
if ret is None:
raise ExtensionError("Process {0} was not terminated: {1}\n{2}".format(process.pid, cmd, msg),
code=extension_error_code)
if ret != 0:
raise ExtensionError("Non-zero exit code: {0}, {1}\n{2}".format(ret, cmd, msg),
code=extension_error_code)
duration = elapsed_milliseconds(begin_utc)
log_msg = "{0}\n{1}".format(cmd, "\n".join([line for line in msg.split('\n') if line != ""]))
self.logger.verbose(log_msg)
self.report_event(message=log_msg, duration=duration, log_event=False)
def capture_from_process_no_timeout(process, cmd):
try:
stdout, stderr = process.communicate()
except OSError as e:
_destroy_process(process, signal.SIGKILL)
raise ExtensionError("Error while running '{0}': {1}".format(
cmd, e.strerror))
except Exception as e:
_destroy_process(process, signal.SIGKILL)
raise ExtensionError("Exception while running '{0}': {1}".format(
cmd, e))
return stdout, stderr
def capture_from_process_pre_33(process, cmd, timeout):
"""
Can't use process.communicate(timeout=), so do it the hard way.
"""
watcher_process_exited = 0
watcher_process_timed_out = 1
def kill_on_timeout(pid, watcher_timeout):
"""
Check for the continued existence of pid once per second. If pid no longer exists, exit with code 0.
If timeout (in seconds) elapses, kill pid and exit with code 1.
"""
for iteration in range(watcher_timeout):
time.sleep(1)
try:
os.kill(pid, 0)
except OSError as ex:
if ESRCH == ex.errno: # Process no longer exists
exit(watcher_process_exited)
os.killpg(os.getpgid(pid), signal.SIGKILL)
exit(watcher_process_timed_out)
watcher = Process(target=kill_on_timeout, args=(process.pid, timeout))
watcher.start()
try:
# Now, block "forever" waiting on process. If the timeout-limited Event wait in the watcher pops,
# it will kill the process and Popen.communicate() will return
stdout, stderr = process.communicate()
except OSError as e:
_destroy_process(process, signal.SIGKILL)
raise ExtensionError("Error while running '{0}': {1}".format(
cmd, e.strerror))
except Exception as e:
_destroy_process(process, signal.SIGKILL)
raise ExtensionError("Exception while running '{0}': {1}".format(
cmd, e))
timeout_happened = False
watcher.join(1)
if watcher.is_alive():
watcher.terminate()
else:
timeout_happened = (watcher.exitcode == watcher_process_timed_out)
if timeout_happened:
msg = format_stdout_stderr(sanitize(stdout), sanitize(stderr))
raise ExtensionError("Timeout({0}): {1}\n{2}".format(
timeout, cmd, msg))
return stdout, stderr
def capture_from_process_raw(process, cmd, timeout):
"""
Captures stdout and stderr from an already-created process.
:param subprocess.Popen process: Created by subprocess.Popen()
:param str cmd: The command string to be included in any exceptions
:param int timeout: Number of seconds the process is permitted to run
:return: The stdout and stderr captured from the process
:rtype: (str, str)
:raises ExtensionError: if a timeout occurred or if anything was raised by Popen.communicate()
"""
if not timeout:
stdout, stderr = capture_from_process_no_timeout(process, cmd)
else:
if os.getpgid(process.pid) != process.pid:
_destroy_process(process, signal.SIGKILL)
raise ExtensionError(
"Subprocess was not root of its own process group")
if sys.version_info < (3, 3):
stdout, stderr = capture_from_process_pre_33(process, cmd, timeout)
else:
stdout, stderr = capture_from_process_modern(process, cmd, timeout)
return stdout, stderr
def handle_process_completion(process, command, timeout, stdout, stderr,
error_code): # pylint: disable=R0913
"""
Utility function that waits for process completion and retrieves its output (stdout and stderr) if it completed
before the timeout period. Otherwise, the process will get killed and an ExtensionError will be raised.
In case the return code is non-zero, ExtensionError will be raised.
:param process: Reference to a running process
:param command: The extension command to run
:param timeout: Number of seconds to wait before killing the process
:param stdout: Must be a file since we seek on it when parsing the subprocess output
:param stderr: Must be a file since we seek on it when parsing the subprocess outputs
:param error_code: The error code to set if we raise an ExtensionError
:return:
"""
# Wait for process completion or timeout
timed_out, return_code = wait_for_process_completion_or_timeout(
process, timeout)
process_output = read_output(stdout, stderr)
if timed_out:
raise ExtensionError(
"Timeout({0}): {1}\n{2}".format(timeout, command, process_output),
code=ExtensionErrorCodes.PluginHandlerScriptTimedout)
if return_code != 0:
raise ExtensionOperationError(
"Non-zero exit code: {0}, {1}\n{2}".format(return_code, command,
process_output),
code=error_code,
exit_code=return_code)
return process_output
def update_settings_file(self, settings_file, settings):
settings_file = os.path.join(self.get_conf_dir(), settings_file)
try:
fileutil.write_file(settings_file, settings)
except IOError as e:
fileutil.clean_ioerror(e, paths=[settings_file])
raise ExtensionError(u"Failed to update settings file", e)
def handle_enable(self, ext_handler_i):
old_ext_handler_i = ext_handler_i.get_installed_ext_handler()
if old_ext_handler_i is not None and \
old_ext_handler_i.version_gt(ext_handler_i):
raise ExtensionError(u"Downgrade not allowed")
handler_state = ext_handler_i.get_handler_state()
ext_handler_i.logger.info("[Enable] current handler state is: {0}",
handler_state.lower())
if handler_state == ExtHandlerState.NotInstalled:
ext_handler_i.set_handler_state(ExtHandlerState.NotInstalled)
ext_handler_i.download()
ext_handler_i.update_settings()
if old_ext_handler_i is None:
ext_handler_i.install()
elif ext_handler_i.version_gt(old_ext_handler_i):
old_ext_handler_i.disable()
ext_handler_i.copy_status_files(old_ext_handler_i)
ext_handler_i.update()
old_ext_handler_i.uninstall()
old_ext_handler_i.rm_ext_handler_dir()
ext_handler_i.update_with_install()
else:
ext_handler_i.update_settings()
ext_handler_i.enable()
def handle_ext_handler(self, ext_handler, etag):
ext_handler_i = ExtHandlerInstance(ext_handler, self.protocol)
try:
ext_handler_i.decide_version()
if not ext_handler_i.is_upgrade and self.last_etag == etag:
if self.log_etag:
ext_handler_i.logger.verbose(
"Version {0} is current for etag {1}",
ext_handler_i.pkg.version, etag)
self.log_etag = False
return
self.log_etag = True
state = ext_handler.properties.state
ext_handler_i.logger.info("Target handler state: {0}", state)
if state == u"enabled":
self.handle_enable(ext_handler_i)
elif state == u"disabled":
self.handle_disable(ext_handler_i)
elif state == u"uninstall":
self.handle_uninstall(ext_handler_i)
else:
message = u"Unknown ext handler state:{0}".format(state)
raise ExtensionError(message)
except ExtensionError as e:
ext_handler_i.set_handler_status(message=ustr(e), code=-1)
ext_handler_i.report_event(message=ustr(e), is_success=False)
def handle_ext_handler(self, ext_handler, etag):
ext_handler_i = ExtHandlerInstance(ext_handler, self.protocol)
try:
state = ext_handler.properties.state
# The extension is to be enabled, there is an upgrade GUID
# and the GUID is NOT new
if state == u"enabled" and \
ext_handler.properties.upgradeGuid is not None and \
not self.is_new_guid(ext_handler):
ext_handler_i.ext_handler.properties.version = ext_handler_i.get_installed_version()
ext_handler_i.set_logger()
if self.last_etag != etag:
self.set_log_upgrade_guid(ext_handler, True)
msg = "New GUID is the same as the old GUID. Exiting without upgrading."
if self.get_log_upgrade_guid(ext_handler):
ext_handler_i.logger.info(msg)
self.set_log_upgrade_guid(ext_handler, False)
ext_handler_i.set_handler_state(ExtHandlerState.Enabled)
ext_handler_i.set_handler_status(status="Ready", message="No change")
ext_handler_i.set_operation(WALAEventOperation.SkipUpdate)
ext_handler_i.report_event(message=ustr(msg), is_success=True)
return
self.set_log_upgrade_guid(ext_handler, True)
ext_handler_i.decide_version(target_state=state)
if not ext_handler_i.is_upgrade and self.last_etag == etag:
if self.log_etag:
ext_handler_i.logger.verbose("Version {0} is current for etag {1}",
ext_handler_i.pkg.version,
etag)
self.log_etag = False
return
self.log_etag = True
ext_handler_i.logger.info("Target handler state: {0}", state)
if state == u"enabled":
self.handle_enable(ext_handler_i)
if ext_handler.properties.upgradeGuid is not None:
ext_handler_i.logger.info("New Upgrade GUID: {0}", ext_handler.properties.upgradeGuid)
self.last_upgrade_guids[ext_handler.name] = (ext_handler.properties.upgradeGuid, True)
elif state == u"disabled":
self.handle_disable(ext_handler_i)
# Remove the GUID from the dictionary so that it is upgraded upon re-enable
self.last_upgrade_guids.pop(ext_handler.name, None)
elif state == u"uninstall":
self.handle_uninstall(ext_handler_i)
# Remove the GUID from the dictionary so that it is upgraded upon re-install
self.last_upgrade_guids.pop(ext_handler.name, None)
else:
message = u"Unknown ext handler state:{0}".format(state)
raise ExtensionError(message)
except Exception as e:
ext_handler_i.set_handler_status(message=ustr(e), code=-1)
ext_handler_i.report_event(message=ustr(e), is_success=False)
def decide_version(self, target_state=None):
self.logger.verbose("Decide which version to use")
try:
pkg_list = self.protocol.get_ext_handler_pkgs(self.ext_handler)
except ProtocolError as e:
raise ExtensionError("Failed to get ext handler pkgs", e)
# Determine the desired and installed versions
requested_version = FlexibleVersion(
str(self.ext_handler.properties.version))
installed_version_string = self.get_installed_version()
installed_version = requested_version \
if installed_version_string is None \
else FlexibleVersion(installed_version_string)
# Divide packages
# - Find the installed package (its version must exactly match)
# - Find the internal candidate (its version must exactly match)
# - Separate the public packages
selected_pkg = None
installed_pkg = None
pkg_list.versions.sort(key=lambda p: FlexibleVersion(p.version))
for pkg in pkg_list.versions:
pkg_version = FlexibleVersion(pkg.version)
if pkg_version == installed_version:
installed_pkg = pkg
if requested_version.matches(pkg_version):
selected_pkg = pkg
# Finally, update the version only if not downgrading
# Note:
# - A downgrade, which will be bound to the same major version,
# is allowed if the installed version is no longer available
if target_state == u"uninstall" or target_state == u"disabled":
if installed_pkg is None:
msg = "Failed to find installed version of {0} " \
"to uninstall".format(self.ext_handler.name)
self.logger.warn(msg)
self.pkg = installed_pkg
self.ext_handler.properties.version = str(installed_version) \
if installed_version is not None else None
else:
self.pkg = selected_pkg
if self.pkg is not None:
self.ext_handler.properties.version = str(selected_pkg.version)
# Note if the selected package is different than that installed
if installed_pkg is None \
or (self.pkg is not None and FlexibleVersion(self.pkg.version) != FlexibleVersion(installed_pkg.version)):
self.is_upgrade = True
if self.pkg is not None:
self.logger.verbose("Use version: {0}", self.pkg.version)
self.set_logger()
return self.pkg
def download(self):
self.logger.verbose("Download extension package")
self.set_operation(WALAEventOperation.Download)
if self.pkg is None:
raise ExtensionError("No package uri found")
package = None
for uri in self.pkg.uris:
try:
package = self.protocol.download_ext_handler_pkg(uri.uri)
if package is not None:
break
except Exception as e:
logger.warn("Error while downloading extension: {0}", e)
if package is None:
raise ExtensionError("Failed to download extension")
self.logger.verbose("Unpack extension package")
pkg_file = os.path.join(conf.get_lib_dir(),
os.path.basename(uri.uri) + ".zip")
try:
fileutil.write_file(pkg_file, bytearray(package), asbin=True)
zipfile.ZipFile(pkg_file).extractall(self.get_base_dir())
except IOError as e:
raise ExtensionError(u"Failed to write and unzip plugin", e)
#Add user execute permission to all files under the base dir
for file in fileutil.get_all_files(self.get_base_dir()):
fileutil.chmod(file, os.stat(file).st_mode | stat.S_IXUSR)
self.report_event(message="Download succeeded")
self.logger.info("Initialize extension directory")
#Save HandlerManifest.json
man_file = fileutil.search_file(self.get_base_dir(),
'HandlerManifest.json')
if man_file is None:
raise ExtensionError("HandlerManifest.json not found")
try:
man = fileutil.read_file(man_file, remove_bom=True)
fileutil.write_file(self.get_manifest_file(), man)
except IOError as e:
raise ExtensionError(u"Failed to save HandlerManifest.json", e)
#Create status and config dir
try:
status_dir = self.get_status_dir()
fileutil.mkdir(status_dir, mode=0o700)
conf_dir = self.get_conf_dir()
fileutil.mkdir(conf_dir, mode=0o700)
except IOError as e:
raise ExtensionError(u"Failed to create status or config dir", e)
#Save HandlerEnvironment.json
self.create_handler_env()
def collect_heartbeat(self):
man = self.load_manifest()
if not man.is_report_heartbeat():
return
heartbeat_file = os.path.join(conf.get_lib_dir(),
self.get_heartbeat_file())
if not os.path.isfile(heartbeat_file):
raise ExtensionError("Failed to get heart beat file")
if not self.is_responsive(heartbeat_file):
return {
"status": "Unresponsive",
"code": -1,
"message": "Extension heartbeat is not responsive"
}
try:
heartbeat_json = fileutil.read_file(heartbeat_file)
heartbeat = json.loads(heartbeat_json)[0]['heartbeat']
except IOError as e:
raise ExtensionError("Failed to get heartbeat file:{0}".format(e))
except (ValueError, KeyError) as e:
raise ExtensionError("Malformed heartbeat file: {0}".format(e))
return heartbeat
def create_handler_env(self):
env = [{
"name": self.ext_handler.name,
"version": HANDLER_ENVIRONMENT_VERSION,
"handlerEnvironment": {
"logFolder": self.get_log_dir(),
"configFolder": self.get_conf_dir(),
"statusFolder": self.get_status_dir(),
"heartbeatFile": self.get_heartbeat_file()
}
}]
try:
fileutil.write_file(self.get_env_file(), json.dumps(env))
except IOError as e:
raise ExtensionError(u"Failed to save handler environment", e)
def download(self):
self.logger.info("Download extension package")
self.set_operation(WALAEventOperation.Download)
if self.pkg is None:
raise ExtensionError("No package uri found")
package = None
for uri in self.pkg.uris:
try:
package = self.protocol.download_ext_handler_pkg(uri.uri)
except ProtocolError as e:
logger.warn("Failed download extension: {0}", e)
if package is None:
raise ExtensionError("Failed to download extension")
self.logger.info("Unpack extension package")
pkg_file = os.path.join(conf.get_lib_dir(),
os.path.basename(uri.uri) + ".zip")
try:
fileutil.write_file(pkg_file, bytearray(package), asbin=True)
zipfile.ZipFile(pkg_file).extractall(self.get_base_dir())
except IOError as e:
raise ExtensionError(u"Failed to write and unzip plugin", e)
chmod = "find {0} -type f | xargs chmod u+x".format(self.get_base_dir())
shellutil.run(chmod)
self.report_event(message="Download succeeded")
self.logger.info("Initialize extension directory")
#Save HandlerManifest.json
man_file = fileutil.search_file(self.get_base_dir(),
'HandlerManifest.json')
if man_file is None:
raise ExtensionError("HandlerManifest.json not found")
try:
man = fileutil.read_file(man_file, remove_bom=True)
fileutil.write_file(self.get_manifest_file(), man)
except IOError as e:
raise ExtensionError(u"Failed to save HandlerManifest.json", e)
#Create status and config dir
try:
status_dir = self.get_status_dir()
fileutil.mkdir(status_dir, mode=0o700)
conf_dir = self.get_conf_dir()
fileutil.mkdir(conf_dir, mode=0o700)
except IOError as e:
raise ExtensionError(u"Failed to create status or config dir", e)
#Save HandlerEnvironment.json
self.create_handler_env()
def handle_ext_handler(self, ext_handler):
ext_handler_i = ExtHandlerInstance(ext_handler, self.protocol)
try:
state = ext_handler.properties.state
ext_handler_i.logger.info("Expected handler state: {0}", state)
if state == "enabled":
self.handle_enable(ext_handler_i)
elif state == u"disabled":
self.handle_disable(ext_handler_i)
elif state == u"uninstall":
self.handle_uninstall(ext_handler_i)
else:
message = u"Unknown ext handler state:{0}".format(state)
raise ExtensionError(message)
except ExtensionError as e:
ext_handler_i.set_handler_status(message=ustr(e), code=-1)
ext_handler_i.report_event(message=ustr(e), is_success=False)
def handle_ext_handler(self, ext_handler, etag):
ext_handler_i = ExtHandlerInstance(ext_handler, self.protocol)
try:
state = ext_handler.properties.state
max_retries = 5
retry_count = 0
while ext_handler_i.decide_version(target_state=state) is None:
if retry_count >= max_retries:
err_msg = "Unable to find manifest for extension {0}".format(ext_handler_i.ext_handler.name)
ext_handler_i.set_operation(WALAEventOperation.Download)
ext_handler_i.set_handler_status(message=ustr(err_msg), code=-1)
ext_handler_i.report_event(message=ustr(err_msg), is_success=False)
return
time.sleep(2**retry_count * 10)
retry_count += 1
if retry_count != 0:
ext_handler_i.set_operation(WALAEventOperation.Download)
err_msg = "Able to find manifest for extension {0} after {1} failed attempts.".format(
ext_handler_i.ext_handler.name, retry_count)
ext_handler_i.report_event(message=ustr(err_msg))
self.get_artifact_error_state.reset()
if not ext_handler_i.is_upgrade and self.last_etag == etag:
if self.log_etag:
ext_handler_i.logger.verbose("Version {0} is current for etag {1}",
ext_handler_i.pkg.version,
etag)
self.log_etag = False
return
self.log_etag = True
ext_handler_i.logger.info("Target handler state: {0}", state)
if state == u"enabled":
self.handle_enable(ext_handler_i)
elif state == u"disabled":
self.handle_disable(ext_handler_i)
elif state == u"uninstall":
self.handle_uninstall(ext_handler_i)
else:
message = u"Unknown ext handler state:{0}".format(state)
raise ExtensionError(message)
except ExtensionError as e:
self.handle_handle_ext_handler_error(ext_handler_i, e, e.code)
except Exception as e:
self.handle_handle_ext_handler_error(ext_handler_i, e)
def handle_ext_handler(self, ext_handler, etag):
ext_handler_i = ExtHandlerInstance(ext_handler, self.protocol)
try:
state = ext_handler.properties.state
if ext_handler_i.decide_version(target_state=state) is None:
version = ext_handler_i.ext_handler.properties.version
name = ext_handler_i.ext_handler.name
err_msg = "Unable to find version {0} in manifest for extension {1}".format(
version, name)
ext_handler_i.set_operation(WALAEventOperation.Download)
ext_handler_i.set_handler_status(message=ustr(err_msg),
code=-1)
ext_handler_i.report_event(message=ustr(err_msg),
is_success=False)
return
self.get_artifact_error_state.reset()
if not ext_handler_i.is_upgrade and self.last_etag == etag:
if self.log_etag:
ext_handler_i.logger.verbose(
"Version {0} is current for etag {1}",
ext_handler_i.pkg.version, etag)
self.log_etag = False
return
self.log_etag = True
ext_handler_i.logger.info("Target handler state: {0}", state)
if state == u"enabled":
self.handle_enable(ext_handler_i)
elif state == u"disabled":
self.handle_disable(ext_handler_i)
elif state == u"uninstall":
self.handle_uninstall(ext_handler_i)
else:
message = u"Unknown ext handler state:{0}".format(state)
raise ExtensionError(message)
except ExtensionError as e:
self.handle_handle_ext_handler_error(ext_handler_i, e, e.code)
except Exception as e:
self.handle_handle_ext_handler_error(ext_handler_i, e)
def handle_ext_handler(self, ext_handler, etag):
ext_handler_i = ExtHandlerInstance(ext_handler, self.protocol)
ext_handler_i.decide_version()
if not ext_handler_i.is_upgrade and self.last_etag == etag:
ext_handler_i.logger.info("No upgrade detected for etag {0}", etag)
return
try:
state = ext_handler.properties.state
ext_handler_i.logger.info("Expected handler state: {0}", state)
if state == "enabled":
self.handle_enable(ext_handler_i)
elif state == u"disabled":
self.handle_disable(ext_handler_i)
elif state == u"uninstall":
self.handle_uninstall(ext_handler_i)
else:
message = u"Unknown ext handler state:{0}".format(state)
raise ExtensionError(message)
except ExtensionError as e:
ext_handler_i.set_handler_status(message=ustr(e), code=-1)
ext_handler_i.report_event(message=ustr(e), is_success=False)
def validate_in_range(val, valid_range, name):
if val not in valid_range:
raise ExtensionError("Invalid {0}: {1}".format(name, val))
def validate_has_key(obj, key, fullname):
if key not in obj:
raise ExtensionError("Missing: {0}".format(fullname))
def download(self):
begin_utc = datetime.datetime.utcnow()
self.logger.verbose("Download extension package")
self.set_operation(WALAEventOperation.Download)
if self.pkg is None:
raise ExtensionError("No package uri found")
package = None
for uri in self.pkg.uris:
try:
package = self.protocol.download_ext_handler_pkg(uri.uri)
if package is not None:
break
except Exception as e:
logger.warn("Error while downloading extension: {0}", e)
if package is None:
raise ExtensionError("Failed to download extension")
self.logger.verbose("Unpack extension package")
self.pkg_file = os.path.join(conf.get_lib_dir(),
os.path.basename(uri.uri) + ".zip")
try:
fileutil.write_file(self.pkg_file, bytearray(package), asbin=True)
zipfile.ZipFile(self.pkg_file).extractall(self.get_base_dir())
except IOError as e:
fileutil.clean_ioerror(e,
paths=[self.get_base_dir(), self.pkg_file])
raise ExtensionError(u"Failed to write and unzip plugin", e)
#Add user execute permission to all files under the base dir
for file in fileutil.get_all_files(self.get_base_dir()):
fileutil.chmod(file, os.stat(file).st_mode | stat.S_IXUSR)
duration = elapsed_milliseconds(begin_utc)
self.report_event(message="Download succeeded", duration=duration)
self.logger.info("Initialize extension directory")
#Save HandlerManifest.json
man_file = fileutil.search_file(self.get_base_dir(),
'HandlerManifest.json')
if man_file is None:
raise ExtensionError("HandlerManifest.json not found")
try:
man = fileutil.read_file(man_file, remove_bom=True)
fileutil.write_file(self.get_manifest_file(), man)
except IOError as e:
fileutil.clean_ioerror(e,
paths=[self.get_base_dir(), self.pkg_file])
raise ExtensionError(u"Failed to save HandlerManifest.json", e)
#Create status and config dir
try:
status_dir = self.get_status_dir()
fileutil.mkdir(status_dir, mode=0o700)
seq_no, status_path = self.get_status_file_path()
if seq_no > -1:
now = datetime.datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ")
status = {
"version": 1.0,
"timestampUTC": now,
"status": {
"name": self.ext_handler.name,
"operation": "Enabling Handler",
"status": "transitioning",
"code": 0
}
}
fileutil.write_file(json.dumps(status), status_path)
conf_dir = self.get_conf_dir()
fileutil.mkdir(conf_dir, mode=0o700)
except IOError as e:
fileutil.clean_ioerror(e,
paths=[self.get_base_dir(), self.pkg_file])
raise ExtensionError(u"Failed to create status or config dir", e)
#Save HandlerEnvironment.json
self.create_handler_env()
def decide_version(self, target_state=None):
self.logger.verbose("Decide which version to use")
try:
pkg_list = self.protocol.get_ext_handler_pkgs(self.ext_handler)
except ProtocolError as e:
raise ExtensionError("Failed to get ext handler pkgs", e)
# Determine the desired and installed versions
requested_version = FlexibleVersion(
str(self.ext_handler.properties.version))
installed_version_string = self.get_installed_version()
installed_version = requested_version \
if installed_version_string is None \
else FlexibleVersion(installed_version_string)
# Divide packages
# - Find the installed package (its version must exactly match)
# - Find the internal candidate (its version must exactly match)
# - Separate the public packages
internal_pkg = None
installed_pkg = None
public_pkgs = []
for pkg in pkg_list.versions:
pkg_version = FlexibleVersion(pkg.version)
if pkg_version == installed_version:
installed_pkg = pkg
if pkg.isinternal and pkg_version == requested_version:
internal_pkg = pkg
if not pkg.isinternal:
public_pkgs.append(pkg)
internal_version = FlexibleVersion(internal_pkg.version) \
if internal_pkg is not None \
else FlexibleVersion()
public_pkgs.sort(key=lambda pkg: FlexibleVersion(pkg.version),
reverse=True)
# Determine the preferred version and type of upgrade occurring
preferred_version = max(requested_version, installed_version)
is_major_upgrade = preferred_version.major > installed_version.major
allow_minor_upgrade = self.ext_handler.properties.upgradePolicy == 'auto'
# Find the first public candidate which
# - Matches the preferred major version
# - Does not upgrade to a new, disallowed major version
# - And only increments the minor version if allowed
# Notes:
# - The patch / hotfix version is not considered
public_pkg = None
for pkg in public_pkgs:
pkg_version = FlexibleVersion(pkg.version)
if pkg_version.major == preferred_version.major \
and (not pkg.disallow_major_upgrade or not is_major_upgrade) \
and (allow_minor_upgrade or pkg_version.minor == preferred_version.minor):
public_pkg = pkg
break
# If there are no candidates, locate the highest public version whose
# major matches that installed
if internal_pkg is None and public_pkg is None:
for pkg in public_pkgs:
pkg_version = FlexibleVersion(pkg.version)
if pkg_version.major == installed_version.major:
public_pkg = pkg
break
public_version = FlexibleVersion(public_pkg.version) \
if public_pkg is not None \
else FlexibleVersion()
# Select the candidate
# - Use the public candidate if there is no internal candidate or
# the public is more recent (e.g., a hotfix patch)
# - Otherwise use the internal candidate
if internal_pkg is None or (public_pkg is not None
and public_version > internal_version):
selected_pkg = public_pkg
else:
selected_pkg = internal_pkg
selected_version = FlexibleVersion(selected_pkg.version) \
if selected_pkg is not None \
else FlexibleVersion()
# Finally, update the version only if not downgrading
# Note:
# - A downgrade, which will be bound to the same major version,
# is allowed if the installed version is no longer available
if target_state == u"uninstall":
if installed_pkg is None:
msg = "Failed to find installed version of {0} " \
"to uninstall".format(self.ext_handler.name)
self.logger.warn(msg)
self.pkg = installed_pkg
self.ext_handler.properties.version = str(installed_version) \
if installed_version is not None else None
elif selected_pkg is None \
or (installed_pkg is not None and selected_version < installed_version):
self.pkg = installed_pkg
self.ext_handler.properties.version = str(installed_version) \
if installed_version is not None else None
else:
self.pkg = selected_pkg
self.ext_handler.properties.version = str(selected_pkg.version)
# Note if the selected package is greater than that installed
if installed_pkg is None \
or FlexibleVersion(self.pkg.version) > FlexibleVersion(installed_pkg.version):
self.is_upgrade = True
if self.pkg is None:
raise ExtensionError("Failed to find any valid extension package")
self.logger.verbose("Use version: {0}", self.pkg.version)
return
