def validate_has_key(obj, key, fullname):
if key not in obj:
raise ExtensionError("Missing: {0}".format(fullname))
def launch_command(self,
cmd,
timeout=300,
extension_error_code=1000,
env=None):
begin_utc = datetime.datetime.utcnow()
self.logger.verbose("Launch command: [{0}]", cmd)
base_dir = self.get_base_dir()
if env is None:
env = {}
env.update(os.environ)
try:
# This should be .run(), but due to the wide variety
# of Python versions we must support we must use .communicate().
# Some extensions erroneously begin cmd with a slash; don't interpret those
# as root-relative. (Issue #1170)
full_path = os.path.join(base_dir, cmd.lstrip(os.path.sep))
def pre_exec_function():
"""
Change process state before the actual target process is started. Effectively, this runs between
the fork() and the exec() of sub-process creation.
:return:
"""
os.setsid()
CGroups.add_to_extension_cgroup(self.ext_handler.name)
process = subprocess.Popen(full_path,
shell=True,
cwd=base_dir,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env=env,
preexec_fn=pre_exec_function)
except OSError as e:
raise ExtensionError("Failed to launch '{0}': {1}".format(
full_path, e.strerror),
code=extension_error_code)
cg = CGroups.for_extension(self.ext_handler.name)
CGroupsTelemetry.track_extension(self.ext_handler.name, cg)
msg = capture_from_process(process, cmd, timeout, extension_error_code)
ret = process.poll()
if ret is None:
raise ExtensionError(
"Process {0} was not terminated: {1}\n{2}".format(
process.pid, cmd, msg),
code=extension_error_code)
if ret != 0:
raise ExtensionError("Non-zero exit code: {0}, {1}\n{2}".format(
ret, cmd, msg),
code=extension_error_code)
duration = elapsed_milliseconds(begin_utc)
log_msg = "{0}\n{1}".format(
cmd, "\n".join([line for line in msg.split('\n') if line != ""]))
self.logger.verbose(log_msg)
self.report_event(message=log_msg, duration=duration, log_event=False)
def decide_version(self):
self.logger.verbose("Decide which version to use")
try:
pkg_list = self.protocol.get_ext_handler_pkgs(self.ext_handler)
except ProtocolError as e:
raise ExtensionError("Failed to get ext handler pkgs", e)
# Determine the desired and installed versions
requested_version = FlexibleVersion(
self.ext_handler.properties.version)
installed_version = FlexibleVersion(self.get_installed_version())
if installed_version is None:
installed_version = requested_version
# Divide packages
# - Find the installed package (its version must exactly match)
# - Find the internal candidate (its version must exactly match)
# - Separate the public packages
internal_pkg = None
installed_pkg = None
public_pkgs = []
for pkg in pkg_list.versions:
pkg_version = FlexibleVersion(pkg.version)
if pkg_version == installed_version:
installed_pkg = pkg
if pkg.isinternal and pkg_version == requested_version:
internal_pkg = pkg
if not pkg.isinternal:
public_pkgs.append(pkg)
internal_version = FlexibleVersion(internal_pkg.version) \
if internal_pkg is not None \
else FlexibleVersion()
public_pkgs.sort(key=lambda pkg: FlexibleVersion(pkg.version),
reverse=True)
# Determine the preferred version and type of upgrade occurring
preferred_version = max(requested_version, installed_version)
is_major_upgrade = preferred_version.major > installed_version.major
allow_minor_upgrade = self.ext_handler.properties.upgradePolicy == 'auto'
# Find the first public candidate which
# - Matches the preferred major version
# - Does not upgrade to a new, disallowed major version
# - And only increments the minor version if allowed
# Notes:
# - The patch / hotfix version is not considered
public_pkg = None
for pkg in public_pkgs:
pkg_version = FlexibleVersion(pkg.version)
if pkg_version.major == preferred_version.major \
and (not pkg.disallow_major_upgrade or not is_major_upgrade) \
and (allow_minor_upgrade or pkg_version.minor == preferred_version.minor):
public_pkg = pkg
break
# If there are no candidates, locate the highest public version whose
# major matches that installed
if internal_pkg is None and public_pkg is None:
for pkg in public_pkgs:
pkg_version = FlexibleVersion(pkg.version)
if pkg_version.major == installed_version.major:
public_pkg = pkg
break
public_version = FlexibleVersion(public_pkg.version) \
if public_pkg is not None \
else FlexibleVersion()
# Select the candidate
# - Use the public candidate if there is no internal candidate or
# the public is more recent (e.g., a hotfix patch)
# - Otherwise use the internal candidate
if internal_pkg is None or (public_pkg is not None
and public_version > internal_version):
selected_pkg = public_pkg
else:
selected_pkg = internal_pkg
selected_version = FlexibleVersion(selected_pkg.version) \
if selected_pkg is not None \
else FlexibleVersion()
# Finally, update the version only if not downgrading
# Note:
# - A downgrade, which will be bound to the same major version,
# is allowed if the installed version is no longer available
if selected_pkg is None \
or (installed_pkg is not None and selected_version < installed_version):
self.pkg = installed_pkg
self.ext_handler.properties.version = installed_version
else:
self.pkg = selected_pkg
self.ext_handler.properties.version = selected_pkg.version
# Note if the selected package is greater than that installed
if installed_pkg is None \
or FlexibleVersion(self.pkg.version) > FlexibleVersion(installed_pkg.version):
self.is_upgrade = True
if self.pkg is None:
raise ExtensionError("Failed to find any valid extension package")
self.logger.verbose("Use version: {0}", self.pkg.version)
return
def validate_in_range(val, valid_range, name):
if val not in valid_range:
raise ExtensionError("Invalid {0}: {1}".format(name, val))
def download(self):
begin_utc = datetime.datetime.utcnow()
self.logger.verbose("Download extension package")
self.set_operation(WALAEventOperation.Download)
if self.pkg is None:
raise ExtensionError("No package uri found")
uris_shuffled = self.pkg.uris
random.shuffle(uris_shuffled)
file_downloaded = False
for uri in uris_shuffled:
try:
destination = os.path.join(conf.get_lib_dir(),
os.path.basename(uri.uri) + ".zip")
file_downloaded = self.protocol.download_ext_handler_pkg(
uri.uri, destination)
if file_downloaded and os.path.exists(destination):
self.pkg_file = destination
break
except Exception as e:
logger.warn("Error while downloading extension: {0}", ustr(e))
if not file_downloaded:
raise ExtensionError("Failed to download extension", code=1001)
self.logger.verbose("Unzip extension package")
try:
zipfile.ZipFile(self.pkg_file).extractall(self.get_base_dir())
os.remove(self.pkg_file)
except IOError as e:
fileutil.clean_ioerror(e,
paths=[self.get_base_dir(), self.pkg_file])
raise ExtensionError(u"Failed to unzip extension package",
e,
code=1001)
# Add user execute permission to all files under the base dir
for file in fileutil.get_all_files(self.get_base_dir()):
fileutil.chmod(file, os.stat(file).st_mode | stat.S_IXUSR)
duration = elapsed_milliseconds(begin_utc)
self.report_event(message="Download succeeded", duration=duration)
self.logger.info("Initialize extension directory")
# Save HandlerManifest.json
man_file = fileutil.search_file(self.get_base_dir(),
'HandlerManifest.json')
if man_file is None:
raise ExtensionError("HandlerManifest.json not found")
try:
man = fileutil.read_file(man_file, remove_bom=True)
fileutil.write_file(self.get_manifest_file(), man)
except IOError as e:
fileutil.clean_ioerror(e,
paths=[self.get_base_dir(), self.pkg_file])
raise ExtensionError(u"Failed to save HandlerManifest.json", e)
# Create status and config dir
try:
status_dir = self.get_status_dir()
fileutil.mkdir(status_dir, mode=0o700)
seq_no, status_path = self.get_status_file_path()
if status_path is not None:
now = datetime.datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ")
status = {
"version": 1.0,
"timestampUTC": now,
"status": {
"name": self.ext_handler.name,
"operation": "Enabling Handler",
"status": "transitioning",
"code": 0
}
}
fileutil.write_file(status_path, json.dumps(status))
conf_dir = self.get_conf_dir()
fileutil.mkdir(conf_dir, mode=0o700)
except IOError as e:
fileutil.clean_ioerror(e,
paths=[self.get_base_dir(), self.pkg_file])
raise ExtensionError(u"Failed to create status or config dir", e)
# Save HandlerEnvironment.json
self.create_handler_env()
def __init__(self, data):
if data is None or data['handlerManifest'] is None:
raise ExtensionError('Malformed manifest file.')
self.data = data
def update_settings_file(self, settings_file, settings):
settings_file = os.path.join(self.get_conf_dir(), settings_file)
try:
fileutil.write_file(settings_file, settings)
except IOError as e:
raise ExtensionError(u"Failed to update settings file", e)
def handle_ext_handler(self, ext_handler, etag):
ext_handler_i = ExtHandlerInstance(ext_handler, self.protocol)
try:
state = ext_handler.properties.state
# The extension is to be enabled, there is an upgrade GUID
# and the GUID is NOT new
if state == u"enabled" and \
ext_handler.properties.upgradeGuid is not None and \
not self.is_new_guid(ext_handler):
ext_handler_i.ext_handler.properties.version = ext_handler_i.get_installed_version()
ext_handler_i.set_logger()
if self.last_etag != etag:
self.set_log_upgrade_guid(ext_handler, True)
msg = "New GUID is the same as the old GUID. Exiting without upgrading."
if self.get_log_upgrade_guid(ext_handler):
ext_handler_i.logger.info(msg)
self.set_log_upgrade_guid(ext_handler, False)
ext_handler_i.set_handler_state(ExtHandlerState.Enabled)
ext_handler_i.set_handler_status(status="Ready", message="No change")
ext_handler_i.set_operation(WALAEventOperation.SkipUpdate)
ext_handler_i.report_event(message=ustr(msg), is_success=True)
return
self.set_log_upgrade_guid(ext_handler, True)
ext_handler_i.decide_version(target_state=state)
self.get_artifact_error_state.reset()
if not ext_handler_i.is_upgrade and self.last_etag == etag:
if self.log_etag:
ext_handler_i.logger.verbose("Version {0} is current for etag {1}",
ext_handler_i.pkg.version,
etag)
self.log_etag = False
return
self.log_etag = True
ext_handler_i.logger.info("Target handler state: {0}", state)
if state == u"enabled":
self.handle_enable(ext_handler_i)
if ext_handler.properties.upgradeGuid is not None:
ext_handler_i.logger.info("New Upgrade GUID: {0}", ext_handler.properties.upgradeGuid)
self.last_upgrade_guids[ext_handler.name] = (ext_handler.properties.upgradeGuid, True)
elif state == u"disabled":
self.handle_disable(ext_handler_i)
# Remove the GUID from the dictionary so that it is upgraded upon re-enable
self.last_upgrade_guids.pop(ext_handler.name, None)
elif state == u"uninstall":
self.handle_uninstall(ext_handler_i)
# Remove the GUID from the dictionary so that it is upgraded upon re-install
self.last_upgrade_guids.pop(ext_handler.name, None)
else:
message = u"Unknown ext handler state:{0}".format(state)
raise ExtensionError(message)
except Exception as e:
msg = ustr(e)
ext_handler_i.set_handler_status(message=msg, code=-1)
self.get_artifact_error_state.incr()
if self.get_artifact_error_state.is_triggered():
report_event(op=WALAEventOperation.GetArtifactExtended,
message="Failed to get artifact for over "
"{0}: {1}".format(self.get_artifact_error_state.min_timedelta, msg),
is_success=False)
self.get_artifact_error_state.reset()
else:
ext_handler_i.logger.warn(msg)
