def check_required_modules(self, additional_required_modules=None): """ :param additional_required_modules: A list of 5-tuples (module_name, min_version, min_inclusive, max_version, max_inclusive) that specifies additional required modules to test. The list may be None. All tuple entries but module_name may be None. """ # Python standard modules. # TODO: Does it really make sense to test those?! if EnvironmentHelper.is_python3(): self.check_module('io') else: self.check_module('StringIO') self.check_module('datetime', class_names=['datetime']) self.check_module('math') self.check_module('socket') self.check_module('struct') self.check_module('base64') self.check_module('traceback') self.check_module('os') self.check_module('pickle') self.check_module('imp') self.check_module('types') # Non-standard modules. self.check_module('numpy') global _default_min_pandas_version min_pandas_version = _default_min_pandas_version self.check_module('pandas', min_version=min_pandas_version, class_names=['DataFrame']) # Additional modules. if additional_required_modules is not None: self.check_additional_modules(additional_required_modules)
def _get_extension_by_index(index, extensions): """ Get the path to a module at position index in the passed lists of paths (extensions) and load it. Return the loaded module. @param index a position in the extensions array @param extensions a list of paths to python modules (usually self._serializers or self._deserializers) @return the module loaded from the specified path """ if index >= len(extensions): return None type_extension = extensions[index] if not isinstance(type_extension, types.ModuleType): path = type_extension last_separator = path.rfind(os.sep) file_extension_start = path.rfind('.') module_name = path[last_separator + 1:file_extension_start] try: if EnvironmentHelper.is_python3(): type_extension = importlib.machinery.SourceFileLoader( module_name, path).load_module() else: type_extension = imp.load_source(module_name, path) except ImportError as error: raise ImportError( 'Error while loading python type extension ' + module_name + '\nCause: ' + str(error)) extensions[index] = type_extension return type_extension
def is_nat(value): """ Checks if the given value is NaT. """ if EnvironmentHelper.is_tslib_available(): return value is NaT else: return False
def _respond(self, request, response_message_id, workspace): name = PayloadDecoder(request.payload).get_next_string() image = workspace.get_variable_or_default(name, None) if EnvironmentHelper.is_python3(): if type(image) is bytes: data_bytes = image else: data_bytes = bytearray() else: if type(image) is str: data_bytes = image else: data_bytes = '' return AbstractRequestHandler._create_response(request, response_message_id, response_payload=_create_byte_array_payload(data_bytes))
def _standardize_input(notebook_directory, notebook_name, notebook_version, only_include_tag): if notebook_directory is None: raise ValueError("Notebook directory must not be None.") if notebook_name is None: raise ValueError("Notebook name must not be None.") notebook_directory = str(notebook_directory) notebook_name = str(notebook_name) notebook_path = None if notebook_directory.startswith("knime:"): exception = None try: notebook_path = posixpath.join(notebook_directory, notebook_name) notebook_path = _resolve_knime_url(notebook_path) except Exception as ex: exception = ex if exception is not None: # Raise exception outside of the original catch block to avoid polluting the console with the full # traceback. raise ValueError(str(exception)) if not EnvironmentHelper.is_python3(): # Java always returns a unicode string, Python 2 wants non-unicode. notebook_path = str(notebook_path) else: notebook_path = os.path.join(notebook_directory, notebook_name) if not os.path.isfile(notebook_path): raise ValueError("Notebook path '" + notebook_path + "' does not point to an existing file.") if notebook_version is None: notebook_version = NO_CONVERT else: try: notebook_version = int(notebook_version) except ValueError: notebook_version = None # Raise exception outside of the original catch block to avoid polluting the console with the full traceback. if notebook_version is None: raise ValueError("Notebook version must be an integer or castable to an integer.") if only_include_tag is not None: only_include_tag = str(only_include_tag) return notebook_path, notebook_version, only_include_tag
def load_module_from_path(path): """ Load a python module from a source file. @param path the path to the source file (string) @return the module loaded from the specified path """ last_separator = path.rfind(os.sep) file_extension_start = path.rfind('.') module_name = path[last_separator + 1:file_extension_start] try: if EnvironmentHelper.is_python3(): loaded_module = importlib.machinery.SourceFileLoader( module_name, path).load_module() else: loaded_module = imp.load_source(module_name, path) except ImportError as error: raise ImportError('Error while loading python module ' + module_name + '\nCause: ' + str(error)) return loaded_module
def object_to_string(data_object): """ Convert data_object to a (possibly truncated) string representation. """ if EnvironmentHelper.is_python3(): try: object_as_string = str(data_object) except Exception: return '' else: try: object_as_string = unicode(data_object) except UnicodeDecodeError: object_as_string = '(base64 encoded)\n' + base64.b64encode( data_object) except Exception: return '' return (object_as_string[:996] + '\n...') if len(object_as_string) > 1000 else object_as_string
def serialize(object_value): if EnvironmentHelper.is_tslib_available( ) and type(object_value) is EnvironmentHelper.Timestamp: object_value = object_value.to_pydatetime() # format: yyyy-mm-dd HH:MM:SS (+microseconds and offset to UTC if available) datestr = object_value.isoformat(' ') # Add timezone name (offset already included in isoformat) if object_value.tzname(): datestr += '[' + object_value.tzname() + ']' # Find +/- sign of timezone offset tzidx = max(datestr.find('+'), datestr.find('-', 10)) if tzidx < 0: tzidx = len(datestr) # If microseconds available cut down to millis otherwise add millis if object_value.microsecond > 0: datestr = datestr[:tzidx - 3] + datestr[tzidx:] else: datestr = datestr[:tzidx] + '.000' + datestr[tzidx:] return datestr.encode('utf-8')
def list_variables(self): """ List all currently loaded modules and defined classes, functions and variables. """ # create lists of modules, classes, functions and variables modules = [] classes = [] functions = [] variables = [] # iterate over dictionary to and put modules, classes, functions and variables in their respective lists for key, value in dict(self._exec_env).items(): # get name of the type var_type = type(value).__name__ # class type changed from classobj to type in python 3 class_type = 'classobj' if EnvironmentHelper.is_python3(): class_type = 'type' if var_type == 'module': modules.append({'name': key, 'type': var_type, 'value': ''}) elif var_type == class_type: classes.append({'name': key, 'type': var_type, 'value': ''}) elif var_type == 'function': functions.append({'name': key, 'type': var_type, 'value': ''}) elif key != '__builtins__': value = object_to_string(value) variables.append({ 'name': key, 'type': var_type, 'value': value }) # sort lists by name modules = sorted(modules, key=lambda k: k['name']) classes = sorted(classes, key=lambda k: k['name']) functions = sorted(functions, key=lambda k: k['name']) variables = sorted(variables, key=lambda k: k['name']) # create response list and add contents of the other lists in the order they should be displayed response = [] response.extend(modules) response.extend(classes) response.extend(functions) response.extend(variables) return response
# prepare and propagate Nodes, in each case even if such Nodes are # propagated with or for interoperation with KNIME. The owner of a Node # may freely choose the license terms applicable to such Node, including # when such Node is propagated with or for interoperation with KNIME. # ------------------------------------------------------------------------ """ @author Clemens von Schwerin, KNIME GmbH, Konstanz, Germany @author Patrick Winter, KNIME GmbH, Konstanz, Germany @author Marcel Wiedenmann, KNIME GmbH, Konstanz, Germany @author Christian Dietz, KNIME GmbH, Konstanz, Germany """ # This should be the first statement in each module that makes specific demands on the Python environment. import EnvironmentHelper if EnvironmentHelper.is_python3(): import importlib else: import imp if EnvironmentHelper.is_tslib_available(): from EnvironmentHelper import Timestamp from EnvironmentHelper import NaT else: Timestamp = None NaT = None import base64 import inspect import math import numpy
# ------------------------------------------------------------------------ """ @author Clemens von Schwerin, KNIME GmbH, Konstanz, Germany @author Patrick Winter, KNIME GmbH, Konstanz, Germany @author Marcel Wiedenmann, KNIME GmbH, Konstanz, Germany @author Christian Dietz, KNIME GmbH, Konstanz, Germany """ from __future__ import print_function # This should be the first statement in each module (except for __future__ statements) that makes specific demands on # the Python environment. import EnvironmentHelper EnvironmentHelper.dummy_call() import sys import traceback # Debugging: # Uncomment to enable debugging. You may want to disable breakpoints since they require an external debugger. See # debug_util module for information on how to set up a debug environment. # import debug_util # debug_util.init_debug(enable_breakpoints=True, enable_debug_log=True, debug_log_to_stderr=False) # Start Python kernel: if __name__ == "__main__": if EnvironmentHelper.is_python3():
# propagated with or for interoperation with KNIME. The owner of a Node # may freely choose the license terms applicable to such Node, including # when such Node is propagated with or for interoperation with KNIME. # ------------------------------------------------------------------------ """ @author Clemens von Schwerin, KNIME GmbH, Konstanz, Germany @author Patrick Winter, KNIME GmbH, Konstanz, Germany @author Marcel Wiedenmann, KNIME GmbH, Konstanz, Germany @author Christian Dietz, KNIME GmbH, Konstanz, Germany """ # This should be the first statement in each module that makes specific demands on the Python environment. import EnvironmentHelper # Make sure the import stays at the top (when auto-formatting). EnvironmentHelper.dummy_call() import abc import collections import os import pickle import sys import pandas import PythonUtils import traceback from debug_util import debug_msg from DBUtil import DBUtil
def has_auto_complete(): """ Returns true if autocomplete is available, false otherwise. """ return EnvironmentHelper.is_jedi_available()
# may freely choose the license terms applicable to such Node, including # when such Node is propagated with or for interoperation with KNIME. # ------------------------------------------------------------------------ """ @author Clemens von Schwerin, KNIME GmbH, Konstanz, Germany @author Patrick Winter, KNIME GmbH, Konstanz, Germany @author Marcel Wiedenmann, KNIME GmbH, Konstanz, Germany @author Christian Dietz, KNIME GmbH, Konstanz, Germany """ # This should be the first statement in each module (except for __future__ statements) that makes specific demands on # the Python environment. import EnvironmentHelper if EnvironmentHelper.is_python3(): from io import StringIO else: from StringIO import StringIO import abc import os import socket import sys import traceback import warnings from debug_util import debug_msg from Borg import Borg from messaging import RequestHandlers
# prepare and propagate Nodes, in each case even if such Nodes are # propagated with or for interoperation with KNIME. The owner of a Node # may freely choose the license terms applicable to such Node, including # when such Node is propagated with or for interoperation with KNIME. # ------------------------------------------------------------------------ """ @author Clemens von Schwerin, KNIME GmbH, Konstanz, Germany @author Patrick Winter, KNIME GmbH, Konstanz, Germany @author Marcel Wiedenmann, KNIME GmbH, Konstanz, Germany @author Christian Dietz, KNIME GmbH, Konstanz, Germany """ # This should be the first statement in each module that makes specific demands on the Python environment. import EnvironmentHelper if EnvironmentHelper.is_python3(): import importlib else: import imp import os import types from Borg import Borg class TypeExtensionManager(Borg): """ Used for managing all registered serializers and deserializers. Serializers and deserializers can be accessed using the identifier, which is the id of the java extension point or the type_string corresponding to the python type. This type string is set in the extension point's specification in