def getByID(id): """ This functions retrieves an object from an external storage technology from the obj object. This dummy returns the same object as submited by the parameter obj. @param obj: key/object of the object to be retrieved. @return: the real object. """ #print "-----------------------------------------------------" #print "| WARNING!!! - YOU ARE USING THE DUMMY STORAGE API. |" #print "| Call to: getByID |" #print "| ********************************************* |" #print "| *** Check that you really want to use the *** |" #print "| ************* dummy storage api ************* |" #print "| ********************************************* |" #print "-----------------------------------------------------" if id is not None: try: file_name = id + '.PSCO' file_path = storage_path + file_name obj = deserialize_from_file(file_path) obj.setID(id) return obj except ValueError: # The id does not complain uuid4 --> raise an exception print( "Error: the ID for getByID does not complain the uuid4 format." ) raise ValueError( 'Using the dummy storage API getByID with wrong id.') else: # Using a None id --> raise an exception print("Error: the ID for getByID is None.") raise ValueError('Using the dummy storage API getByID with None id.')
def __init__(self, tf_depth, tf_base_sleep_time, tf_sleep_random_range, file_elements): """ Initializes the command line arguments instance with the given values. """ from pycompss.util.serializer import deserialize_from_file self.elements = [] for fe in file_elements: elem = deserialize_from_file(fe) self.elements.append(elem) self.tf_depth = int(tf_depth) self.tf_base_sleep_time = int(tf_base_sleep_time) self.tf_sleep_random_range = int(tf_sleep_random_range)
def execute_task(process_name, storage_conf, params): """ ExecuteTask main method """ logger = logging.getLogger('pycompss.worker.worker') if __debug__: logger.debug("[PYTHON WORKER %s] Begin task execution" % process_name) persistent_storage = False if storage_conf != 'null': persistent_storage = True from pycompss.util.persistent_storage import storage_task_context # COMPSs keywords for tasks (ie: tracing, process name...) compss_kwargs = { 'compss_tracing': tracing, 'compss_process_name': process_name, 'compss_storage_conf': storage_conf } # Retrieve the parameters from the params argument path = params[0] method_name = params[1] numSlaves = int(params[2]) slaves = [] for i in range(2, 2 + numSlaves): slaves.append(params[i]) argPosition = 3 + numSlaves args = params[argPosition:] cus = args[0] args = args[1:] has_target = args[0] return_type = args[1] num_params = int(args[2]) args = args[3:] if __debug__: logger.debug("[PYTHON WORKER %s] Storage conf: %s" % (str(process_name), str(storage_conf))) logger.debug("[PYTHON WORKER %s] Params: %s" % (str(process_name), str(params))) logger.debug("[PYTHON WORKER %s] Path: %s" % (str(process_name), str(path))) logger.debug("[PYTHON WORKER %s] Method name: %s" % (str(process_name), str(method_name))) logger.debug("[PYTHON WORKER %s] Num slaves: %s" % (str(process_name), str(numSlaves))) logger.debug("[PYTHON WORKER %s] Slaves: %s" % (str(process_name), str(slaves))) logger.debug("[PYTHON WORKER %s] Cus: %s" % (str(process_name), str(cus))) logger.debug("[PYTHON WORKER %s] Has target: %s" % (str(process_name), str(has_target))) logger.debug("[PYTHON WORKER %s] Num Params: %s" % (str(process_name), str(num_params))) logger.debug("[PYTHON WORKER %s] Args: %r" % (str(process_name), args)) # if tracing: # pyextrae.event(TASK_EVENTS, 0) # pyextrae.event(TASK_EVENTS, PARAMETER_PROCESSING) # Get all parameter values if __debug__: logger.debug("[PYTHON WORKER %s] Processing parameters:" % process_name) values, types, streams, prefixes = get_input_params(num_params, logger, args, process_name, persistent_storage) # if tracing: # pyextrae.event(TASK_EVENTS, 0) # pyextrae.event(TASK_EVENTS, LOGGING) if __debug__: logger.debug("[PYTHON WORKER %s] RUN TASK with arguments: " % process_name) logger.debug("[PYTHON WORKER %s] \t- Path: %s" % (process_name, path)) logger.debug("[PYTHON WORKER %s] \t- Method/function name: %s" % (process_name, method_name)) logger.debug("[PYTHON WORKER %s] \t- Has target: %s" % (process_name, str(has_target))) logger.debug("[PYTHON WORKER %s] \t- # parameters: %s" % (process_name, str(num_params))) logger.debug("[PYTHON WORKER %s] \t- Values:" % process_name) for v in values: logger.debug("[PYTHON WORKER %s] \t\t %r" % (process_name, v)) logger.debug("[PYTHON WORKER %s] \t- COMPSs types:" % process_name) for t in types: logger.debug("[PYTHON WORKER %s] \t\t %s" % (process_name, str(t))) # if tracing: # pyextrae.event(TASK_EVENTS, 0) # pyextrae.event(TASK_EVENTS, MODULES_IMPORT) import_error = False newTypes = [] newValues = [] try: # Try to import the module (for functions) if __debug__: logger.debug("[PYTHON WORKER %s] Trying to import the user module: %s" % (process_name, path)) if sys.version_info >= (2, 7): import importlib module = importlib.import_module(path) # Python 2.7 if __debug__: logger.debug("[PYTHON WORKER %s] Module successfully loaded (Python version >= 2.7)" % process_name) else: module = __import__(path, globals(), locals(), [path], -1) if __debug__: logger.debug("[PYTHON WORKER %s] Module successfully loaded (Python version < 2.7" % process_name) def task_execution_1(): return task_execution(logger, process_name, module, method_name, types, values, compss_kwargs) if persistent_storage: with storage_task_context(logger, values, config_file_path=storage_conf): newTypes, newValues = task_execution_1() else: newTypes, newValues = task_execution_1() # ========================================================================== except AttributeError: # Appears with functions that have not been well defined. exc_type, exc_value, exc_traceback = sys.exc_info() lines = traceback.format_exception(exc_type, exc_value, exc_traceback) logger.exception("[PYTHON WORKER %s] WORKER EXCEPTION - Attribute Error Exception" % process_name) logger.exception(''.join(line for line in lines)) logger.exception("[PYTHON WORKER %s] Check that all parameters have been defined with an absolute import path (even if in the same file)" % process_name) # If exception is raised during the task execution, newTypes and # newValues are empty return 1, newTypes, newValues # ========================================================================== except ImportError: import_error = True # ========================================================================== except Exception: exc_type, exc_value, exc_traceback = sys.exc_info() lines = traceback.format_exception(exc_type, exc_value, exc_traceback) logger.exception("[PYTHON WORKER %s] WORKER EXCEPTION" % process_name) logger.exception(''.join(line for line in lines)) # If exception is raised during the task execution, newTypes and # newValues are empty return 1, newTypes, newValues if import_error: if __debug__: logger.debug("[PYTHON WORKER %s] Could not import the module. Reason: Method in class." % process_name) # Not the path of a module, it ends with a class name class_name = path.split('.')[-1] module_name = '.'.join(path.split('.')[0:-1]) if '.' in path: module_name = '.'.join(path.split('.')[0:-1]) else: module_name = path module = __import__(module_name, fromlist=[class_name]) klass = getattr(module, class_name) if __debug__: logger.debug("[PYTHON WORKER %s] Method in class %s of module %s" % (process_name, class_name, module_name)) if has_target == 'true': # Instance method last_elem = values.pop() if is_PSCO(last_elem): obj = last_elem else: file_name = last_elem.split(':')[-1] if __debug__: logger.debug("[PYTHON WORKER %s] Deserialize self from file." % process_name) obj = deserialize_from_file(file_name) if __debug__: logger.debug("[PYTHON WORKER %s] Processing callee, a hidden object of %s in file %s" % (process_name, file_name, type(obj))) values.insert(0, obj) types.pop() types.insert(0, TYPE.OBJECT if not is_PSCO(last_elem) else TYPE.EXTERNAL_PSCO) def task_execution_2(): return task_execution(logger, process_name, klass, method_name, types, values, compss_kwargs) if persistent_storage: with storage_task_context(logger, values, config_file_path=storage_conf): newTypes, newValues = task_execution_2() else: newTypes, newValues = task_execution_2() if is_PSCO(last_elem): # There is no update PSCO on the storage API. Consequently, the changes on the PSCO must have been # pushed into the storage automatically on each PSCO modification. if __debug__: logger.debug("[PYTHON WORKER %s] The changes on the PSCO must have been automatically updated by the storage." % process_name) pass else: if __debug__: logger.debug("[PYTHON WORKER %s] Serializing self to file." % process_name) serialize_to_file(obj, file_name) if __debug__: logger.debug("[PYTHON WORKER %s] Obj: %r" % (process_name, obj)) else: # Class method - class is not included in values (e.g. values = [7]) types.insert(0, None) # class must be first type def task_execution_3(): return task_execution(logger, process_name, klass, method_name, types, values, compss_kwargs) if persistent_storage: with storage_task_context(logger, values, config_file_path=storage_conf): newTypes, newValues = task_execution_3() else: newTypes, newValues = task_execution_3() # EVERYTHING OK if __debug__: logger.debug("[PYTHON WORKER %s] End task execution. Status: Ok" % process_name) # if tracing: # pyextrae.eventandcounters(TASK_EVENTS, 0) return 0, newTypes, newValues # Exit code, updated params
def compss_worker(persistent_storage): """ Worker main method (invocated from __main__). :param persistent_storage: Persistent storage boolean :return: None """ logger = logging.getLogger('pycompss.worker.worker') logger.debug("Starting Worker") # Set the binding in worker mode set_pycompss_context('WORKER') args = sys.argv[6:] path = args[0] method_name = args[1] num_slaves = int(args[2]) slaves = [] for i in range(2, 2 + num_slaves): slaves.append(args[i]) arg_position = 3 + num_slaves args = args[arg_position:] cus = args[0] args = args[1:] has_target = args[0] return_type = args[1] num_returns = int(args[2]) num_params = int(args[3]) args = args[4:] # COMPSs keywords for tasks (ie: tracing, process name...) compss_kwargs = { 'compss_tracing': tracing, 'compss_process_name': "GAT", 'compss_storage_conf': storage_conf, 'compss_return_length': num_returns } values = [] types = [] streams = [] prefixes = [] if tracing: pyextrae.event(TASK_EVENTS, 0) pyextrae.event(TASK_EVENTS, PARAMETER_PROCESSING) if persistent_storage: from pycompss.util.persistent_storage import storage_task_context # Get all parameter values logger.debug("Processing parameters:") pos = 0 for i in range(0, num_params): p_type = int(args[pos]) p_stream = int(args[pos + 1]) p_prefix = args[pos + 2] p_value = args[pos + 3] logger.debug("Parameter : " + str(i)) logger.debug("\t * Type : " + str(p_type)) logger.debug("\t * Stream : " + str(p_stream)) logger.debug("\t * Prefix : " + str(p_prefix)) logger.debug("\t * Value: " + str(p_value)) types.append(p_type) streams.append(p_stream) prefixes.append(p_prefix) if p_type == TYPE.FILE: values.append(p_value) elif p_type == TYPE.EXTERNAL_PSCO: po = get_by_id(p_value) values.append(po) pos += 1 # Skip info about direction (R, W) elif p_type == TYPE.STRING: num_substrings = int(p_value) aux = '' first_substring = True for j in range(4, num_substrings + 4): if not first_substring: aux += ' ' first_substring = False aux += args[pos + j] # Decode the string received aux = base64.b64decode(aux.encode()) if aux.decode() == EMPTY_STRING_KEY: # Then it is an empty string aux = "" else: ####### # Check if the string is really an object # Required in order to recover objects passed as parameters. # - Option object_conversion real_value = aux try: # try to recover the real object if IS_PYTHON3: # decode removes double backslash, and encode returns as binary aux = deserialize_from_string(aux.decode(STR_ESCAPE).encode()) else: # decode removes double backslash, and str casts the output aux = deserialize_from_string(str(aux.decode(STR_ESCAPE))) except (SerializerException, ValueError, EOFError): # was not an object aux = str(real_value.decode()) ####### values.append(aux) logger.debug("\t * Final Value: " + str(aux)) pos += num_substrings elif p_type == TYPE.INT: values.append(int(p_value)) elif p_type == TYPE.LONG: my_l = long(p_value) if my_l > JAVA_MAX_INT or my_l < JAVA_MIN_INT: # A Python int was converted to a Java long to prevent overflow # We are sure we will not overflow Python int, otherwise this # would have been passed as a serialized object. my_l = int(my_l) values.append(my_l) elif p_type == TYPE.DOUBLE: values.append(float(p_value)) elif p_type == TYPE.BOOLEAN: if p_value == 'true': values.append(True) else: values.append(False) # elif (p_type == TYPE.OBJECT): # pass else: logger.fatal("Invalid type (%d) for parameter %d" % (p_type, i)) exit(1) pos += 4 if tracing: pyextrae.event(TASK_EVENTS, 0) pyextrae.event(TASK_EVENTS, LOGGING) if logger.isEnabledFor(logging.DEBUG): values_str = '' types_str = '' for v in values: values_str += "\t\t" + str(v) + "\n" for t in types: types_str += str(t) + " " logger.debug("RUN TASK with arguments\n" + "\t- Path: " + path + "\n" + "\t- Method/function name: " + method_name + "\n" + "\t- Has target: " + has_target + "\n" + "\t- # parameters: " + str(num_params) + "\n" + "\t- Values:\n" + values_str + "\t- COMPSs types: " + types_str) if tracing: pyextrae.event(TASK_EVENTS, 0) pyextrae.event(TASK_EVENTS, MODULES_IMPORT) try: # Try to import the module (for functions) logger.debug("Trying to import the user module.") if sys.version_info >= (2, 7): module = importlib.import_module(path) # Python 2.7 logger.debug("Module successfully loaded (Python version >= 2.7)") else: module = __import__(path, globals(), locals(), [path], -1) logger.debug("Module successfully loaded (Python version < 2.7") if persistent_storage: with storage_task_context(logger, values, config_file_path=storage_conf): getattr(module, method_name)(*values, compss_types=types, **compss_kwargs) if tracing: pyextrae.eventandcounters(TASK_EVENTS, 0) pyextrae.eventandcounters(TASK_EVENTS, WORKER_END) else: getattr(module, method_name)(*values, compss_types=types, **compss_kwargs) if tracing: pyextrae.eventandcounters(TASK_EVENTS, 0) pyextrae.eventandcounters(TASK_EVENTS, WORKER_END) # ========================================================================== except AttributeError: # Appears with functions that have not been well defined. exc_type, exc_value, exc_traceback = sys.exc_info() lines = traceback.format_exception(exc_type, exc_value, exc_traceback) logger.exception("WORKER EXCEPTION - Attribute Error Exception") logger.exception(''.join(line for line in lines)) logger.exception("Check that all parameters have been defined with " + "an absolute import path (even if in the same file)") exit(1) # ========================================================================== except ImportError: logger.debug("Could not import the module. Reason: Method in class.") # Not the path of a module, it ends with a class name class_name = path.split('.')[-1] module_name = '.'.join(path.split('.')[0:-1]) if '.' in path: module_name = '.'.join(path.split('.')[0:-1]) else: module_name = path module = __import__(module_name, fromlist=[class_name]) klass = getattr(module, class_name) logger.debug("Method in class %s of module %s" % (class_name, module_name)) if has_target == 'true': # Instance method file_name = values.pop() logger.debug("Deserialize self from file.") obj = deserialize_from_file(file_name) logger.debug("Processing callee, a hidden object of %s in file %s" % (file_name, type(obj))) values.insert(0, obj) types.pop() types.insert(0, TYPE.OBJECT) if persistent_storage: with storage_task_context(logger, values, config_file_path=storage_conf): getattr(klass, method_name)(*values, compss_types=types, **compss_kwargs) if tracing: pyextrae.eventandcounters(TASK_EVENTS, 0) pyextrae.eventandcounters(TASK_EVENTS, WORKER_END) else: getattr(klass, method_name)(*values, compss_types=types, **compss_kwargs) if tracing: pyextrae.eventandcounters(TASK_EVENTS, 0) pyextrae.eventandcounters(TASK_EVENTS, WORKER_END) logger.debug("Serializing self to file") logger.debug("Obj: " + str(obj)) serialize_to_file(obj, file_name) else: # Class method - class is not included in values (e.g. values = [7]) types.insert(0, None) # class must be first type if persistent_storage: with storage_task_context(logger, values, config_file_path=storage_conf): getattr(klass, method_name)(*values, compss_types=types, **compss_kwargs) if tracing: pyextrae.eventandcounters(TASK_EVENTS, 0) pyextrae.eventandcounters(TASK_EVENTS, WORKER_END) else: getattr(klass, method_name)(*values, compss_types=types, **compss_kwargs) if tracing: pyextrae.eventandcounters(TASK_EVENTS, 0) pyextrae.eventandcounters(TASK_EVENTS, WORKER_END) # ========================================================================== except Exception: exc_type, exc_value, exc_traceback = sys.exc_info() lines = traceback.format_exception(exc_type, exc_value, exc_traceback) logger.exception("WORKER EXCEPTION") logger.exception(''.join(line for line in lines)) exit(1)
def reveal_objects(values, spec_args, deco_kwargs, compss_types, process_name, has_return, is_multi_return, num_return): """ Function that goes through all parameters in order to find and open the files. :param values: <List> - The value of each parameter. :param spec_args: <List> - Specific arguments. :param deco_kwargs: <List> - The decoratos. :param compss_types: <List> - The types of the values. :param process_name: <String> - Process name (id). :param has_returns: <Boolean> - If the function returns a value. :param is_multi_return: <Boolean> - If the return is a multireturn. :param num_return: <Int> - Number of returns :return: a list with the real values """ from pycompss.api.parameter import Parameter from pycompss.api.parameter import TYPE from pycompss.api.parameter import DIRECTION num_pars = len(spec_args) real_values = [] to_serialize = [] if has_return: num_pars -= num_return # return value must not be passed to the function call # print('Spec args: %s'%(str(spec_args))) # print('COMPSs Types: %s'%(str(compss_types))) # print('Values: %s'%(str(values))) for i in range(num_pars): spec_arg = spec_args[i] compss_type = compss_types[i] value = values[i] if i == 0: if spec_arg == 'self': # callee object if deco_kwargs['isModifier']: d = DIRECTION.INOUT else: d = DIRECTION.IN deco_kwargs[spec_arg] = Parameter(p_type=TYPE.OBJECT, p_direction=d) elif inspect.isclass(value): # class (it's a class method) real_values.append(value) continue p = deco_kwargs.get(spec_arg) if p is None: # decoration not present, using default p = deco_kwargs['varargsType'] if spec_arg.startswith( '*args') else Parameter() if compss_type == TYPE.FILE and p.type != TYPE.FILE: # Getting ids and file names from passed files and objects pattern # is: "originalDataID:destinationDataID;flagToPreserveOriginalData:flagToWrite:PathToFile" complete_fname = value.split(':') if len(complete_fname) > 1: # In NIO we get more information forig = complete_fname[0] fdest = complete_fname[1] preserve = complete_fname[2] write_final = complete_fname[3] fname = complete_fname[4] preserve, write_final = list( map(lambda x: x == "true", [preserve, write_final])) suffix_name = forig else: # In GAT we only get the name fname = complete_fname[0] value = fname # For COMPSs it is a file, but it is actually a Python object logger.debug("Processing a hidden object in parameter %d", i) obj = deserialize_from_file(value) real_values.append(obj) if p.direction != DIRECTION.IN: to_serialize.append((obj, value)) else: # print('compss_type' + str(compss_type) + ' type' + str(p.type)) if compss_type == TYPE.FILE: complete_fname = value.split(':') if len(complete_fname) > 1: # In NIO we get more information forig = complete_fname[0] fdest = complete_fname[1] preserve = complete_fname[2] write_final = complete_fname[3] fname = complete_fname[4] else: # In GAT we only get the name fname = complete_fname[0] value = fname real_values.append(value) return real_values, to_serialize
def compss_worker(persistent_storage): """ Worker main method (invocated from __main__). """ logger = logging.getLogger('pycompss.worker.worker') logger.debug("Starting Worker") args = sys.argv[6:] path = args[0] method_name = args[1] numSlaves = int(args[2]) slaves = [] for i in range(2, 2 + numSlaves): slaves.append(args[i]) argPosition = 3 + numSlaves args = args[argPosition:] cus = args[0] args = args[1:] has_target = args[0] return_type = args[1] num_params = int(args[2]) args = args[3:] pos = 0 values = [] types = [] streams = [] prefixes = [] if tracing: pyextrae.event(TASK_EVENTS, 0) pyextrae.event(TASK_EVENTS, PARAMETER_PROCESSING) if persistent_storage: from storage.api import getByID from storage.api import TaskContext # Get all parameter values logger.debug("Processing parameters:") for i in range(0, num_params): pType = int(args[pos]) pStream = int(args[pos + 1]) pPrefix = args[pos + 2] pValue = args[pos + 3] logger.debug("Parameter : " + str(i)) logger.debug("\t * Type : " + str(pType)) logger.debug("\t * Stream : " + str(pStream)) logger.debug("\t * Prefix : " + str(pPrefix)) logger.debug("\t * Value: " + str(pValue)) types.append(pType) streams.append(pStream) prefixes.append(pPrefix) if pType == TYPE.FILE: ''' # check if it is a persistent object # TODO: I find that it makes no sense to identify PSCOs this way # Why do not we simply check if the object of a subclass of the # storage_object? if 'getID' in dir(pValue) and pValue.getID() is not None: po = getByID(pValue.getID()) values.append(po) else: values.append(pValue) ''' values.append(pValue) elif pType == TYPE.EXTERNAL_PSCO: po = getByID(pValue) values.append(po) pos += 1 # Skip info about direction (R, W) elif pType == TYPE.STRING: num_substrings = int(pValue) aux = '' first_substring = True for j in range(4, num_substrings + 4): if not first_substring: aux += ' ' first_substring = False aux += args[pos + j] # Decode the string received aux = base64.b64decode(aux) ####### # Check if the string is really an object # Required in order to recover objects passed as parameters. # - Option object_conversion real_value = aux try: # try to recover the real object aux = deserialize_from_string(aux.decode('string_escape')) except (SerializerException, ValueError, EOFError): # was not an object aux = real_value ####### values.append(aux) logger.debug("\t * Final Value: " + str(aux)) pos += num_substrings elif pType == TYPE.INT: values.append(int(pValue)) elif pType == TYPE.LONG: l = long(pValue) if l > JAVA_MAX_INT or l < JAVA_MIN_INT: # A Python int was converted to a Java long to prevent overflow # We are sure we will not overflow Python int, otherwise this # would have been passed as a serialized object. l = int(l) values.append(l) elif pType == TYPE.DOUBLE: values.append(float(pValue)) elif pType == TYPE.BOOLEAN: if pValue == 'true': values.append(True) else: values.append(False) # elif (pType == TYPE.OBJECT): # pass else: logger.fatal("Invalid type (%d) for parameter %d" % (pType, i)) exit(1) pos += 4 if tracing: pyextrae.event(TASK_EVENTS, 0) pyextrae.event(TASK_EVENTS, LOGGING) if logger.isEnabledFor(logging.DEBUG): values_str = '' types_str = '' for v in values: values_str += "\t\t" + str(v) + "\n" for t in types: types_str += str(t) + " " logger.debug("RUN TASK with arguments\n" + "\t- Path: " + path + "\n" + "\t- Method/function name: " + method_name + "\n" + "\t- Has target: " + has_target + "\n" + "\t- # parameters: " + str(num_params) + "\n" + "\t- Values:\n" + values_str + "\t- COMPSs types: " + types_str) if tracing: pyextrae.event(TASK_EVENTS, 0) pyextrae.event(TASK_EVENTS, MODULES_IMPORT) try: # Try to import the module (for functions) logger.debug("Trying to import the user module.") if sys.version_info >= (2, 7): module = importlib.import_module(path) # Python 2.7 logger.debug("Module successfully loaded (Python version >= 2.7)") else: module = __import__(path, globals(), locals(), [path], -1) logger.debug("Module successfully loaded (Python version < 2.7") if persistent_storage: with TaskContext(logger, values, config_file_path=storage_conf): getattr(module, method_name)(*values, compss_types=types, compss_tracing=tracing) if tracing: pyextrae.eventandcounters(TASK_EVENTS, 0) pyextrae.eventandcounters(TASK_EVENTS, WORKER_END) else: getattr(module, method_name)(*values, compss_types=types, compss_tracing=tracing) if tracing: pyextrae.eventandcounters(TASK_EVENTS, 0) pyextrae.eventandcounters(TASK_EVENTS, WORKER_END) # ========================================================================== except AttributeError: # Appears with functions that have not been well defined. exc_type, exc_value, exc_traceback = sys.exc_info() lines = traceback.format_exception(exc_type, exc_value, exc_traceback) logger.exception("WORKER EXCEPTION - Attribute Error Exception") logger.exception(''.join(line for line in lines)) logger.exception("Check that all parameters have been defined with " + "an absolute import path (even if in the same file)") exit(1) # ========================================================================== except ImportError: logger.debug("Could not import the module. Reason: Method in class.") # Not the path of a module, it ends with a class name class_name = path.split('.')[-1] module_name = '.'.join(path.split('.')[0:-1]) if '.' in path: module_name = '.'.join(path.split('.')[0:-1]) else: module_name = path module = __import__(module_name, fromlist=[class_name]) klass = getattr(module, class_name) logger.debug("Method in class %s of module %s" % (class_name, module_name)) if has_target == 'true': # Instance method file_name = values.pop() logger.debug("Deserialize self from file.") obj = deserialize_from_file(file_name) logger.debug("Processing callee, a hidden object of %s in file %s" % (file_name, type(obj))) values.insert(0, obj) types.pop() types.insert(0, TYPE.OBJECT) if persistent_storage: with TaskContext(logger, values, config_file_path=storage_conf): getattr(klass, method_name)(*values, compss_types=types, compss_tracing=tracing) if tracing: pyextrae.eventandcounters(TASK_EVENTS, 0) pyextrae.eventandcounters(TASK_EVENTS, WORKER_END) else: getattr(klass, method_name)(*values, compss_types=types, compss_tracing=tracing) if tracing: pyextrae.eventandcounters(TASK_EVENTS, 0) pyextrae.eventandcounters(TASK_EVENTS, WORKER_END) logger.debug("Serializing self to file") logger.debug("Obj: " + str(obj)) serialize_to_file(obj, file_name) else: # Class method - class is not included in values (e.g. values = [7]) types.insert(0, None) # class must be first type if persistent_storage: with TaskContext(logger, values, config_file_path=storage_conf): getattr(klass, method_name)(*values, compss_types=types, compss_tracing=tracing) if tracing: pyextrae.eventandcounters(TASK_EVENTS, 0) pyextrae.eventandcounters(TASK_EVENTS, WORKER_END) else: getattr(klass, method_name)(*values, compss_types=types, compss_tracing=tracing) if tracing: pyextrae.eventandcounters(TASK_EVENTS, 0) pyextrae.eventandcounters(TASK_EVENTS, WORKER_END) # ========================================================================== except Exception: exc_type, exc_value, exc_traceback = sys.exc_info() lines = traceback.format_exception(exc_type, exc_value, exc_traceback) logger.exception("WORKER EXCEPTION") logger.exception(''.join(line for line in lines)) exit(1)