Ejemplo n.º 1
0
    def submit(self,
               func,
               args=(),
               depfuncs=(),
               modules=(),
               callback=None,
               callbackargs=(),
               group='default',
               globals=None,
               functionToSkip=None):
        """Submits function to the execution queue

            func - function to be executed
            args - tuple with arguments of the 'func'
            depfuncs - tuple with functions which might be called from 'func'
            modules - tuple with module names to import
            callback - callback function which will be called with argument
                    list equal to callbackargs+(result,)
                    as soon as calculation is done
            callbackargs - additional arguments for callback function
            group - job group, is used when wait(group) is called to wait for
            jobs in a given group to finish
            globals - dictionary from which all modules, functions and classes
            will be imported, for instance: globals=globals()
            functionToSkip - list of functions that need to be skipped from pickling
        """

        # perform some checks for frequent mistakes
        if self._exiting:
            raise DestroyedServerError("Cannot submit jobs: server"\
                    " instance has been destroyed")

        if not isinstance(args, tuple):
            raise TypeError("args argument must be a tuple")

        if not isinstance(depfuncs, tuple):
            raise TypeError("depfuncs argument must be a tuple")

        if not isinstance(modules, tuple):
            raise TypeError("modules argument must be a tuple")

        if not isinstance(callbackargs, tuple):
            raise TypeError("callbackargs argument must be a tuple")

        if globals is not None and not isinstance(globals, dict):
            raise TypeError("globals argument must be a dictionary")

        for module in modules:
            if not isinstance(module, str):
                raise TypeError("modules argument must be a list of strings")

        tid = self.__gentid()

        other_type = types.FunctionType if six.PY3 else types.ClassType
        if globals:
            modules += tuple(self.__find_modules("", globals))
            modules = tuple(set(modules))
            self.logger.debug("Task %i will autoimport next modules: %s" %
                              (tid, str(modules)))
            for object1 in globals.values():
                if isinstance(object1, types.FunctionType) \
                        or isinstance(object1, other_type):
                    depfuncs += (object1, )

        task = _Task(self, tid, callback, callbackargs, group)

        self.__waittasks_lock.acquire()
        self.__waittasks.append(task)
        self.__waittasks_lock.release()

        # if the function is a method of a class add self to the arguments list
        if isinstance(func, types.MethodType):
            func_self = func.__self__ if six.PY3 else func.im_self
            if func_self is not None:
                args = (func_self, ) + args

        # if there is an instance of a user defined class in the arguments add
        # whole class to dependancies
        for arg in args:
            # Checks for both classic or new class instances
            if (six.PY2 and isinstance(arg, types.InstanceType)) \
                        or str(type(arg))[:6] == "<class":
                # in PY3, all instances are <class... so skip the builtins
                if getattr(inspect.getmodule(arg), '__name__', None) \
                   in ['builtins', '__builtin__', None]:
                    pass
                    # do not include source for imported modules
                elif ppc.is_not_imported(arg, modules):
                    #RAVEN CHANGE: adding functionToSkip removing.
                    clshier = ppc.get_class_hierarchy(arg.__class__)
                    if functionToSkip != None:
                        tempclshier = []
                        clshierstr = [
                            str(elem) for elem in clshier
                        ]  # we use string in order to avoid baseclass identity!
                        functionToSkipSet = set(
                            [str(elem) for elem in functionToSkip])
                        for cnt, clshierfun in enumerate(clshierstr):
                            if clshierfun not in functionToSkipSet:
                                tempclshier.append(clshier[cnt])
                        clshier = tempclshier
                    depfuncs += tuple(clshier)

        # if there is a function in the arguments add this
        # function to dependancies
        for arg in args:
            if isinstance(arg, types.FunctionType):
                depfuncs += (arg, )

        sfunc = self.__dumpsfunc((func, ) + depfuncs, modules)
        sargs = pickle.dumps(args, self.__pickle_proto)

        self.__queue_lock.acquire()
        self.__queue.append((task, sfunc, sargs))
        self.__queue_lock.release()

        self.logger.debug("Task %i submited, function='%s'" %
                          (tid, getname(func)))
        self.__scheduler()
        return task
Ejemplo n.º 2
0
    def submit(self, func, args=(), depfuncs=(), modules=(),
            callback=None, callbackargs=(), group='default', globals=None):
        """Submits function to the execution queue

            func - function to be executed
            args - tuple with arguments of the 'func'
            depfuncs - tuple with functions which might be called from 'func'
            modules - tuple with module names to import
            callback - callback function which will be called with argument
                    list equal to callbackargs+(result,)
                    as soon as calculation is done
            callbackargs - additional arguments for callback function
            group - job group, is used when wait(group) is called to wait for
            jobs in a given group to finish
            globals - dictionary from which all modules, functions and classes
            will be imported, for instance: globals=globals()
        """

        # perform some checks for frequent mistakes
        if self._exiting:
            raise DestroyedServerError("Cannot submit jobs: server"\
                    " instance has been destroyed")

        if not isinstance(args, tuple):
            raise TypeError("args argument must be a tuple")

        if not isinstance(depfuncs, tuple):
            raise TypeError("depfuncs argument must be a tuple")

        if not isinstance(modules, tuple):
            raise TypeError("modules argument must be a tuple")

        if not isinstance(callbackargs, tuple):
            raise TypeError("callbackargs argument must be a tuple")

        if globals is not None and not isinstance(globals, dict):
            raise TypeError("globals argument must be a dictionary")

        for module in modules:
            if not isinstance(module, types.StringType):
                raise TypeError("modules argument must be a list of strings")

        tid = self.__gentid()

        if globals:
            modules += tuple(self.__find_modules("", globals))
            modules = tuple(set(modules))
            self.logger.debug("Task %i will autoimport next modules: %s" %
                    (tid, str(modules)))
            for object1 in globals.values():
                if isinstance(object1, types.FunctionType) \
                        or isinstance(object1, types.ClassType):
                    depfuncs += (object1, )

        task = _Task(self, tid, callback, callbackargs, group)

        self.__waittasks_lock.acquire()
        self.__waittasks.append(task)
        self.__waittasks_lock.release()

        # if the function is a method of a class add self to the arguments list
        if isinstance(func, types.MethodType) and func.im_self is not None:
            args = (func.im_self, ) + args

        # if there is an instance of a user deined class in the arguments add
        # whole class to dependancies
        for arg in args:
            # Checks for both classic or new class instances
            if isinstance(arg, types.InstanceType) \
                    or str(type(arg))[:6] == "<class":
                # do not include source for imported modules
                if ppcommon.is_not_imported(arg, modules):
                    depfuncs += tuple(ppcommon.get_class_hierarchy(arg.__class__))

        # if there is a function in the arguments add this
        # function to dependancies
        for arg in args:
            if isinstance(arg, types.FunctionType):
                depfuncs += (arg, )

        sfunc = self.__dumpsfunc((func, ) + depfuncs, modules)
        sargs = pickle.dumps(args, self.__pickle_proto)

        self.__queue_lock.acquire()
        self.__queue.append((task, sfunc, sargs))
        self.__queue_lock.release()

        self.logger.debug("Task %i submited, function='%s'" %
                (tid, func.func_name))
        self.__scheduler()
        return task
Ejemplo n.º 3
0
    def submit(self,
               func,
               args=(),
               depfuncs=(),
               modules=(),
               callback=None,
               callbackargs=(),
               group='default',
               globals=None):
        """Submits function to the execution queue

            func - function to be executed
            args - tuple with arguments of the 'func'
            depfuncs - tuple with functions which might be called from 'func'
            modules - tuple with module names to import
            callback - callback function which will be called with argument
                    list equal to callbackargs+(result,)
                    as soon as calculation is done
            callbackargs - additional arguments for callback function
            group - job group, is used when wait(group) is called to wait for
            jobs in a given group to finish
            globals - dictionary from which all modules, functions and classes
            will be imported, for instance: globals=globals()
        """

        # perform some checks for frequent mistakes
        if self._exiting:
            raise DestroyedServerError("Cannot submit jobs: server"\
                    " instance has been destroyed")

        if not isinstance(args, tuple):
            raise TypeError("args argument must be a tuple")

        if not isinstance(depfuncs, tuple):
            raise TypeError("depfuncs argument must be a tuple")

        if not isinstance(modules, tuple):
            raise TypeError("modules argument must be a tuple")

        if not isinstance(callbackargs, tuple):
            raise TypeError("callbackargs argument must be a tuple")

        if globals is not None and not isinstance(globals, dict):
            raise TypeError("globals argument must be a dictionary")

        for module in modules:
            if not isinstance(module, types.StringType):
                raise TypeError("modules argument must be a list of strings")

        tid = self.__gentid()

        if globals:
            modules += tuple(self.__find_modules("", globals))
            modules = tuple(set(modules))
            self.logger.debug("Task %i will autoimport next modules: %s" %
                              (tid, str(modules)))
            for object1 in globals.values():
                if isinstance(object1, types.FunctionType) \
                        or isinstance(object1, types.ClassType):
                    depfuncs += (object1, )

        task = _Task(self, tid, callback, callbackargs, group)

        self.__waittasks_lock.acquire()
        self.__waittasks.append(task)
        self.__waittasks_lock.release()

        # if the function is a method of a class add self to the arguments list
        if isinstance(func, types.MethodType) and func.im_self is not None:
            args = (func.im_self, ) + args

        # if there is an instance of a user deined class in the arguments add
        # whole class to dependancies
        for arg in args:
            # Checks for both classic or new class instances
            if isinstance(arg, types.InstanceType) \
                    or str(type(arg))[:6] == "<class":
                # do not include source for imported modules
                if ppcommon.is_not_imported(arg, modules):
                    depfuncs += tuple(
                        ppcommon.get_class_hierarchy(arg.__class__))

        # if there is a function in the arguments add this
        # function to dependancies
        for arg in args:
            if isinstance(arg, types.FunctionType):
                depfuncs += (arg, )

        sfunc = self.__dumpsfunc((func, ) + depfuncs, modules)
        sargs = pickle.dumps(args, self.__pickle_proto)

        self.__queue_lock.acquire()
        self.__queue.append((task, sfunc, sargs))
        self.__queue_lock.release()

        self.logger.debug("Task %i submited, function='%s'" %
                          (tid, func.func_name))
        self.__scheduler()
        return task