Beispiel #1
0
 def _email_unexpected_error(self, task, formatted_traceback):
     formatted_traceback = notifications.wrap_traceback(formatted_traceback)
     subject = "Luigi: Framework error while scheduling {task}".format(
         task=task)
     message = "Luigi framework error:\n{traceback}".format(
         traceback=formatted_traceback)
     notifications.send_error_email(subject, message)
Beispiel #2
0
 def _email_complete_error(self, task, formatted_traceback):
     # like logger.exception but with WARNING level
     formatted_traceback = notifications.wrap_traceback(formatted_traceback)
     subject = "Luigi: {task} failed scheduling".format(task=task)
     message = "Will not schedule {task} or any dependencies due to error in complete() method:\n{traceback}".format(
         task=task, traceback=formatted_traceback)
     notifications.send_error_email(subject, message)
Beispiel #3
0
 def _email_complete_error(self, task, formatted_traceback):
     # like logger.exception but with WARNING level
     formatted_traceback = notifications.wrap_traceback(formatted_traceback)
     subject = "Luigi: {task} failed scheduling".format(task=task)
     message = "Will not schedule {task} or any dependencies due to error in complete() method:\n{traceback}".format(
         task=task, traceback=formatted_traceback
     )
     notifications.send_error_email(subject, message)
Beispiel #4
0
    def run(self):
        logger.info('[pid %s] Worker %s running   %s', os.getpid(),
                    self.worker_id, self.task.task_id)

        if self.random_seed:
            # Need to have different random seeds if running in separate processes
            random.seed((os.getpid(), time.time()))

        status = FAILED
        error_message = ''
        missing = []
        try:
            # Verify that all the tasks are fulfilled!
            missing = [
                dep.task_id for dep in self.task.deps() if not dep.complete()
            ]
            if missing:
                deps = 'dependency' if len(missing) == 1 else 'dependencies'
                raise RuntimeError('Unfulfilled %s at run time: %s' %
                                   (deps, ', '.join(missing)))
            self.task.trigger_event(Event.START, self.task)
            t0 = time.time()
            try:
                self.task.run()
            finally:
                self.task.trigger_event(Event.PROCESSING_TIME, self.task,
                                        time.time() - t0)
            error_message = json.dumps(self.task.on_success())
            logger.info('[pid %s] Worker %s done      %s', os.getpid(),
                        self.worker_id, self.task.task_id)
            self.task.trigger_event(Event.SUCCESS, self.task)
            status = DONE

        except KeyboardInterrupt:
            raise
        except BaseException as ex:
            status = FAILED
            logger.exception("[pid %s] Worker %s failed    %s", os.getpid(),
                             self.worker_id, self.task)
            error_message = notifications.wrap_traceback(
                self.task.on_failure(ex))
            self.task.trigger_event(Event.FAILURE, self.task, ex)
            subject = "Luigi: %s FAILED" % self.task
            notifications.send_error_email(subject, error_message)
        finally:
            self.result_queue.put(
                (self.task.task_id, status, error_message, missing))
Beispiel #5
0
    def run(self):
        logger.info('[pid %s] Worker %s running   %s', os.getpid(),
                    self.worker_id, self.task.task_id)

        if self.random_seed:
            # Need to have different random seeds if running in separate processes
            random.seed((os.getpid(), time.time()))

        status = FAILED
        error_message = ''
        try:
            self.task.trigger_event(Event.START, self.task)
            t0 = time.time()
            status = None
            try:
                status = DONE if self.task.complete() else FAILED
                logger.debug("[pid %s] Task %s has status %s" %
                             (os.getpid(), self.task, status))
            finally:
                self.task.trigger_event(Event.PROCESSING_TIME, self.task,
                                        time.time() - t0)

            error_message = json.dumps(self.task.on_success())
            logger.info('[pid %s] Worker %s done      %s', os.getpid(),
                        self.worker_id, self.task.task_id)
            self.task.trigger_event(Event.SUCCESS, self.task)

        except KeyboardInterrupt:
            raise
        except BaseException as ex:
            status = FAILED
            logger.exception("[pid %s] Worker %s failed    %s", os.getpid(),
                             self.worker_id, self.task)
            error_message = notifications.wrap_traceback(
                self.task.on_failure(ex))
            self.task.trigger_event(Event.FAILURE, self.task, ex)
            subject = "Luigi: %s FAILED" % self.task
            notifications.send_error_email(subject, error_message)
        finally:
            logger.debug("Putting result into queue: %s %s %s" %
                         (self.task.task_id, status, error_message))
            self.result_queue.put(
                (self.task.task_id, status, error_message, [], []))
Beispiel #6
0
    def run(self):
        logger.info('[pid %s] Worker %s running   %s', os.getpid(), self.worker_id, self.task.task_id)

        if self.random_seed:
            # Need to have different random seeds if running in separate processes
            random.seed((os.getpid(), time.time()))

        status = FAILED
        error_message = ''
        missing = []
        try:
            # Verify that all the tasks are fulfilled!
            missing = [dep.task_id for dep in self.task.deps() if not dep.complete()]
            if missing:
                deps = 'dependency' if len(missing) == 1 else 'dependencies'
                raise RuntimeError('Unfulfilled %s at run time: %s' % (deps, ', '.join(missing)))
            self.task.trigger_event(Event.START, self.task)
            t0 = time.time()
            try:
                self.task.run()
            finally:
                self.task.trigger_event(Event.PROCESSING_TIME, self.task, time.time() - t0)
            error_message = json.dumps(self.task.on_success())
            logger.info('[pid %s] Worker %s done      %s', os.getpid(), self.worker_id, self.task.task_id)
            self.task.trigger_event(Event.SUCCESS, self.task)
            status = DONE

        except KeyboardInterrupt:
            raise
        except BaseException as ex:
            status = FAILED
            logger.exception("[pid %s] Worker %s failed    %s", os.getpid(), self.worker_id, self.task)
            error_message = notifications.wrap_traceback(self.task.on_failure(ex))
            self.task.trigger_event(Event.FAILURE, self.task, ex)
            subject = "Luigi: %s FAILED" % self.task
            notifications.send_error_email(subject, error_message)
        finally:
            self.result_queue.put(
                (self.task.task_id, status, error_message, missing))
Beispiel #7
0
    def run(self):
        logger.info('[pid %s] Worker %s running   %s', os.getpid(), self.worker_id, self.task.task_id)

        if self.random_seed:
            # Need to have different random seeds if running in separate processes
            random.seed((os.getpid(), time.time()))

        status = FAILED
        error_message = ''
        missing = []
        new_deps = []
        try:
            # Verify that all the tasks are fulfilled!
            missing = [dep.task_id for dep in self.task.deps() if not dep.complete()]
            if missing:
                deps = 'dependency' if len(missing) == 1 else 'dependencies'
                raise RuntimeError('Unfulfilled %s at run time: %s' % (deps, ', '.join(missing)))
            self.task.trigger_event(Event.START, self.task)
            t0 = time.time()
            status = None
            try:
                task_gen = self.task.run()
                if isinstance(task_gen, types.GeneratorType):  # new deps
                    next_send = None
                    while True:
                        try:
                            if next_send is None:
                                requires = task_gen.next()
                            else:
                                requires = task_gen.send(next_send)
                        except StopIteration:
                            break

                        new_req = flatten(requires)
                        status = (RUNNING if all(t.complete() for t in new_req)
                                  else SUSPENDED)
                        new_deps = [(t.task_family, t.to_str_params())
                                    for t in new_req]
                        if status == RUNNING:
                            self.result_queue.put(
                                (self.task.task_id, status, '', missing,
                                 new_deps))
                            next_send = getpaths(requires)
                        else:
                            logger.info(
                                '[pid %s] Worker %s new requirements      %s',
                                os.getpid(), self.worker_id, self.task.task_id)
                            return
            finally:
                if status != SUSPENDED:
                    self.task.trigger_event(
                        Event.PROCESSING_TIME, self.task, time.time() - t0)
            error_message = json.dumps(self.task.on_success())
            logger.info('[pid %s] Worker %s done      %s', os.getpid(),
                        self.worker_id, self.task.task_id)
            self.task.trigger_event(Event.SUCCESS, self.task)
            status = DONE

        except KeyboardInterrupt:
            raise
        except BaseException as ex:
            status = FAILED
            logger.exception("[pid %s] Worker %s failed    %s", os.getpid(), self.worker_id, self.task)
            error_message = notifications.wrap_traceback(self.task.on_failure(ex))
            self.task.trigger_event(Event.FAILURE, self.task, ex)
            subject = "Luigi: %s FAILED" % self.task
            notifications.send_error_email(subject, error_message)
        finally:
            self.result_queue.put(
                (self.task.task_id, status, error_message, missing, new_deps))
Beispiel #8
0
 def _email_unexpected_error(self, task, formatted_traceback):
     formatted_traceback = notifications.wrap_traceback(formatted_traceback)
     subject = "Luigi: Framework error while scheduling {task}".format(task=task)
     message = "Luigi framework error:\n{traceback}".format(traceback=formatted_traceback)
     notifications.send_error_email(subject, message)
Beispiel #9
0
    def run(self):
        logger.info('[pid %s] Worker %s running   %s', os.getpid(),
                    self.worker_id, self.task.task_id)

        if self.random_seed:
            # Need to have different random seeds if running in separate processes
            random.seed((os.getpid(), time.time()))

        status = FAILED
        error_message = ''
        missing = []
        new_deps = []
        try:
            # Verify that all the tasks are fulfilled!
            missing = [
                dep.task_id for dep in self.task.deps() if not dep.complete()
            ]
            if missing:
                deps = 'dependency' if len(missing) == 1 else 'dependencies'
                raise RuntimeError('Unfulfilled %s at run time: %s' %
                                   (deps, ', '.join(missing)))
            self.task.trigger_event(Event.START, self.task)
            t0 = time.time()
            status = None
            try:
                task_gen = self.task.run()
                if isinstance(task_gen, types.GeneratorType):  # new deps
                    next_send = None
                    while True:
                        try:
                            if next_send is None:
                                requires = task_gen.next()
                            else:
                                requires = task_gen.send(next_send)
                        except StopIteration:
                            break

                        new_req = flatten(requires)
                        status = (RUNNING if all(
                            t.complete() for t in new_req) else SUSPENDED)
                        new_deps = [(t.task_module, t.task_family,
                                     t.to_str_params()) for t in new_req]
                        if status == RUNNING:
                            self.result_queue.put((self.task.task_id, status,
                                                   '', missing, new_deps))
                            next_send = getpaths(requires)
                            new_deps = []
                        else:
                            logger.info(
                                '[pid %s] Worker %s new requirements      %s',
                                os.getpid(), self.worker_id, self.task.task_id)
                            return
            finally:
                if status != SUSPENDED:
                    self.task.trigger_event(Event.PROCESSING_TIME, self.task,
                                            time.time() - t0)
            error_message = json.dumps(self.task.on_success())
            logger.info('[pid %s] Worker %s done      %s', os.getpid(),
                        self.worker_id, self.task.task_id)
            self.task.trigger_event(Event.SUCCESS, self.task)
            status = DONE

        except KeyboardInterrupt:
            raise
        except BaseException as ex:
            status = FAILED
            logger.exception("[pid %s] Worker %s failed    %s", os.getpid(),
                             self.worker_id, self.task)
            error_message = notifications.wrap_traceback(
                self.task.on_failure(ex))
            self.task.trigger_event(Event.FAILURE, self.task, ex)
            subject = "Luigi: %s FAILED" % self.task
            notifications.send_error_email(subject, error_message)
        finally:
            self.result_queue.put(
                (self.task.task_id, status, error_message, missing, new_deps))