Ejemplo n.º 1
0
Archivo: config.py Proyecto: pbs/flowy
    def try_register_remote(self, swf_client, domain, name, version):
        """Register the activity remotely.

        Returns True if registration is successful and False if another
        activity with the same name is already registered.

        Raise SWFRegistrationError in case of SWF communication errors and
        ValueError if any configuration values can't be converted to the
        required types.

        :type swf_client: :class:`flowy.swf.client.SWFClient`
        :param swf_client: an implementation or duck typing of `SWFClient`
        :param domain: the domain name where to register the activity
        :param name: name of the activity
        :param version: version of the activity
        :rtype: bool
        """
        d_t_l, d_h, d_sch_c, d_sch_s, d_s_c = self._cvt_values()
        try:
            swf_client.register_activity_type(domain, name, version,
                                              default_task_list=d_t_l,
                                              default_heartbeat_timeout=d_h,
                                              default_exec_timeout=d_s_c,
                                              default_start_timeout=d_sch_s,
                                              default_close_timeout=d_sch_c)
        except ClientError as e:
            if e.response['Error']['Code'] == 'TypeAlreadyExistsFault':
                return False
            logger.exception('Error while registering the activity:')
            raise SWFRegistrationError(e)
        return True
Ejemplo n.º 2
0
    def try_register_remote(self, swf_layer1, domain, name, version):
        """Register the workflow remotely.

        Returns True if registration is successful and False if another
        workflow with the same name is already registered.

        A name should be set before calling this method or RuntimeError is
        raised.

        Raise SWFRegistrationError in case of SWF communication errors and
        ValueError if any configuration values can't be converted to the
        required types.
        """
        d_t_l, d_w_d, d_d_d, d_c_p = self._cvt_values()
        try:
            swf_layer1.register_workflow_type(
                str(domain),
                name=str(name),
                version=str(version),
                task_list=d_t_l,
                default_execution_start_to_close_timeout=d_w_d,
                default_task_start_to_close_timeout=d_d_d,
                default_child_policy=d_c_p)
        except SWFTypeAlreadyExistsError:
            return False
        except SWFResponseError as e:
            if 'TypeAlreadyExistsFault' in str(e):  # eucalyptus
                return False
            logger.exception('Error while registering the workflow:')
            raise SWFRegistrationError(e)
        return True
Ejemplo n.º 3
0
    def try_register_remote(self, swf_client, domain, name, version):
        """Register the workflow remotely.

        Returns True if registration is successful and False if another
        workflow with the same name is already registered.

        A name should be set before calling this method or RuntimeError is
        raised.

        Raise SWFRegistrationError in case of SWF communication errors and
        ValueError if any configuration values can't be converted to the
        required types.

        :type swf_client: :class:`flowy.swf.client.SWFClient`
        :param swf_client: an implementation or duck typing of `SWFClient`
        :param domain: the domain name where to register the workflow
        :param name: name of the workflow
        :param version: version of the workflow
        :rtype: bool
        """
        d_t_l, d_w_d, d_d_d, d_c_p = self._cvt_values()
        try:
            swf_client.register_workflow_type(domain,
                                              name,
                                              version,
                                              default_task_list=d_t_l,
                                              default_task_timeout=d_d_d,
                                              default_exec_timeout=d_w_d,
                                              default_child_policy=d_c_p)
        except ClientError as e:
            if e.response['Error']['Code'] == 'TypeAlreadyExistsFault':
                return False
            logger.exception('Error while registering the workflow:')
            raise SWFRegistrationError(e)
        return True
Ejemplo n.º 4
0
    def try_register_remote(self, swf_layer1, domain, name, version):
        """Register the activity remotely.

        Returns True if registration is successful and False if another
        activity with the same name is already registered.

        Raise SWFRegistrationError in case of SWF communication errors and
        ValueError if any configuration values can't be converted to the
        required types.
        """
        d_t_l, d_h, d_sch_c, d_sch_s, d_s_c = self._cvt_values()
        try:
            swf_layer1.register_activity_type(
                str(domain),
                name=str(name),
                version=str(version),
                task_list=d_t_l,
                default_task_heartbeat_timeout=d_h,
                default_task_schedule_to_close_timeout=d_sch_c,
                default_task_schedule_to_start_timeout=d_sch_s,
                default_task_start_to_close_timeout=d_s_c)
        except SWFTypeAlreadyExistsError:
            return False
        except SWFResponseError as e:
            if 'TypeAlreadyExistsFault' in str(e):  # eucalyptus
                return False
            logger.exception('Error while registering the activity:')
            raise SWFRegistrationError(e)
        return True
Ejemplo n.º 5
0
 def heartbeat(self):
     try:
         self.layer1.record_activity_task_heartbeat(task_token=str(self.token))
     except SWFResponseError:
         logger.exception('Error while sending the heartbeat:')
         return False
     return True
Ejemplo n.º 6
0
    def try_register_remote(self, swf_client, domain, name, version):
        """Register the activity remotely.

        Returns True if registration is successful and False if another
        activity with the same name is already registered.

        Raise SWFRegistrationError in case of SWF communication errors and
        ValueError if any configuration values can't be converted to the
        required types.

        :type swf_client: :class:`flowy.swf.client.SWFClient`
        :param swf_client: an implementation or duck typing of `SWFClient`
        :param domain: the domain name where to register the activity
        :param name: name of the activity
        :param version: version of the activity
        :rtype: bool
        """
        d_t_l, d_h, d_sch_c, d_sch_s, d_s_c = self._cvt_values()
        try:
            swf_client.register_activity_type(domain,
                                              name,
                                              version,
                                              default_task_list=d_t_l,
                                              default_heartbeat_timeout=d_h,
                                              default_exec_timeout=d_s_c,
                                              default_start_timeout=d_sch_s,
                                              default_close_timeout=d_sch_c)
        except ClientError as e:
            if e.response['Error']['Code'] == 'TypeAlreadyExistsFault':
                return False
            logger.exception('Error while registering the activity:')
            raise SWFRegistrationError(e)
        return True
Ejemplo n.º 7
0
    def __call__(self, key, input_data, decision, *extra_args):
        """Execute the wrapped func registered with key passing the input_data.

        Any exra_args are also passed along to the wrapped_key.

        The actual actions are dispatched to the decision object and can be one
        of:
            * flush() - nothing to do, any pending actions should be commited
            * fail(e) - ignore pending actions, fail the execution
            * finish(e) - ignore pending actions, complete the execution
            * restart(serialized_input) - ignore pending actions, restart the execution
        """
        try:
            wrapped_func = self.registry[key]
        except KeyError:
            logger.error("Colud not find implementation for key: %r", (key,))
            return  # Let it timeout
        try:
            serialized_result = wrapped_func(input_data, *extra_args)
        except SuspendTask:
            decision.flush()
        except TaskError as e:
            logger.exception('Unhandled task error in task:')
            decision.fail(e)
        except Restart as e:
            decision.restart(e.input_data)
        except Exception as e:
            logger.exception('Unhandled exception in task:')
            decision.fail(e)
        else:
            decision.finish(serialized_result)
Ejemplo n.º 8
0
    def run_forever(self, domain, task_list,
                    layer1=None,
                    setup_log=True,
                    register_remote=True,
                    identity=None):
        """Same as SWFWorkflowWorker.run_forever but for activities."""
        if setup_log:
            setup_default_logger()
        identity = identity if identity is not None else default_identity()
        identity = str(identity)[:_IDENTITY_SIZE]
        layer1 = layer1 if layer1 is not None else Layer1()
        if register_remote:
            self.register_remote(layer1, domain)
        try:
            while 1:
                if self.break_loop():
                    break
                swf_response = {}
                while ('taskToken' not in swf_response or not swf_response['taskToken']):
                    try:
                        swf_response = layer1.poll_for_activity_task(
                            domain=domain,
                            task_list=task_list,
                            identity=identity)
                    except SWFResponseError:
                        # add a delay before retrying?
                        logger.exception('Error while polling for activities:')

                at = swf_response['activityType']
                decision = SWFActivityDecision(layer1, swf_response['taskToken'])
                self(at['name'], at['version'], swf_response['input'], decision)
        except KeyboardInterrupt:
            pass
Ejemplo n.º 9
0
    def __call__(self, key, input_data, decision, *extra_args):
        """Execute the wrapped func registered with key passing the input_data.

        Any exra_args are also passed along to the wrapped_key.

        The actual actions are dispatched to the decision object and can be one
        of:
            * flush() - nothing to do, any pending actions should be commited
            * fail(e) - ignore pending actions, fail the execution
            * finish(e) - ignore pending actions, complete the execution
            * restart(serialized_input) - ignore pending actions, restart the execution
        """
        try:
            wrapped_func = self.registry[key]
        except KeyError:
            logger.error("Colud not find implementation for key: %r", (key, ))
            return  # Let it timeout
        try:
            serialized_result = wrapped_func(input_data, *extra_args)
        except SuspendTask:  # only from workflows
            decision.flush()
        except TaskError as e:  # only from workflows
            logger.exception('Unhandled task error in task:')
            decision.fail(e)
        except Restart as e:  # only from workflows
            decision.restart(e.input_data)
        except Exception as e:
            logger.exception('Unhandled exception in task:')
            decision.fail(e)
        else:
            decision.finish(serialized_result)
Ejemplo n.º 10
0
    def run_forever(self,
                    domain,
                    task_list,
                    swf_client=None,
                    setup_log=True,
                    register_remote=True,
                    identity=None):
        """Same as SWFWorkflowWorker.run_forever but for activities."""
        if setup_log:
            setup_default_logger()
        identity = default_identity() if identity is None else identity
        swf_client = SWFClient() if swf_client is None else swf_client
        if register_remote:
            self.register_remote(swf_client, domain)
        try:
            while 1:
                if self.break_loop():
                    break
                swf_response = {}
                while not swf_response.get('taskToken'):
                    try:
                        swf_response = swf_client.poll_for_activity_task(
                            domain, task_list, identity=identity)
                    except ClientError:
                        # add a delay before retrying?
                        logger.exception('Error while polling for activities:')

                at = swf_response['activityType']
                decision = SWFActivityDecision(swf_client,
                                               swf_response['taskToken'])
                self(at['name'], at['version'], swf_response['input'],
                     decision)
        except KeyboardInterrupt:
            pass
Ejemplo n.º 11
0
    def check_compatible(self, swf_layer1, domain, name, version):
        """Check if the remote config has the same defaults as this one.

        Raise SWFRegistrationError in case of SWF communication errors or
        incompatibility and ValueError if any configuration values can't be
        converted to the required types.
        """
        d_t_l, d_w_d, d_d_d, d_c_p = self._cvt_values()
        try:
            w_descr = swf_layer1.describe_workflow_type(str(domain), str(name), str(version))
            w_descr = w_descr['configuration']
        except SWFResponseError as e:
            logger.exception('Error while checking workflow compatibility:')
            raise SWFRegistrationError(e)
        r_d_t_l = w_descr.get('defaultTaskList', {}).get('name')
        if r_d_t_l != d_t_l:
            raise SWFRegistrationError(
                'Default task list for %r version %r does not match: %r != %r' %
                (name, version, r_d_t_l, d_t_l))
        r_d_d_d = w_descr.get('defaultTaskStartToCloseTimeout')
        if r_d_d_d != d_d_d:
            raise SWFRegistrationError(
                'Default decision duration for %r version %r does not match: %r != %r'
                % (name, version, r_d_d_d, d_d_d))
        r_d_w_d = w_descr.get('defaultExecutionStartToCloseTimeout')
        if r_d_w_d != d_w_d:
            raise SWFRegistrationError(
                'Default workflow duration for %r version %r does not match: %r != %r'
                % (name, version, r_d_w_d, d_w_d))
        r_d_c_p = w_descr.get('defaultChildPolicy')
        if r_d_c_p != d_c_p:
            raise SWFRegistrationError(
                'Default child policy for %r version %r does not match: %r != %r' %
                (name, version, r_d_c_p, d_c_p))
Ejemplo n.º 12
0
Archivo: config.py Proyecto: pbs/flowy
    def try_register_remote(self, swf_client, domain, name, version):
        """Register the workflow remotely.

        Returns True if registration is successful and False if another
        workflow with the same name is already registered.

        A name should be set before calling this method or RuntimeError is
        raised.

        Raise SWFRegistrationError in case of SWF communication errors and
        ValueError if any configuration values can't be converted to the
        required types.

        :type swf_client: :class:`flowy.swf.client.SWFClient`
        :param swf_client: an implementation or duck typing of `SWFClient`
        :param domain: the domain name where to register the workflow
        :param name: name of the workflow
        :param version: version of the workflow
        :rtype: bool
        """
        d_t_l, d_w_d, d_d_d, d_c_p = self._cvt_values()
        try:
            swf_client.register_workflow_type(domain, name, version,
                                              default_task_list=d_t_l,
                                              default_task_timeout=d_d_d,
                                              default_exec_timeout=d_w_d,
                                              default_child_policy=d_c_p)
        except ClientError as e:
            if e.response['Error']['Code'] == 'TypeAlreadyExistsFault':
                return False
            logger.exception('Error while registering the workflow:')
            raise SWFRegistrationError(e)
        return True
Ejemplo n.º 13
0
    def really_start(*args, **kwargs):
        """Use this function to start a workflow by passing in the args."""
        swf = swf_client if swf_client is not None else SWFClient()
        l_wid = wid  # closure hack
        if l_wid is None:
            l_wid = uuid.uuid4()
        if serialize_input is None:
            input_data = Proxy.serialize_input(*args, **kwargs)
        else:
            input_data = serialize_input(*args, **kwargs)
        if len(input_data) > INPUT_SIZE:
            logger.error(
                "Input too large: %s/%s" % (len(input_data), INPUT_SIZE))
            raise ValueError('Input too large.')
        try:
            r = swf.start_workflow_execution(
                domain, l_wid, name, version, input=input_data,
                priority=priority, task_list=task_list,
                execution_start_to_close_timeout=task_duration,
                task_start_to_close_timeout=workflow_duration,
                child_policy=child_policy, tags=tags, lambda_role=lambda_role)
        except ClientError as e:
            logger.exception('Error while starting the workflow:')
            raise e

        return r['runId']
Ejemplo n.º 14
0
 def really_start(*args, **kwargs):
     """Use this function to start a workflow by passing in the args."""
     l1 = layer1 if layer1 is not None else Layer1()
     l_wid = wid  # closue hack
     if l_wid is None:
         l_wid = uuid.uuid4()
     if serialize_input is None:
         input_data = Proxy.serialize_input(*args, **kwargs)
     else:
         input_data = serialize_input(*args, **kwargs)
     if len(input_data) > INPUT_SIZE:
         logger.error("Input too large: %s/%s" % (len(input_data), INPUT_SIZE))
         raise ValueError('Input too large.')
     try:
         r = l1.start_workflow_execution(
             str(domain), str(l_wid), str(name), str(version),
             task_list=str_or_none(task_list),
             execution_start_to_close_timeout=str_or_none(workflow_duration),
             task_start_to_close_timeout=str_or_none(decision_duration),
             input=str(input_data),
             child_policy=cp_encode(child_policy),
             tag_list=tags_encode(tags))
     except SWFResponseError:
         logger.exception('Error while starting the workflow:')
         raise RuntimeError('Cannot start the workflow.')
     return r['runId']
Ejemplo n.º 15
0
Archivo: worker.py Proyecto: pbs/flowy
    def run_forever(self, domain, task_list,
                    swf_client=None,
                    setup_log=True,
                    register_remote=True,
                    identity=None):
        """Same as SWFWorkflowWorker.run_forever but for activities."""
        if setup_log:
            setup_default_logger()
        identity = default_identity() if identity is None else identity
        swf_client = SWFClient() if swf_client is None else swf_client
        if register_remote:
            self.register_remote(swf_client, domain)
        try:
            while 1:
                if self.break_loop():
                    break
                swf_response = {}
                while not swf_response.get('taskToken'):
                    try:
                        swf_response = swf_client.poll_for_activity_task(
                            domain, task_list, identity=identity)
                    except ClientError:
                        # add a delay before retrying?
                        logger.exception('Error while polling for activities:')

                at = swf_response['activityType']
                decision = SWFActivityDecision(swf_client, swf_response['taskToken'])
                self(at['name'], at['version'], swf_response['input'], decision)
        except KeyboardInterrupt:
            pass
Ejemplo n.º 16
0
 def fail(self, reason):
     try:
         self.layer1.respond_activity_task_failed(
             reason=str(reason)[:256], task_token=str(self.token))
     except SWFResponseError:
         logger.exception('Error while failing the activity:')
         return False
     return True
Ejemplo n.º 17
0
 def fail(self, reason):
     try:
         self.swf_client.respond_activity_task_failed(self.token,
                                                      reason=reason)
     except ClientError:
         logger.exception('Error while failing the activity:')
         return False
     return True
Ejemplo n.º 18
0
 def flush(self):
     """Flush the decisions; no other decisions can be sent after that."""
     if self.closed:
         return
     self.closed = True
     try:
         self.swf_client.respond_decision_task_completed(
             self.token, decisions=self.decisions._data)
     except ClientError:
         logger.exception('Error while sending the decisions:')
Ejemplo n.º 19
0
 def flush(self):
     """Flush the decisions; no other decisions can be sent after that."""
     if self.closed:
         return
     self.closed = True
     try:
         self.layer1.respond_decision_task_completed(
             task_token=str(self.token), decisions=self.decisions._data)
     except SWFResponseError:
         logger.exception('Error while sending the decisions:')
Ejemplo n.º 20
0
 def finish(self, result):
     result = str(result)
     if len(result) > RESULT_SIZE:
         self.fail("Result too large: %s/%s" % (len(result), RESULT_SIZE))
     try:
         self.layer1.respond_activity_task_completed(
             result=result, task_token=str(self.token))
     except SWFResponseError:
         logger.exception('Error while finishing the activity:')
         return False
     return True
Ejemplo n.º 21
0
 def finish(self, result):
     result = str(result)
     if len(result) > RESULT_SIZE:
         self.fail("Result too large: %s/%s" % (len(result), RESULT_SIZE))
     try:
         self.swf_client.respond_activity_task_completed(self.token,
                                                         result=result)
     except ClientError:
         logger.exception('Error while finishing the activity:')
         return False
     return True
Ejemplo n.º 22
0
def _activity_wrapper(self, func, input_data, *extra_args):
    try:
        args, kwargs = self.deserialize_input(input_data)
    except Exception:
        logger.exception('Cannot deserialize the input:')
        raise ValueError('Cannot deserialize the input: %r' % (input_data,))
    result = func(*(tuple(extra_args) + tuple(args)), **kwargs)
    try:
        return self.serialize_result(result)
    except Exception:
        logger.exception('Cannot serialize the result:')
        raise ValueError('Cannot serialize the result: %r' % (result,))
Ejemplo n.º 23
0
def poll_first_page(layer1, domain, task_list, identity=None):
    """Return the response from loading the first page.

    In case of errors, empty responses or whatnot retry until a valid response.
    """
    swf_response = {}
    while 'taskToken' not in swf_response or not swf_response['taskToken']:
        try:
            swf_response = layer1.poll_for_decision_task(
                str(domain), str(task_list), str_or_none(identity))
        except SWFResponseError:
            logger.exception('Error while polling for decisions:')
    return swf_response
Ejemplo n.º 24
0
def poll_response_page(layer1, domain, task_list, token, identity=None):
    """Return a specific page. In case of errors retry a number of times."""
    swf_response = None
    for _ in range(7):  # give up after a limited number of retries
        try:
            swf_response = layer1.poll_for_decision_task(
                str(domain), str(task_list), str_or_none(identity),
                next_page_token=str(token))
            break
        except SWFResponseError:
            logger.exception('Error while polling for decision page:')
    else:
        raise _PaginationError()
    return swf_response
Ejemplo n.º 25
0
def _workflow_wrapper(self, factory, input_data, *extra_args):
    wf_kwargs = {}
    for dep_name, proxy in self.proxy_factory_registry.items():
        wf_kwargs[dep_name] = proxy(*extra_args)
    func = factory(**wf_kwargs)
    try:
        args, kwargs = self.deserialize_input(input_data)
    except Exception:
        logger.exception('Cannot deserialize the input:')
        raise ValueError('Cannot deserialize the input: %r' % (input_data,))
    result = func(*args, **kwargs)
    # Can't use directly isinstance(result, restart_type) because if the
    # result is a single result proxy it will be evaluated. This also
    # fixes another issue, on python2 isinstance() swallows any
    # exception while python3 it doesn't.
    if not is_result_proxy(result) and isinstance(result, restart_type):
        try:
            traversed_input, (error, placeholders) =  traverse_data(
                [result.args, result.kwargs])
        except Exception:
            logger.exception('Cannot traverse the restart arguments:')
            raise ValueError(
                'Cannot traverse the restart arguments: %r, %r' %
                result.args, result.kwargs)
        wait(error)  # raise if not None
        if placeholders:
            raise SuspendTask
        r_args, r_kwargs = traversed_input
        try:
            serialized_input = self.serialize_restart_input(*r_args, **r_kwargs)
        except Exception:
            logger.exception('Cannot serialize the restart arguments:')
            raise ValueError(
                'Cannot serialize the restart arguments: %r, %r' %
                result.args, result.kwargs)
        raise Restart(serialized_input)
    try:
        traversed_result, (error, placeholders) = traverse_data(result)
    except Exception:
        logger.exception('Cannot traverse the result:')
        raise ValueError('Cannot traverse the result: %r' % result)
    wait(error)
    if placeholders:
        raise SuspendTask
    try:
        return self.serialize_result(traversed_result)
    except Exception:
        logger.exception('Cannot serialize the result:')
        raise ValueError('Cannot serialize the result: %r' % (result,))
Ejemplo n.º 26
0
    def heartbeat(self, details=None):
        """Used to report that the activity is still making progress. Details
        about progress can be passed.

        :type details: str
        :param details: details about the progress made, None for not setting it

        :rtype: bool
        :returns: did someone heard my heartbeat?
        """
        try:
            self.swf_client.record_activity_task_heartbeat(self.token,
                                                           details=details)
        except ClientError:
            logger.exception('Error while sending the heartbeat:')
            return False
        return True
Ejemplo n.º 27
0
Archivo: config.py Proyecto: pbs/flowy
    def check_compatible(self, swf_client, domain, name, version):
        """Check if the remote config has the same defaults as this one.

        Raise SWFRegistrationError in case of SWF communication errors or
        incompatibility and ValueError if any configuration values can't be
        converted to the required types.

        :type swf_client: :class:`flowy.swf.client.SWFClient`
        :param swf_client: an implementation or duck typing of `SWFClient`
        :param domain: the domain name where to register the activity
        :param name: name of the activity
        :param version: version of the activity
        """
        d_t_l, d_h, d_sch_c, d_sch_s, d_s_c = self._cvt_values()
        try:
            a_descr = swf_client.describe_activity_type(domain, name, version)
            a_descr = a_descr['configuration']
        except ClientError as e:
            logger.exception('Error while checking activity compatibility:')
            raise SWFRegistrationError(e)
        r_d_t_l = a_descr.get('defaultTaskList', {}).get('name')
        if r_d_t_l != d_t_l:
            raise SWFRegistrationError(
                'Default task list for %r version %r does not match: %r != %r' %
                (name, version, r_d_t_l, d_t_l))
        r_d_h = a_descr.get('defaultTaskHeartbeatTimeout')
        if r_d_h != d_h:
            raise SWFRegistrationError(
                'Default heartbeat for %r version %r does not match: %r != %r' %
                (name, version, r_d_h, d_h))
        r_d_sch_c = a_descr.get('defaultTaskScheduleToCloseTimeout')
        if r_d_sch_c != d_sch_c:
            raise SWFRegistrationError(
                'Default schedule to close for %r version %r does not match: %r != %r'
                % (name, version, r_d_sch_c, d_sch_c))
        r_d_sch_s = a_descr.get('defaultTaskScheduleToStartTimeout')
        if r_d_sch_s != d_sch_s:
            raise SWFRegistrationError(
                'Default schedule to start for %r version %r does not match: %r != %r'
                % (name, version, r_d_sch_s, d_sch_s))
        r_d_s_c = a_descr.get('defaultTaskStartToCloseTimeout')
        if r_d_s_c != d_s_c:
            raise SWFRegistrationError(
                'Default start to close for %r version %r does not match: %r != %r'
                % (name, version, r_d_s_c, d_s_c))
Ejemplo n.º 28
0
    def check_compatible(self, swf_client, domain, name, version):
        """Check if the remote config has the same defaults as this one.

        Raise SWFRegistrationError in case of SWF communication errors or
        incompatibility and ValueError if any configuration values can't be
        converted to the required types.

        :type swf_client: :class:`flowy.swf.client.SWFClient`
        :param swf_client: an implementation or duck typing of `SWFClient`
        :param domain: the domain name where to register the activity
        :param name: name of the activity
        :param version: version of the activity
        """
        d_t_l, d_h, d_sch_c, d_sch_s, d_s_c = self._cvt_values()
        try:
            a_descr = swf_client.describe_activity_type(domain, name, version)
            a_descr = a_descr['configuration']
        except ClientError as e:
            logger.exception('Error while checking activity compatibility:')
            raise SWFRegistrationError(e)
        r_d_t_l = a_descr.get('defaultTaskList', {}).get('name')
        if r_d_t_l != d_t_l:
            raise SWFRegistrationError(
                'Default task list for %r version %r does not match: %r != %r'
                % (name, version, r_d_t_l, d_t_l))
        r_d_h = a_descr.get('defaultTaskHeartbeatTimeout')
        if r_d_h != d_h:
            raise SWFRegistrationError(
                'Default heartbeat for %r version %r does not match: %r != %r'
                % (name, version, r_d_h, d_h))
        r_d_sch_c = a_descr.get('defaultTaskScheduleToCloseTimeout')
        if r_d_sch_c != d_sch_c:
            raise SWFRegistrationError(
                'Default schedule to close for %r version %r does not match: %r != %r'
                % (name, version, r_d_sch_c, d_sch_c))
        r_d_sch_s = a_descr.get('defaultTaskScheduleToStartTimeout')
        if r_d_sch_s != d_sch_s:
            raise SWFRegistrationError(
                'Default schedule to start for %r version %r does not match: %r != %r'
                % (name, version, r_d_sch_s, d_sch_s))
        r_d_s_c = a_descr.get('defaultTaskStartToCloseTimeout')
        if r_d_s_c != d_s_c:
            raise SWFRegistrationError(
                'Default start to close for %r version %r does not match: %r != %r'
                % (name, version, r_d_s_c, d_s_c))
Ejemplo n.º 29
0
    def check_compatible(self, swf_client, domain, name, version):
        """Check if the remote config has the same defaults as this one.

        Raise SWFRegistrationError in case of SWF communication errors or
        incompatibility and ValueError if any configuration values can't be
        converted to the required types.

        :type swf_client: :class:`flowy.swf.client.SWFClient`
        :param swf_client: an implementation or duck typing of `SWFClient`
        :param domain: the domain name where to check the workflow
        :param name: name of the workflow
        :param version: version of the workflow
        """
        d_t_l, d_w_d, d_d_d, d_c_p = self._cvt_values()
        try:
            w_descr = swf_client.describe_workflow_type(
                str(domain), str(name), str(version))
            w_descr = w_descr['configuration']
        except ClientError as e:
            logger.exception('Error while checking workflow compatibility:')
            raise SWFRegistrationError(e)
        r_d_t_l = w_descr.get('defaultTaskList', {}).get('name')
        if r_d_t_l != d_t_l:
            raise SWFRegistrationError(
                'Default task list for %r version %r does not match: %r != %r'
                % (name, version, r_d_t_l, d_t_l))
        r_d_d_d = w_descr.get('defaultTaskStartToCloseTimeout')
        if r_d_d_d != d_d_d:
            raise SWFRegistrationError(
                'Default decision duration for %r version %r does not match: %r != %r'
                % (name, version, r_d_d_d, d_d_d))
        r_d_w_d = w_descr.get('defaultExecutionStartToCloseTimeout')
        if r_d_w_d != d_w_d:
            raise SWFRegistrationError(
                'Default workflow duration for %r version %r does not match: %r != %r'
                % (name, version, r_d_w_d, d_w_d))
        r_d_c_p = w_descr.get('defaultChildPolicy')
        if r_d_c_p != d_c_p:
            raise SWFRegistrationError(
                'Default child policy for %r version %r does not match: %r != %r'
                % (name, version, r_d_c_p, d_c_p))
Ejemplo n.º 30
0
Archivo: worker.py Proyecto: pbs/flowy
def poll_first_page(swf_client, domain, task_list, identity=None):
    """Return the response from loading the first page. In case of errors,
    empty responses or whatnot retry until a valid response.

    :type swf_client: :class:`SWFClient`
    :param swf_client: an implementation or duck typing of :class:`SWFClient`
    :param domain: the domain containing the task list to poll
    :param task_list: the task list from which to poll for events
    :param identity: an identity str of the request maker

    :rtype: dict[str, str|int|list|dict]
    :returns: a dict containing workflow information and list of events
    """
    swf_response = {}
    while not swf_response.get('taskToken'):
        try:
            swf_response = swf_client.poll_for_decision_task(domain, task_list,
                                                             identity=identity)
        except ClientError:
            logger.exception('Error while polling for decisions:')
    return swf_response
Ejemplo n.º 31
0
Archivo: config.py Proyecto: pbs/flowy
    def check_compatible(self, swf_client, domain, name, version):
        """Check if the remote config has the same defaults as this one.

        Raise SWFRegistrationError in case of SWF communication errors or
        incompatibility and ValueError if any configuration values can't be
        converted to the required types.

        :type swf_client: :class:`flowy.swf.client.SWFClient`
        :param swf_client: an implementation or duck typing of `SWFClient`
        :param domain: the domain name where to check the workflow
        :param name: name of the workflow
        :param version: version of the workflow
        """
        d_t_l, d_w_d, d_d_d, d_c_p = self._cvt_values()
        try:
            w_descr = swf_client.describe_workflow_type(str(domain), str(name), str(version))
            w_descr = w_descr['configuration']
        except ClientError as e:
            logger.exception('Error while checking workflow compatibility:')
            raise SWFRegistrationError(e)
        r_d_t_l = w_descr.get('defaultTaskList', {}).get('name')
        if r_d_t_l != d_t_l:
            raise SWFRegistrationError(
                'Default task list for %r version %r does not match: %r != %r' %
                (name, version, r_d_t_l, d_t_l))
        r_d_d_d = w_descr.get('defaultTaskStartToCloseTimeout')
        if r_d_d_d != d_d_d:
            raise SWFRegistrationError(
                'Default decision duration for %r version %r does not match: %r != %r'
                % (name, version, r_d_d_d, d_d_d))
        r_d_w_d = w_descr.get('defaultExecutionStartToCloseTimeout')
        if r_d_w_d != d_w_d:
            raise SWFRegistrationError(
                'Default workflow duration for %r version %r does not match: %r != %r'
                % (name, version, r_d_w_d, d_w_d))
        r_d_c_p = w_descr.get('defaultChildPolicy')
        if r_d_c_p != d_c_p:
            raise SWFRegistrationError(
                'Default child policy for %r version %r does not match: %r != %r' %
                (name, version, r_d_c_p, d_c_p))
Ejemplo n.º 32
0
    def check_compatible(self, swf_layer1, domain, name, version):
        """Check if the remote config has the same defaults as this one.

        Raise SWFRegistrationError in case of SWF communication errors or
        incompatibility and ValueError if any configuration values can't be
        converted to the required types.
        """
        d_t_l, d_h, d_sch_c, d_sch_s, d_s_c = self._cvt_values()
        try:
            a_descr = swf_layer1.describe_activity_type(str(domain), str(name), str(version))
            a_descr = a_descr['configuration']
        except SWFResponseError as e:
            logger.exception('Error while checking activity compatibility:')
            raise SWFRegistrationError(e)
        r_d_t_l = a_descr.get('defaultTaskList', {}).get('name')
        if r_d_t_l != d_t_l:
            raise SWFRegistrationError(
                'Default task list for %r version %r does not match: %r != %r' %
                (name, version, r_d_t_l, d_t_l))
        r_d_h = a_descr.get('defaultTaskHeartbeatTimeout')
        if r_d_h != d_h:
            raise SWFRegistrationError(
                'Default heartbeat for %r version %r does not match: %r != %r' %
                (name, version, r_d_h, d_h))
        r_d_sch_c = a_descr.get('defaultTaskScheduleToCloseTimeout')
        if r_d_sch_c != d_sch_c:
            raise SWFRegistrationError(
                'Default schedule to close for %r version %r does not match: %r != %r'
                % (name, version, r_d_sch_c, d_sch_c))
        r_d_sch_s = a_descr.get('defaultTaskScheduleToStartTimeout')
        if r_d_sch_s != d_sch_s:
            raise SWFRegistrationError(
                'Default schedule to start for %r version %r does not match: %r != %r'
                % (name, version, r_d_sch_s, d_sch_s))
        r_d_s_c = a_descr.get('defaultTaskStartToCloseTimeout')
        if r_d_s_c != d_s_c:
            raise SWFRegistrationError(
                'Default start to close for %r version %r does not match: %r != %r'
                % (name, version, r_d_s_c, d_s_c))
Ejemplo n.º 33
0
def poll_first_page(swf_client, domain, task_list, identity=None):
    """Return the response from loading the first page. In case of errors,
    empty responses or whatnot retry until a valid response.

    :type swf_client: :class:`SWFClient`
    :param swf_client: an implementation or duck typing of :class:`SWFClient`
    :param domain: the domain containing the task list to poll
    :param task_list: the task list from which to poll for events
    :param identity: an identity str of the request maker

    :rtype: dict[str, str|int|list|dict]
    :returns: a dict containing workflow information and list of events
    """
    swf_response = {}
    while not swf_response.get('taskToken'):
        try:
            swf_response = swf_client.poll_for_decision_task(domain,
                                                             task_list,
                                                             identity=identity)
        except ClientError:
            logger.exception('Error while polling for decisions:')
    return swf_response
Ejemplo n.º 34
0
Archivo: worker.py Proyecto: pbs/flowy
def poll_page(swf_client, domain, task_list, token, identity=None):
    """Return a specific page. In case of errors retry a number of times.

    :type swf_client: :class:`SWFClient`
    :param swf_client: an implementation or duck typing of :class:`SWFClient`
    :param domain: the domain containing the task list to poll
    :param task_list: the task list from which to poll for events
    :param token: the token string for the requested page
    :param identity: an identity str of the request maker

    :rtype: dict[str, str|int|list|dict]
    :returns: a dict containing workflow information and list of events
    """
    for _ in range(7):  # give up after a limited number of retries
        try:
            swf_response = swf_client.poll_for_decision_task(
                domain, task_list, identity=identity, next_page_token=token)
            break
        except ClientError:
            logger.exception('Error while polling for decision page:')
    else:
        raise _PaginationError()
    return swf_response
Ejemplo n.º 35
0
def poll_page(swf_client, domain, task_list, token, identity=None):
    """Return a specific page. In case of errors retry a number of times.

    :type swf_client: :class:`SWFClient`
    :param swf_client: an implementation or duck typing of :class:`SWFClient`
    :param domain: the domain containing the task list to poll
    :param task_list: the task list from which to poll for events
    :param token: the token string for the requested page
    :param identity: an identity str of the request maker

    :rtype: dict[str, str|int|list|dict]
    :returns: a dict containing workflow information and list of events
    """
    for _ in range(7):  # give up after a limited number of retries
        try:
            swf_response = swf_client.poll_for_decision_task(
                domain, task_list, identity=identity, next_page_token=token)
            break
        except ClientError:
            logger.exception('Error while polling for decision page:')
    else:
        raise _PaginationError()
    return swf_response
Ejemplo n.º 36
0
    def __call__(self, *args, **kwargs):
        """Consult the execution history for results or schedule a new task.

        This is method gets called from the user workflow code.
        When calling it, the task it refers to can be in one of the following
        states: RUNNING, READY, FAILED, TIMEDOUT or NOTSCHEDULED.

        * If the task is RUNNING this returns a Placeholder. The Placeholder
          interrupts the workflow execution if its result is accessed by
          raising a SuspendTask exception.
        * If the task is READY this returns a Result object. Calling the result
          method on this object will just return the final value the task
          produced.
        * If the task is FAILED this returns an Error object. Calling the
          result method on this object will raise a TaskError exception
          containing the error message set by the task.
        * In case of a TIMEOUT this returns an Timeout object. Calling the
          result method on this object will raise TaskTimedout exception, a
          subclass of TaskError.
        * If the task was NOTSCHEDULED yet:
            * If any errors in arguments, propagate the error by returning
              another error.
            * If any placeholders in arguments, don't do anything because there
              are unresolved dependencies.
            * Finally, if all the arguments look OK, schedule it for execution.
        """
        task_exec_history = self.task_exec_history
        call_number = self.call_number
        self.call_number += 1
        r = placeholder()
        for retry_number, delay in enumerate(self.retry):
            if task_exec_history.is_timeout(call_number, retry_number):
                continue
            if task_exec_history.is_running(call_number, retry_number):
                break  # result = Placehloder
            if task_exec_history.has_result(call_number, retry_number):
                value = task_exec_history.result(call_number, retry_number)
                order = task_exec_history.order(call_number, retry_number)
                try:
                    value = self.deserialize_result(value)
                except Exception as e:
                    logger.exception(
                        'Error while deserializing the activity result:')
                    self.task_decision.fail(e)
                    break  # result = Placeholder
                r = result(value, order)
                break
            if task_exec_history.is_error(call_number, retry_number):
                err = task_exec_history.error(call_number, retry_number)
                order = task_exec_history.order(call_number, retry_number)
                r = error(err, order)
                break
            traversed_args, (err, placeholders) = traverse_data([args, kwargs])
            if err:
                r = copy_result_proxy(err)
                break
            if placeholders:
                break  # result = Placeholder
            t_args, t_kwargs = traversed_args
            try:
                input_data = self.serialize_input(*t_args, **t_kwargs)
            except Exception as e:
                logger.exception('Error while serializing the task input:')
                self.task_decision.fail(e)
                break  # result = Placeholder
            self.task_decision.schedule(call_number, retry_number, delay,
                                        input_data)
            break  # result = Placeholder
        else:
            # No retries left, it must be a timeout
            order = task_exec_history.order(call_number, retry_number)
            r = timeout(order)
        return r
Ejemplo n.º 37
0
    def __call__(self, *args, **kwargs):
        """Consult the execution history for results or schedule a new task.

        This is method gets called from the user workflow code.
        When calling it, the task it refers to can be in one of the following
        states: RUNNING, READY, FAILED, TIMEDOUT or NOTSCHEDULED.

        * If the task is RUNNING this returns a Placeholder. The Placeholder
          interrupts the workflow execution if its result is accessed by
          raising a SuspendTask exception.
        * If the task is READY this returns a Result object. Calling the result
          method on this object will just return the final value the task
          produced.
        * If the task is FAILED this returns an Error object. Calling the
          result method on this object will raise a TaskError exception
          containing the error message set by the task.
        * In case of a TIMEOUT this returns an Timeout object. Calling the
          result method on this object will raise TaskTimedout exception, a
          subclass of TaskError.
        * If the task was NOTSCHEDULED yet:
            * If any errors in arguments, propagate the error by returning
              another error.
            * If any placeholders in arguments, don't do anything because there
              are unresolved dependencies.
            * Finally, if all the arguments look OK, schedule it for execution.
        """
        task_exec_history = self.task_exec_history
        call_number = self.call_number
        self.call_number += 1
        r = placeholder()
        for retry_number, delay in enumerate(self.retry):
            if task_exec_history.is_timeout(call_number, retry_number):
                continue
            if task_exec_history.is_running(call_number, retry_number):
                break  # result = Placehloder
            if task_exec_history.has_result(call_number, retry_number):
                value = task_exec_history.result(call_number, retry_number)
                order = task_exec_history.order(call_number, retry_number)
                try:
                    value = self.deserialize_result(value)
                except Exception as e:
                    logger.exception('Error while deserializing the activity result:')
                    self.task_decision.fail(e)
                    break  # result = Placeholder
                r = result(value, order)
                break
            if task_exec_history.is_error(call_number, retry_number):
                err = task_exec_history.error(call_number, retry_number)
                order = task_exec_history.order(call_number, retry_number)
                r = error(err, order)
                break
            traversed_args, (err, placeholders) = traverse_data([args, kwargs])
            if err:
                r = copy_result_proxy(err)
                break
            if placeholders:
                break  # result = Placeholder
            t_args, t_kwargs = traversed_args
            try:
                input_data = self.serialize_input(*t_args, **t_kwargs)
            except Exception as e:
                logger.exception('Error while serializing the task input:')
                self.task_decision.fail(e)
                break  # result = Placeholder
            self.task_decision.schedule(call_number, retry_number, delay, input_data)
            break  # result = Placeholder
        else:
            # No retries left, it must be a timeout
            order = task_exec_history.order(call_number, retry_number)
            r = timeout(order)
        return r