def try_register_remote(self, swf_client, domain, name, version):
"""Register the activity remotely.
Returns True if registration is successful and False if another
activity with the same name is already registered.
Raise SWFRegistrationError in case of SWF communication errors and
ValueError if any configuration values can't be converted to the
required types.
:type swf_client: :class:`flowy.swf.client.SWFClient`
:param swf_client: an implementation or duck typing of `SWFClient`
:param domain: the domain name where to register the activity
:param name: name of the activity
:param version: version of the activity
:rtype: bool
"""
d_t_l, d_h, d_sch_c, d_sch_s, d_s_c = self._cvt_values()
try:
swf_client.register_activity_type(domain, name, version,
default_task_list=d_t_l,
default_heartbeat_timeout=d_h,
default_exec_timeout=d_s_c,
default_start_timeout=d_sch_s,
default_close_timeout=d_sch_c)
except ClientError as e:
if e.response['Error']['Code'] == 'TypeAlreadyExistsFault':
return False
logger.exception('Error while registering the activity:')
raise SWFRegistrationError(e)
return True
def try_register_remote(self, swf_layer1, domain, name, version):
"""Register the workflow remotely.
Returns True if registration is successful and False if another
workflow with the same name is already registered.
A name should be set before calling this method or RuntimeError is
raised.
Raise SWFRegistrationError in case of SWF communication errors and
ValueError if any configuration values can't be converted to the
required types.
"""
d_t_l, d_w_d, d_d_d, d_c_p = self._cvt_values()
try:
swf_layer1.register_workflow_type(
str(domain),
name=str(name),
version=str(version),
task_list=d_t_l,
default_execution_start_to_close_timeout=d_w_d,
default_task_start_to_close_timeout=d_d_d,
default_child_policy=d_c_p)
except SWFTypeAlreadyExistsError:
return False
except SWFResponseError as e:
if 'TypeAlreadyExistsFault' in str(e): # eucalyptus
return False
logger.exception('Error while registering the workflow:')
raise SWFRegistrationError(e)
return True
def try_register_remote(self, swf_client, domain, name, version):
"""Register the workflow remotely.
Returns True if registration is successful and False if another
workflow with the same name is already registered.
A name should be set before calling this method or RuntimeError is
raised.
Raise SWFRegistrationError in case of SWF communication errors and
ValueError if any configuration values can't be converted to the
required types.
:type swf_client: :class:`flowy.swf.client.SWFClient`
:param swf_client: an implementation or duck typing of `SWFClient`
:param domain: the domain name where to register the workflow
:param name: name of the workflow
:param version: version of the workflow
:rtype: bool
"""
d_t_l, d_w_d, d_d_d, d_c_p = self._cvt_values()
try:
swf_client.register_workflow_type(domain,
name,
version,
default_task_list=d_t_l,
default_task_timeout=d_d_d,
default_exec_timeout=d_w_d,
default_child_policy=d_c_p)
except ClientError as e:
if e.response['Error']['Code'] == 'TypeAlreadyExistsFault':
return False
logger.exception('Error while registering the workflow:')
raise SWFRegistrationError(e)
return True
def try_register_remote(self, swf_layer1, domain, name, version):
"""Register the activity remotely.
Returns True if registration is successful and False if another
activity with the same name is already registered.
Raise SWFRegistrationError in case of SWF communication errors and
ValueError if any configuration values can't be converted to the
required types.
"""
d_t_l, d_h, d_sch_c, d_sch_s, d_s_c = self._cvt_values()
try:
swf_layer1.register_activity_type(
str(domain),
name=str(name),
version=str(version),
task_list=d_t_l,
default_task_heartbeat_timeout=d_h,
default_task_schedule_to_close_timeout=d_sch_c,
default_task_schedule_to_start_timeout=d_sch_s,
default_task_start_to_close_timeout=d_s_c)
except SWFTypeAlreadyExistsError:
return False
except SWFResponseError as e:
if 'TypeAlreadyExistsFault' in str(e): # eucalyptus
return False
logger.exception('Error while registering the activity:')
raise SWFRegistrationError(e)
return True
def heartbeat(self):
try:
self.layer1.record_activity_task_heartbeat(task_token=str(self.token))
except SWFResponseError:
logger.exception('Error while sending the heartbeat:')
return False
return True
def try_register_remote(self, swf_client, domain, name, version):
"""Register the activity remotely.
Returns True if registration is successful and False if another
activity with the same name is already registered.
Raise SWFRegistrationError in case of SWF communication errors and
ValueError if any configuration values can't be converted to the
required types.
:type swf_client: :class:`flowy.swf.client.SWFClient`
:param swf_client: an implementation or duck typing of `SWFClient`
:param domain: the domain name where to register the activity
:param name: name of the activity
:param version: version of the activity
:rtype: bool
"""
d_t_l, d_h, d_sch_c, d_sch_s, d_s_c = self._cvt_values()
try:
swf_client.register_activity_type(domain,
name,
version,
default_task_list=d_t_l,
default_heartbeat_timeout=d_h,
default_exec_timeout=d_s_c,
default_start_timeout=d_sch_s,
default_close_timeout=d_sch_c)
except ClientError as e:
if e.response['Error']['Code'] == 'TypeAlreadyExistsFault':
return False
logger.exception('Error while registering the activity:')
raise SWFRegistrationError(e)
return True
def __call__(self, key, input_data, decision, *extra_args):
"""Execute the wrapped func registered with key passing the input_data.
Any exra_args are also passed along to the wrapped_key.
The actual actions are dispatched to the decision object and can be one
of:
* flush() - nothing to do, any pending actions should be commited
* fail(e) - ignore pending actions, fail the execution
* finish(e) - ignore pending actions, complete the execution
* restart(serialized_input) - ignore pending actions, restart the execution
"""
try:
wrapped_func = self.registry[key]
except KeyError:
logger.error("Colud not find implementation for key: %r", (key,))
return # Let it timeout
try:
serialized_result = wrapped_func(input_data, *extra_args)
except SuspendTask:
decision.flush()
except TaskError as e:
logger.exception('Unhandled task error in task:')
decision.fail(e)
except Restart as e:
decision.restart(e.input_data)
except Exception as e:
logger.exception('Unhandled exception in task:')
decision.fail(e)
else:
decision.finish(serialized_result)
def run_forever(self, domain, task_list,
layer1=None,
setup_log=True,
register_remote=True,
identity=None):
"""Same as SWFWorkflowWorker.run_forever but for activities."""
if setup_log:
setup_default_logger()
identity = identity if identity is not None else default_identity()
identity = str(identity)[:_IDENTITY_SIZE]
layer1 = layer1 if layer1 is not None else Layer1()
if register_remote:
self.register_remote(layer1, domain)
try:
while 1:
if self.break_loop():
break
swf_response = {}
while ('taskToken' not in swf_response or not swf_response['taskToken']):
try:
swf_response = layer1.poll_for_activity_task(
domain=domain,
task_list=task_list,
identity=identity)
except SWFResponseError:
# add a delay before retrying?
logger.exception('Error while polling for activities:')
at = swf_response['activityType']
decision = SWFActivityDecision(layer1, swf_response['taskToken'])
self(at['name'], at['version'], swf_response['input'], decision)
except KeyboardInterrupt:
pass
def __call__(self, key, input_data, decision, *extra_args):
"""Execute the wrapped func registered with key passing the input_data.
Any exra_args are also passed along to the wrapped_key.
The actual actions are dispatched to the decision object and can be one
of:
* flush() - nothing to do, any pending actions should be commited
* fail(e) - ignore pending actions, fail the execution
* finish(e) - ignore pending actions, complete the execution
* restart(serialized_input) - ignore pending actions, restart the execution
"""
try:
wrapped_func = self.registry[key]
except KeyError:
logger.error("Colud not find implementation for key: %r", (key, ))
return # Let it timeout
try:
serialized_result = wrapped_func(input_data, *extra_args)
except SuspendTask: # only from workflows
decision.flush()
except TaskError as e: # only from workflows
logger.exception('Unhandled task error in task:')
decision.fail(e)
except Restart as e: # only from workflows
decision.restart(e.input_data)
except Exception as e:
logger.exception('Unhandled exception in task:')
decision.fail(e)
else:
decision.finish(serialized_result)
def run_forever(self,
domain,
task_list,
swf_client=None,
setup_log=True,
register_remote=True,
identity=None):
"""Same as SWFWorkflowWorker.run_forever but for activities."""
if setup_log:
setup_default_logger()
identity = default_identity() if identity is None else identity
swf_client = SWFClient() if swf_client is None else swf_client
if register_remote:
self.register_remote(swf_client, domain)
try:
while 1:
if self.break_loop():
break
swf_response = {}
while not swf_response.get('taskToken'):
try:
swf_response = swf_client.poll_for_activity_task(
domain, task_list, identity=identity)
except ClientError:
# add a delay before retrying?
logger.exception('Error while polling for activities:')
at = swf_response['activityType']
decision = SWFActivityDecision(swf_client,
swf_response['taskToken'])
self(at['name'], at['version'], swf_response['input'],
decision)
except KeyboardInterrupt:
pass
def check_compatible(self, swf_layer1, domain, name, version):
"""Check if the remote config has the same defaults as this one.
Raise SWFRegistrationError in case of SWF communication errors or
incompatibility and ValueError if any configuration values can't be
converted to the required types.
"""
d_t_l, d_w_d, d_d_d, d_c_p = self._cvt_values()
try:
w_descr = swf_layer1.describe_workflow_type(str(domain), str(name), str(version))
w_descr = w_descr['configuration']
except SWFResponseError as e:
logger.exception('Error while checking workflow compatibility:')
raise SWFRegistrationError(e)
r_d_t_l = w_descr.get('defaultTaskList', {}).get('name')
if r_d_t_l != d_t_l:
raise SWFRegistrationError(
'Default task list for %r version %r does not match: %r != %r' %
(name, version, r_d_t_l, d_t_l))
r_d_d_d = w_descr.get('defaultTaskStartToCloseTimeout')
if r_d_d_d != d_d_d:
raise SWFRegistrationError(
'Default decision duration for %r version %r does not match: %r != %r'
% (name, version, r_d_d_d, d_d_d))
r_d_w_d = w_descr.get('defaultExecutionStartToCloseTimeout')
if r_d_w_d != d_w_d:
raise SWFRegistrationError(
'Default workflow duration for %r version %r does not match: %r != %r'
% (name, version, r_d_w_d, d_w_d))
r_d_c_p = w_descr.get('defaultChildPolicy')
if r_d_c_p != d_c_p:
raise SWFRegistrationError(
'Default child policy for %r version %r does not match: %r != %r' %
(name, version, r_d_c_p, d_c_p))
def try_register_remote(self, swf_client, domain, name, version):
"""Register the workflow remotely.
Returns True if registration is successful and False if another
workflow with the same name is already registered.
A name should be set before calling this method or RuntimeError is
raised.
Raise SWFRegistrationError in case of SWF communication errors and
ValueError if any configuration values can't be converted to the
required types.
:type swf_client: :class:`flowy.swf.client.SWFClient`
:param swf_client: an implementation or duck typing of `SWFClient`
:param domain: the domain name where to register the workflow
:param name: name of the workflow
:param version: version of the workflow
:rtype: bool
"""
d_t_l, d_w_d, d_d_d, d_c_p = self._cvt_values()
try:
swf_client.register_workflow_type(domain, name, version,
default_task_list=d_t_l,
default_task_timeout=d_d_d,
default_exec_timeout=d_w_d,
default_child_policy=d_c_p)
except ClientError as e:
if e.response['Error']['Code'] == 'TypeAlreadyExistsFault':
return False
logger.exception('Error while registering the workflow:')
raise SWFRegistrationError(e)
return True
def really_start(*args, **kwargs):
"""Use this function to start a workflow by passing in the args."""
swf = swf_client if swf_client is not None else SWFClient()
l_wid = wid # closure hack
if l_wid is None:
l_wid = uuid.uuid4()
if serialize_input is None:
input_data = Proxy.serialize_input(*args, **kwargs)
else:
input_data = serialize_input(*args, **kwargs)
if len(input_data) > INPUT_SIZE:
logger.error(
"Input too large: %s/%s" % (len(input_data), INPUT_SIZE))
raise ValueError('Input too large.')
try:
r = swf.start_workflow_execution(
domain, l_wid, name, version, input=input_data,
priority=priority, task_list=task_list,
execution_start_to_close_timeout=task_duration,
task_start_to_close_timeout=workflow_duration,
child_policy=child_policy, tags=tags, lambda_role=lambda_role)
except ClientError as e:
logger.exception('Error while starting the workflow:')
raise e
return r['runId']
def really_start(*args, **kwargs):
"""Use this function to start a workflow by passing in the args."""
l1 = layer1 if layer1 is not None else Layer1()
l_wid = wid # closue hack
if l_wid is None:
l_wid = uuid.uuid4()
if serialize_input is None:
input_data = Proxy.serialize_input(*args, **kwargs)
else:
input_data = serialize_input(*args, **kwargs)
if len(input_data) > INPUT_SIZE:
logger.error("Input too large: %s/%s" % (len(input_data), INPUT_SIZE))
raise ValueError('Input too large.')
try:
r = l1.start_workflow_execution(
str(domain), str(l_wid), str(name), str(version),
task_list=str_or_none(task_list),
execution_start_to_close_timeout=str_or_none(workflow_duration),
task_start_to_close_timeout=str_or_none(decision_duration),
input=str(input_data),
child_policy=cp_encode(child_policy),
tag_list=tags_encode(tags))
except SWFResponseError:
logger.exception('Error while starting the workflow:')
raise RuntimeError('Cannot start the workflow.')
return r['runId']
def run_forever(self, domain, task_list,
swf_client=None,
setup_log=True,
register_remote=True,
identity=None):
"""Same as SWFWorkflowWorker.run_forever but for activities."""
if setup_log:
setup_default_logger()
identity = default_identity() if identity is None else identity
swf_client = SWFClient() if swf_client is None else swf_client
if register_remote:
self.register_remote(swf_client, domain)
try:
while 1:
if self.break_loop():
break
swf_response = {}
while not swf_response.get('taskToken'):
try:
swf_response = swf_client.poll_for_activity_task(
domain, task_list, identity=identity)
except ClientError:
# add a delay before retrying?
logger.exception('Error while polling for activities:')
at = swf_response['activityType']
decision = SWFActivityDecision(swf_client, swf_response['taskToken'])
self(at['name'], at['version'], swf_response['input'], decision)
except KeyboardInterrupt:
pass
def fail(self, reason):
try:
self.layer1.respond_activity_task_failed(
reason=str(reason)[:256], task_token=str(self.token))
except SWFResponseError:
logger.exception('Error while failing the activity:')
return False
return True
def fail(self, reason):
try:
self.swf_client.respond_activity_task_failed(self.token,
reason=reason)
except ClientError:
logger.exception('Error while failing the activity:')
return False
return True
def flush(self):
"""Flush the decisions; no other decisions can be sent after that."""
if self.closed:
return
self.closed = True
try:
self.swf_client.respond_decision_task_completed(
self.token, decisions=self.decisions._data)
except ClientError:
logger.exception('Error while sending the decisions:')
def flush(self):
"""Flush the decisions; no other decisions can be sent after that."""
if self.closed:
return
self.closed = True
try:
self.layer1.respond_decision_task_completed(
task_token=str(self.token), decisions=self.decisions._data)
except SWFResponseError:
logger.exception('Error while sending the decisions:')
def finish(self, result):
result = str(result)
if len(result) > RESULT_SIZE:
self.fail("Result too large: %s/%s" % (len(result), RESULT_SIZE))
try:
self.layer1.respond_activity_task_completed(
result=result, task_token=str(self.token))
except SWFResponseError:
logger.exception('Error while finishing the activity:')
return False
return True
def finish(self, result):
result = str(result)
if len(result) > RESULT_SIZE:
self.fail("Result too large: %s/%s" % (len(result), RESULT_SIZE))
try:
self.swf_client.respond_activity_task_completed(self.token,
result=result)
except ClientError:
logger.exception('Error while finishing the activity:')
return False
return True
def _activity_wrapper(self, func, input_data, *extra_args):
try:
args, kwargs = self.deserialize_input(input_data)
except Exception:
logger.exception('Cannot deserialize the input:')
raise ValueError('Cannot deserialize the input: %r' % (input_data,))
result = func(*(tuple(extra_args) + tuple(args)), **kwargs)
try:
return self.serialize_result(result)
except Exception:
logger.exception('Cannot serialize the result:')
raise ValueError('Cannot serialize the result: %r' % (result,))
def poll_first_page(layer1, domain, task_list, identity=None):
"""Return the response from loading the first page.
In case of errors, empty responses or whatnot retry until a valid response.
"""
swf_response = {}
while 'taskToken' not in swf_response or not swf_response['taskToken']:
try:
swf_response = layer1.poll_for_decision_task(
str(domain), str(task_list), str_or_none(identity))
except SWFResponseError:
logger.exception('Error while polling for decisions:')
return swf_response
def poll_response_page(layer1, domain, task_list, token, identity=None):
"""Return a specific page. In case of errors retry a number of times."""
swf_response = None
for _ in range(7): # give up after a limited number of retries
try:
swf_response = layer1.poll_for_decision_task(
str(domain), str(task_list), str_or_none(identity),
next_page_token=str(token))
break
except SWFResponseError:
logger.exception('Error while polling for decision page:')
else:
raise _PaginationError()
return swf_response
def _workflow_wrapper(self, factory, input_data, *extra_args):
wf_kwargs = {}
for dep_name, proxy in self.proxy_factory_registry.items():
wf_kwargs[dep_name] = proxy(*extra_args)
func = factory(**wf_kwargs)
try:
args, kwargs = self.deserialize_input(input_data)
except Exception:
logger.exception('Cannot deserialize the input:')
raise ValueError('Cannot deserialize the input: %r' % (input_data,))
result = func(*args, **kwargs)
# Can't use directly isinstance(result, restart_type) because if the
# result is a single result proxy it will be evaluated. This also
# fixes another issue, on python2 isinstance() swallows any
# exception while python3 it doesn't.
if not is_result_proxy(result) and isinstance(result, restart_type):
try:
traversed_input, (error, placeholders) = traverse_data(
[result.args, result.kwargs])
except Exception:
logger.exception('Cannot traverse the restart arguments:')
raise ValueError(
'Cannot traverse the restart arguments: %r, %r' %
result.args, result.kwargs)
wait(error) # raise if not None
if placeholders:
raise SuspendTask
r_args, r_kwargs = traversed_input
try:
serialized_input = self.serialize_restart_input(*r_args, **r_kwargs)
except Exception:
logger.exception('Cannot serialize the restart arguments:')
raise ValueError(
'Cannot serialize the restart arguments: %r, %r' %
result.args, result.kwargs)
raise Restart(serialized_input)
try:
traversed_result, (error, placeholders) = traverse_data(result)
except Exception:
logger.exception('Cannot traverse the result:')
raise ValueError('Cannot traverse the result: %r' % result)
wait(error)
if placeholders:
raise SuspendTask
try:
return self.serialize_result(traversed_result)
except Exception:
logger.exception('Cannot serialize the result:')
raise ValueError('Cannot serialize the result: %r' % (result,))
def heartbeat(self, details=None):
"""Used to report that the activity is still making progress. Details
about progress can be passed.
:type details: str
:param details: details about the progress made, None for not setting it
:rtype: bool
:returns: did someone heard my heartbeat?
"""
try:
self.swf_client.record_activity_task_heartbeat(self.token,
details=details)
except ClientError:
logger.exception('Error while sending the heartbeat:')
return False
return True
def check_compatible(self, swf_client, domain, name, version):
"""Check if the remote config has the same defaults as this one.
Raise SWFRegistrationError in case of SWF communication errors or
incompatibility and ValueError if any configuration values can't be
converted to the required types.
:type swf_client: :class:`flowy.swf.client.SWFClient`
:param swf_client: an implementation or duck typing of `SWFClient`
:param domain: the domain name where to register the activity
:param name: name of the activity
:param version: version of the activity
"""
d_t_l, d_h, d_sch_c, d_sch_s, d_s_c = self._cvt_values()
try:
a_descr = swf_client.describe_activity_type(domain, name, version)
a_descr = a_descr['configuration']
except ClientError as e:
logger.exception('Error while checking activity compatibility:')
raise SWFRegistrationError(e)
r_d_t_l = a_descr.get('defaultTaskList', {}).get('name')
if r_d_t_l != d_t_l:
raise SWFRegistrationError(
'Default task list for %r version %r does not match: %r != %r' %
(name, version, r_d_t_l, d_t_l))
r_d_h = a_descr.get('defaultTaskHeartbeatTimeout')
if r_d_h != d_h:
raise SWFRegistrationError(
'Default heartbeat for %r version %r does not match: %r != %r' %
(name, version, r_d_h, d_h))
r_d_sch_c = a_descr.get('defaultTaskScheduleToCloseTimeout')
if r_d_sch_c != d_sch_c:
raise SWFRegistrationError(
'Default schedule to close for %r version %r does not match: %r != %r'
% (name, version, r_d_sch_c, d_sch_c))
r_d_sch_s = a_descr.get('defaultTaskScheduleToStartTimeout')
if r_d_sch_s != d_sch_s:
raise SWFRegistrationError(
'Default schedule to start for %r version %r does not match: %r != %r'
% (name, version, r_d_sch_s, d_sch_s))
r_d_s_c = a_descr.get('defaultTaskStartToCloseTimeout')
if r_d_s_c != d_s_c:
raise SWFRegistrationError(
'Default start to close for %r version %r does not match: %r != %r'
% (name, version, r_d_s_c, d_s_c))
def check_compatible(self, swf_client, domain, name, version):
"""Check if the remote config has the same defaults as this one.
Raise SWFRegistrationError in case of SWF communication errors or
incompatibility and ValueError if any configuration values can't be
converted to the required types.
:type swf_client: :class:`flowy.swf.client.SWFClient`
:param swf_client: an implementation or duck typing of `SWFClient`
:param domain: the domain name where to register the activity
:param name: name of the activity
:param version: version of the activity
"""
d_t_l, d_h, d_sch_c, d_sch_s, d_s_c = self._cvt_values()
try:
a_descr = swf_client.describe_activity_type(domain, name, version)
a_descr = a_descr['configuration']
except ClientError as e:
logger.exception('Error while checking activity compatibility:')
raise SWFRegistrationError(e)
r_d_t_l = a_descr.get('defaultTaskList', {}).get('name')
if r_d_t_l != d_t_l:
raise SWFRegistrationError(
'Default task list for %r version %r does not match: %r != %r'
% (name, version, r_d_t_l, d_t_l))
r_d_h = a_descr.get('defaultTaskHeartbeatTimeout')
if r_d_h != d_h:
raise SWFRegistrationError(
'Default heartbeat for %r version %r does not match: %r != %r'
% (name, version, r_d_h, d_h))
r_d_sch_c = a_descr.get('defaultTaskScheduleToCloseTimeout')
if r_d_sch_c != d_sch_c:
raise SWFRegistrationError(
'Default schedule to close for %r version %r does not match: %r != %r'
% (name, version, r_d_sch_c, d_sch_c))
r_d_sch_s = a_descr.get('defaultTaskScheduleToStartTimeout')
if r_d_sch_s != d_sch_s:
raise SWFRegistrationError(
'Default schedule to start for %r version %r does not match: %r != %r'
% (name, version, r_d_sch_s, d_sch_s))
r_d_s_c = a_descr.get('defaultTaskStartToCloseTimeout')
if r_d_s_c != d_s_c:
raise SWFRegistrationError(
'Default start to close for %r version %r does not match: %r != %r'
% (name, version, r_d_s_c, d_s_c))
def check_compatible(self, swf_client, domain, name, version):
"""Check if the remote config has the same defaults as this one.
Raise SWFRegistrationError in case of SWF communication errors or
incompatibility and ValueError if any configuration values can't be
converted to the required types.
:type swf_client: :class:`flowy.swf.client.SWFClient`
:param swf_client: an implementation or duck typing of `SWFClient`
:param domain: the domain name where to check the workflow
:param name: name of the workflow
:param version: version of the workflow
"""
d_t_l, d_w_d, d_d_d, d_c_p = self._cvt_values()
try:
w_descr = swf_client.describe_workflow_type(
str(domain), str(name), str(version))
w_descr = w_descr['configuration']
except ClientError as e:
logger.exception('Error while checking workflow compatibility:')
raise SWFRegistrationError(e)
r_d_t_l = w_descr.get('defaultTaskList', {}).get('name')
if r_d_t_l != d_t_l:
raise SWFRegistrationError(
'Default task list for %r version %r does not match: %r != %r'
% (name, version, r_d_t_l, d_t_l))
r_d_d_d = w_descr.get('defaultTaskStartToCloseTimeout')
if r_d_d_d != d_d_d:
raise SWFRegistrationError(
'Default decision duration for %r version %r does not match: %r != %r'
% (name, version, r_d_d_d, d_d_d))
r_d_w_d = w_descr.get('defaultExecutionStartToCloseTimeout')
if r_d_w_d != d_w_d:
raise SWFRegistrationError(
'Default workflow duration for %r version %r does not match: %r != %r'
% (name, version, r_d_w_d, d_w_d))
r_d_c_p = w_descr.get('defaultChildPolicy')
if r_d_c_p != d_c_p:
raise SWFRegistrationError(
'Default child policy for %r version %r does not match: %r != %r'
% (name, version, r_d_c_p, d_c_p))
def poll_first_page(swf_client, domain, task_list, identity=None):
"""Return the response from loading the first page. In case of errors,
empty responses or whatnot retry until a valid response.
:type swf_client: :class:`SWFClient`
:param swf_client: an implementation or duck typing of :class:`SWFClient`
:param domain: the domain containing the task list to poll
:param task_list: the task list from which to poll for events
:param identity: an identity str of the request maker
:rtype: dict[str, str|int|list|dict]
:returns: a dict containing workflow information and list of events
"""
swf_response = {}
while not swf_response.get('taskToken'):
try:
swf_response = swf_client.poll_for_decision_task(domain, task_list,
identity=identity)
except ClientError:
logger.exception('Error while polling for decisions:')
return swf_response
def check_compatible(self, swf_client, domain, name, version):
"""Check if the remote config has the same defaults as this one.
Raise SWFRegistrationError in case of SWF communication errors or
incompatibility and ValueError if any configuration values can't be
converted to the required types.
:type swf_client: :class:`flowy.swf.client.SWFClient`
:param swf_client: an implementation or duck typing of `SWFClient`
:param domain: the domain name where to check the workflow
:param name: name of the workflow
:param version: version of the workflow
"""
d_t_l, d_w_d, d_d_d, d_c_p = self._cvt_values()
try:
w_descr = swf_client.describe_workflow_type(str(domain), str(name), str(version))
w_descr = w_descr['configuration']
except ClientError as e:
logger.exception('Error while checking workflow compatibility:')
raise SWFRegistrationError(e)
r_d_t_l = w_descr.get('defaultTaskList', {}).get('name')
if r_d_t_l != d_t_l:
raise SWFRegistrationError(
'Default task list for %r version %r does not match: %r != %r' %
(name, version, r_d_t_l, d_t_l))
r_d_d_d = w_descr.get('defaultTaskStartToCloseTimeout')
if r_d_d_d != d_d_d:
raise SWFRegistrationError(
'Default decision duration for %r version %r does not match: %r != %r'
% (name, version, r_d_d_d, d_d_d))
r_d_w_d = w_descr.get('defaultExecutionStartToCloseTimeout')
if r_d_w_d != d_w_d:
raise SWFRegistrationError(
'Default workflow duration for %r version %r does not match: %r != %r'
% (name, version, r_d_w_d, d_w_d))
r_d_c_p = w_descr.get('defaultChildPolicy')
if r_d_c_p != d_c_p:
raise SWFRegistrationError(
'Default child policy for %r version %r does not match: %r != %r' %
(name, version, r_d_c_p, d_c_p))
def check_compatible(self, swf_layer1, domain, name, version):
"""Check if the remote config has the same defaults as this one.
Raise SWFRegistrationError in case of SWF communication errors or
incompatibility and ValueError if any configuration values can't be
converted to the required types.
"""
d_t_l, d_h, d_sch_c, d_sch_s, d_s_c = self._cvt_values()
try:
a_descr = swf_layer1.describe_activity_type(str(domain), str(name), str(version))
a_descr = a_descr['configuration']
except SWFResponseError as e:
logger.exception('Error while checking activity compatibility:')
raise SWFRegistrationError(e)
r_d_t_l = a_descr.get('defaultTaskList', {}).get('name')
if r_d_t_l != d_t_l:
raise SWFRegistrationError(
'Default task list for %r version %r does not match: %r != %r' %
(name, version, r_d_t_l, d_t_l))
r_d_h = a_descr.get('defaultTaskHeartbeatTimeout')
if r_d_h != d_h:
raise SWFRegistrationError(
'Default heartbeat for %r version %r does not match: %r != %r' %
(name, version, r_d_h, d_h))
r_d_sch_c = a_descr.get('defaultTaskScheduleToCloseTimeout')
if r_d_sch_c != d_sch_c:
raise SWFRegistrationError(
'Default schedule to close for %r version %r does not match: %r != %r'
% (name, version, r_d_sch_c, d_sch_c))
r_d_sch_s = a_descr.get('defaultTaskScheduleToStartTimeout')
if r_d_sch_s != d_sch_s:
raise SWFRegistrationError(
'Default schedule to start for %r version %r does not match: %r != %r'
% (name, version, r_d_sch_s, d_sch_s))
r_d_s_c = a_descr.get('defaultTaskStartToCloseTimeout')
if r_d_s_c != d_s_c:
raise SWFRegistrationError(
'Default start to close for %r version %r does not match: %r != %r'
% (name, version, r_d_s_c, d_s_c))
def poll_first_page(swf_client, domain, task_list, identity=None):
"""Return the response from loading the first page. In case of errors,
empty responses or whatnot retry until a valid response.
:type swf_client: :class:`SWFClient`
:param swf_client: an implementation or duck typing of :class:`SWFClient`
:param domain: the domain containing the task list to poll
:param task_list: the task list from which to poll for events
:param identity: an identity str of the request maker
:rtype: dict[str, str|int|list|dict]
:returns: a dict containing workflow information and list of events
"""
swf_response = {}
while not swf_response.get('taskToken'):
try:
swf_response = swf_client.poll_for_decision_task(domain,
task_list,
identity=identity)
except ClientError:
logger.exception('Error while polling for decisions:')
return swf_response
def poll_page(swf_client, domain, task_list, token, identity=None):
"""Return a specific page. In case of errors retry a number of times.
:type swf_client: :class:`SWFClient`
:param swf_client: an implementation or duck typing of :class:`SWFClient`
:param domain: the domain containing the task list to poll
:param task_list: the task list from which to poll for events
:param token: the token string for the requested page
:param identity: an identity str of the request maker
:rtype: dict[str, str|int|list|dict]
:returns: a dict containing workflow information and list of events
"""
for _ in range(7): # give up after a limited number of retries
try:
swf_response = swf_client.poll_for_decision_task(
domain, task_list, identity=identity, next_page_token=token)
break
except ClientError:
logger.exception('Error while polling for decision page:')
else:
raise _PaginationError()
return swf_response
def poll_page(swf_client, domain, task_list, token, identity=None):
"""Return a specific page. In case of errors retry a number of times.
:type swf_client: :class:`SWFClient`
:param swf_client: an implementation or duck typing of :class:`SWFClient`
:param domain: the domain containing the task list to poll
:param task_list: the task list from which to poll for events
:param token: the token string for the requested page
:param identity: an identity str of the request maker
:rtype: dict[str, str|int|list|dict]
:returns: a dict containing workflow information and list of events
"""
for _ in range(7): # give up after a limited number of retries
try:
swf_response = swf_client.poll_for_decision_task(
domain, task_list, identity=identity, next_page_token=token)
break
except ClientError:
logger.exception('Error while polling for decision page:')
else:
raise _PaginationError()
return swf_response
def __call__(self, *args, **kwargs):
"""Consult the execution history for results or schedule a new task.
This is method gets called from the user workflow code.
When calling it, the task it refers to can be in one of the following
states: RUNNING, READY, FAILED, TIMEDOUT or NOTSCHEDULED.
* If the task is RUNNING this returns a Placeholder. The Placeholder
interrupts the workflow execution if its result is accessed by
raising a SuspendTask exception.
* If the task is READY this returns a Result object. Calling the result
method on this object will just return the final value the task
produced.
* If the task is FAILED this returns an Error object. Calling the
result method on this object will raise a TaskError exception
containing the error message set by the task.
* In case of a TIMEOUT this returns an Timeout object. Calling the
result method on this object will raise TaskTimedout exception, a
subclass of TaskError.
* If the task was NOTSCHEDULED yet:
* If any errors in arguments, propagate the error by returning
another error.
* If any placeholders in arguments, don't do anything because there
are unresolved dependencies.
* Finally, if all the arguments look OK, schedule it for execution.
"""
task_exec_history = self.task_exec_history
call_number = self.call_number
self.call_number += 1
r = placeholder()
for retry_number, delay in enumerate(self.retry):
if task_exec_history.is_timeout(call_number, retry_number):
continue
if task_exec_history.is_running(call_number, retry_number):
break # result = Placehloder
if task_exec_history.has_result(call_number, retry_number):
value = task_exec_history.result(call_number, retry_number)
order = task_exec_history.order(call_number, retry_number)
try:
value = self.deserialize_result(value)
except Exception as e:
logger.exception(
'Error while deserializing the activity result:')
self.task_decision.fail(e)
break # result = Placeholder
r = result(value, order)
break
if task_exec_history.is_error(call_number, retry_number):
err = task_exec_history.error(call_number, retry_number)
order = task_exec_history.order(call_number, retry_number)
r = error(err, order)
break
traversed_args, (err, placeholders) = traverse_data([args, kwargs])
if err:
r = copy_result_proxy(err)
break
if placeholders:
break # result = Placeholder
t_args, t_kwargs = traversed_args
try:
input_data = self.serialize_input(*t_args, **t_kwargs)
except Exception as e:
logger.exception('Error while serializing the task input:')
self.task_decision.fail(e)
break # result = Placeholder
self.task_decision.schedule(call_number, retry_number, delay,
input_data)
break # result = Placeholder
else:
# No retries left, it must be a timeout
order = task_exec_history.order(call_number, retry_number)
r = timeout(order)
return r
def __call__(self, *args, **kwargs):
"""Consult the execution history for results or schedule a new task.
This is method gets called from the user workflow code.
When calling it, the task it refers to can be in one of the following
states: RUNNING, READY, FAILED, TIMEDOUT or NOTSCHEDULED.
* If the task is RUNNING this returns a Placeholder. The Placeholder
interrupts the workflow execution if its result is accessed by
raising a SuspendTask exception.
* If the task is READY this returns a Result object. Calling the result
method on this object will just return the final value the task
produced.
* If the task is FAILED this returns an Error object. Calling the
result method on this object will raise a TaskError exception
containing the error message set by the task.
* In case of a TIMEOUT this returns an Timeout object. Calling the
result method on this object will raise TaskTimedout exception, a
subclass of TaskError.
* If the task was NOTSCHEDULED yet:
* If any errors in arguments, propagate the error by returning
another error.
* If any placeholders in arguments, don't do anything because there
are unresolved dependencies.
* Finally, if all the arguments look OK, schedule it for execution.
"""
task_exec_history = self.task_exec_history
call_number = self.call_number
self.call_number += 1
r = placeholder()
for retry_number, delay in enumerate(self.retry):
if task_exec_history.is_timeout(call_number, retry_number):
continue
if task_exec_history.is_running(call_number, retry_number):
break # result = Placehloder
if task_exec_history.has_result(call_number, retry_number):
value = task_exec_history.result(call_number, retry_number)
order = task_exec_history.order(call_number, retry_number)
try:
value = self.deserialize_result(value)
except Exception as e:
logger.exception('Error while deserializing the activity result:')
self.task_decision.fail(e)
break # result = Placeholder
r = result(value, order)
break
if task_exec_history.is_error(call_number, retry_number):
err = task_exec_history.error(call_number, retry_number)
order = task_exec_history.order(call_number, retry_number)
r = error(err, order)
break
traversed_args, (err, placeholders) = traverse_data([args, kwargs])
if err:
r = copy_result_proxy(err)
break
if placeholders:
break # result = Placeholder
t_args, t_kwargs = traversed_args
try:
input_data = self.serialize_input(*t_args, **t_kwargs)
except Exception as e:
logger.exception('Error while serializing the task input:')
self.task_decision.fail(e)
break # result = Placeholder
self.task_decision.schedule(call_number, retry_number, delay, input_data)
break # result = Placeholder
else:
# No retries left, it must be a timeout
order = task_exec_history.order(call_number, retry_number)
r = timeout(order)
return r