def receive_messages(self, queue_name, count, wait_seconds_timeout, visibility_timeout): """ Attempt to retrieve visible messages from a queue. If a message was read by client and not deleted it is considered to be "inflight" and cannot be read. We make attempts to obtain ``count`` messages but we may return less if messages are in-flight or there are simple not enough messages in the queue. :param string queue_name: The name of the queue to read from. :param int count: The maximum amount of messages to retrieve. :param int visibility_timeout: The number of seconds the message should remain invisible to other queue readers. """ queue = self.get_queue(queue_name) result = [] polling_end = unix_time() + wait_seconds_timeout # queue.messages only contains visible messages while True: for message in queue.messages: if not message.visible: continue message.mark_received( visibility_timeout=visibility_timeout ) result.append(message) if len(result) >= count: break if result or unix_time() > polling_end: break return result
def receive_messages(self, queue_name, count, wait_seconds_timeout, visibility_timeout): """ Attempt to retrieve visible messages from a queue. If a message was read by client and not deleted it is considered to be "inflight" and cannot be read. We make attempts to obtain ``count`` messages but we may return less if messages are in-flight or there are simple not enough messages in the queue. :param string queue_name: The name of the queue to read from. :param int count: The maximum amount of messages to retrieve. :param int visibility_timeout: The number of seconds the message should remain invisible to other queue readers. :param int wait_seconds_timeout: The duration (in seconds) for which the call waits for a message to arrive in the queue before returning. If a message is available, the call returns sooner than WaitTimeSeconds """ queue = self.get_queue(queue_name) result = [] polling_end = unix_time() + wait_seconds_timeout # queue.messages only contains visible messages while True: if result or (wait_seconds_timeout and unix_time() > polling_end): break if len(queue.messages) == 0: # we want to break here, otherwise it will be an infinite loop if wait_seconds_timeout == 0: break import time time.sleep(0.001) continue messages_to_dlq = [] for message in queue.messages: if not message.visible: continue if queue.dead_letter_queue is not None and message.approximate_receive_count >= queue.redrive_policy['maxReceiveCount']: messages_to_dlq.append(message) continue message.mark_received( visibility_timeout=visibility_timeout ) result.append(message) if len(result) >= count: break for message in messages_to_dlq: queue._messages.remove(message) queue.dead_letter_queue.add_message(message) return result
def create_account_status(self): return { 'CreateAccountStatus': { 'Id': self.create_account_status_id, 'AccountName': self.name, 'State': 'SUCCEEDED', 'RequestedTimestamp': unix_time(self.create_time), 'CompletedTimestamp': unix_time(self.create_time), 'AccountId': self.id, } }
def __init__(self, partition_key, data, sequence_number, explicit_hash_key): self.partition_key = partition_key self.data = data self.sequence_number = sequence_number self.explicit_hash_key = explicit_hash_key self.created_at_datetime = datetime.datetime.utcnow() self.created_at = unix_time(self.created_at_datetime)
def __init__(self, name, region, **kwargs): self.name = name self.region = region self.tags = {} self.permissions = {} self._messages = [] now = unix_time() self.created_timestamp = now self.queue_arn = 'arn:aws:sqs:{0}:123456789012:{1}'.format(self.region, self.name) self.dead_letter_queue = None # default settings for a non fifo queue defaults = { 'ContentBasedDeduplication': 'false', 'DelaySeconds': 0, 'FifoQueue': 'false', 'KmsDataKeyReusePeriodSeconds': 300, # five minutes 'KmsMasterKeyId': None, 'MaximumMessageSize': int(64 << 10), 'MessageRetentionPeriod': 86400 * 4, # four days 'Policy': None, 'ReceiveMessageWaitTimeSeconds': 0, 'RedrivePolicy': None, 'VisibilityTimeout': 30, } defaults.update(kwargs) self._set_attributes(defaults, now) # Check some conditions if self.fifo_queue and not self.name.endswith('.fifo'): raise MessageAttributesInvalid('Queue name must end in .fifo for FIFO queues')
def test_list_closed_workflow_executions(): conn = setup_swf_environment() # Leave one workflow execution open to make sure it isn't displayed conn.start_workflow_execution( 'test-domain', 'uid-abcd1234', 'test-workflow', 'v1.0' ) # One closed workflow execution run_id = conn.start_workflow_execution( 'test-domain', 'uid-abcd12345', 'test-workflow', 'v1.0' )['runId'] conn.terminate_workflow_execution('test-domain', 'uid-abcd12345', details='some details', reason='a more complete reason', run_id=run_id) yesterday = datetime.now() - timedelta(days=1) oldest_date = unix_time(yesterday) response = conn.list_closed_workflow_executions( 'test-domain', start_oldest_date=oldest_date, workflow_id='test-workflow') execution_infos = response['executionInfos'] len(execution_infos).should.equal(1) open_workflow = execution_infos[0] open_workflow['workflowType'].should.equal({'version': 'v1.0', 'name': 'test-workflow'}) open_workflow.should.contain('startTimestamp') open_workflow['execution']['workflowId'].should.equal('uid-abcd12345') open_workflow['execution'].should.contain('runId') open_workflow['cancelRequested'].should.be(False) open_workflow['executionStatus'].should.equal('CLOSED')
def describe(self): results = { "Table": { "CreationDateTime": unix_time(self.created_at), "KeySchema": { "HashKeyElement": { "AttributeName": self.hash_key_attr, "AttributeType": self.hash_key_type }, }, "ProvisionedThroughput": { "ReadCapacityUnits": self.read_capacity, "WriteCapacityUnits": self.write_capacity }, "TableName": self.name, "TableStatus": "ACTIVE", "ItemCount": len(self), "TableSizeBytes": 0, } } if self.has_range_key: results["Table"]["KeySchema"]["RangeKeyElement"] = { "AttributeName": self.range_key_attr, "AttributeType": self.range_key_type } return results
def __init__(self, name, region, **kwargs): self.name = name self.visibility_timeout = int(kwargs.get('VisibilityTimeout', 30)) self.region = region self._messages = [] now = unix_time() # kwargs can also have: # [Policy, RedrivePolicy] self.fifo_queue = kwargs.get('FifoQueue', 'false') == 'true' self.content_based_deduplication = kwargs.get('ContentBasedDeduplication', 'false') == 'true' self.kms_master_key_id = kwargs.get('KmsMasterKeyId', 'alias/aws/sqs') self.kms_data_key_reuse_period_seconds = int(kwargs.get('KmsDataKeyReusePeriodSeconds', 300)) self.created_timestamp = now self.delay_seconds = int(kwargs.get('DelaySeconds', 0)) self.last_modified_timestamp = now self.maximum_message_size = int(kwargs.get('MaximumMessageSize', 64 << 10)) self.message_retention_period = int(kwargs.get('MessageRetentionPeriod', 86400 * 4)) # four days self.queue_arn = 'arn:aws:sqs:{0}:123456789012:{1}'.format(self.region, self.name) self.receive_message_wait_time_seconds = int(kwargs.get('ReceiveMessageWaitTimeSeconds', 0)) # wait_time_seconds will be set to immediate return messages self.wait_time_seconds = int(kwargs.get('WaitTimeSeconds', 0)) # Check some conditions if self.fifo_queue and not self.name.endswith('.fifo'): raise MessageAttributesInvalid('Queue name must end in .fifo for FIFO queues')
def complete(self, event_id, result=None): self.execution_status = "CLOSED" self.close_status = "COMPLETED" self.close_timestamp = unix_time() self._add_event( "WorkflowExecutionCompleted", decision_task_completed_event_id=event_id, result=result, )
def fail(self, event_id, details=None, reason=None): # TODO: implement lenght constraints on details/reason self.execution_status = "CLOSED" self.close_status = "FAILED" self.close_timestamp = unix_time() self._add_event( "WorkflowExecutionFailed", decision_task_completed_event_id=event_id, details=details, reason=reason, )
def describe(self): return { 'Account': { 'Id': self.id, 'Arn': self.arn, 'Email': self.email, 'Name': self.name, 'Status': self.status, 'JoinedMethod': self.joined_method, 'JoinedTimestamp': unix_time(self.create_time), } }
def start(self): self.start_timestamp = unix_time() self._add_event( "WorkflowExecutionStarted", child_policy=self.child_policy, execution_start_to_close_timeout=self.execution_start_to_close_timeout, # TODO: fix this hardcoded value parent_initiated_event_id=0, task_list=self.task_list, task_start_to_close_timeout=self.task_start_to_close_timeout, workflow_type=self.workflow_type, ) self.schedule_decision_task()
def describe(self): results = { 'Table': { 'AttributeDefinitions': self.attr, 'ProvisionedThroughput': self.throughput, 'TableSizeBytes': 0, 'TableName': self.name, 'TableStatus': 'ACTIVE', 'KeySchema': self.schema, 'ItemCount': len(self), 'CreationDateTime': unix_time(self.created_at), 'GlobalSecondaryIndexes': [index for index in self.global_indexes], } } return results
def to_dict(self): key_dict = { "KeyMetadata": { "AWSAccountId": self.account_id, "Arn": self.arn, "CreationDate": "%d" % unix_time(), "Description": self.description, "Enabled": self.enabled, "KeyId": self.id, "KeyUsage": self.key_usage, "KeyState": self.key_state, } } if self.key_state == 'PendingDeletion': key_dict['KeyMetadata']['DeletionDate'] = iso_8601_datetime_without_milliseconds(self.deletion_date) return key_dict
def describe(self): results = { "Table": { "AttributeDefinitions": self.attr, "ProvisionedThroughput": self.throughput, "TableSizeBytes": 0, "TableName": self.name, "TableStatus": "ACTIVE", "KeySchema": self.schema, "ItemCount": len(self), "CreationDateTime": unix_time(self.created_at), "GlobalSecondaryIndexes": [index for index in self.global_indexes], "LocalSecondaryIndexes": [index for index in self.indexes], } } return results
def __init__(self, activity_id, activity_type, scheduled_event_id, workflow_execution, timeouts, input=None): self.activity_id = activity_id self.activity_type = activity_type self.details = None self.input = input self.last_heartbeat_timestamp = unix_time() self.scheduled_event_id = scheduled_event_id self.started_event_id = None self.state = "SCHEDULED" self.task_token = str(uuid.uuid4()) self.timeouts = timeouts self.timeout_type = None self.workflow_execution = workflow_execution # this is *not* necessarily coherent with workflow execution history, # but that shouldn't be a problem for tests self.scheduled_at = datetime.utcnow()
def __init__(self, name, visibility_timeout, wait_time_seconds, region): self.name = name self.visibility_timeout = visibility_timeout or 30 self.region = region # wait_time_seconds will be set to immediate return messages self.wait_time_seconds = wait_time_seconds or 0 self._messages = [] now = unix_time() self.created_timestamp = now self.delay_seconds = 0 self.last_modified_timestamp = now self.maximum_message_size = 64 << 10 self.message_retention_period = 86400 * 4 # four days self.queue_arn = 'arn:aws:sqs:{0}:123456789012:{1}'.format(self.region, self.name) self.receive_message_wait_time_seconds = 0
def describe(self, base_key='TableDescription'): results = { base_key: { 'AttributeDefinitions': self.attr, 'ProvisionedThroughput': self.throughput, 'TableSizeBytes': 0, 'TableName': self.name, 'TableStatus': 'ACTIVE', 'TableArn': self.table_arn, 'KeySchema': self.schema, 'ItemCount': len(self), 'CreationDateTime': unix_time(self.created_at), 'GlobalSecondaryIndexes': [index for index in self.global_indexes], 'LocalSecondaryIndexes': [index for index in self.indexes], } } if self.stream_specification and self.stream_specification['StreamEnabled']: results[base_key]['StreamSpecification'] = self.stream_specification if self.latest_stream_label: results[base_key]['LatestStreamLabel'] = self.latest_stream_label results[base_key]['LatestStreamArn'] = self.table_arn + '/stream/' + self.latest_stream_label return results
def __init__(self, event_id, event_type, event_timestamp=None, **kwargs): if event_type not in SUPPORTED_HISTORY_EVENT_TYPES: raise NotImplementedError( "HistoryEvent does not implement attributes for type '{0}'".format(event_type) ) self.event_id = event_id self.event_type = event_type if event_timestamp: self.event_timestamp = event_timestamp else: self.event_timestamp = unix_time() # pre-populate a dict: {"camelCaseKey": value} self.event_attributes = {} for key, value in kwargs.items(): if value: camel_key = underscores_to_camelcase(key) if key == "task_list": value = {"name": value} elif key == "workflow_type": value = {"name": value.name, "version": value.version} elif key == "activity_type": value = value.to_short_dict() self.event_attributes[camel_key] = value
def _set_attributes(self, attributes, now=None): if not now: now = unix_time() integer_fields = ('DelaySeconds', 'KmsDataKeyreusePeriodSeconds', 'MaximumMessageSize', 'MessageRetentionPeriod', 'ReceiveMessageWaitTime', 'VisibilityTimeout') bool_fields = ('ContentBasedDeduplication', 'FifoQueue') for key, value in six.iteritems(attributes): if key in integer_fields: value = int(value) if key in bool_fields: value = value == "true" if key == 'RedrivePolicy' and value is not None: continue setattr(self, camelcase_to_underscores(key), value) if attributes.get('RedrivePolicy', None): self._setup_dlq(attributes['RedrivePolicy']) self.last_modified_timestamp = now
def schedule_activity_task(self, event_id, attributes): # Helper function to avoid repeating ourselves in the next sections def fail_schedule_activity_task(_type, _cause): # TODO: implement other possible failure mode: OPEN_ACTIVITIES_LIMIT_EXCEEDED # NB: some failure modes are not implemented and probably won't be implemented in # the future, such as ACTIVITY_CREATION_RATE_EXCEEDED or # OPERATION_NOT_PERMITTED self._add_event( "ScheduleActivityTaskFailed", activity_id=attributes["activityId"], activity_type=_type, cause=_cause, decision_task_completed_event_id=event_id, ) self.should_schedule_decision_next = True activity_type = self.domain.get_type( "activity", attributes["activityType"]["name"], attributes["activityType"]["version"], ignore_empty=True, ) if not activity_type: fake_type = ActivityType(attributes["activityType"]["name"], attributes["activityType"]["version"]) fail_schedule_activity_task(fake_type, "ACTIVITY_TYPE_DOES_NOT_EXIST") return if activity_type.status == "DEPRECATED": fail_schedule_activity_task(activity_type, "ACTIVITY_TYPE_DEPRECATED") return if any(at for at in self.activity_tasks if at.activity_id == attributes["activityId"]): fail_schedule_activity_task(activity_type, "ACTIVITY_ID_ALREADY_IN_USE") return # find task list or default task list, else fail task_list = attributes.get("taskList", {}).get("name") if not task_list and activity_type.task_list: task_list = activity_type.task_list if not task_list: fail_schedule_activity_task(activity_type, "DEFAULT_TASK_LIST_UNDEFINED") return # find timeouts or default timeout, else fail timeouts = {} for _type in [ "scheduleToStartTimeout", "scheduleToCloseTimeout", "startToCloseTimeout", "heartbeatTimeout" ]: default_key = "default_task_" + camelcase_to_underscores(_type) default_value = getattr(activity_type, default_key) timeouts[_type] = attributes.get(_type, default_value) if not timeouts[_type]: error_key = default_key.replace("default_task_", "default_") fail_schedule_activity_task( activity_type, "{0}_UNDEFINED".format(error_key.upper())) return # Only add event and increment counters now that nothing went wrong evt = self._add_event( "ActivityTaskScheduled", activity_id=attributes["activityId"], activity_type=activity_type, control=attributes.get("control"), decision_task_completed_event_id=event_id, heartbeat_timeout=attributes.get("heartbeatTimeout"), input=attributes.get("input"), schedule_to_close_timeout=attributes.get("scheduleToCloseTimeout"), schedule_to_start_timeout=attributes.get("scheduleToStartTimeout"), start_to_close_timeout=attributes.get("startToCloseTimeout"), task_list=task_list, task_priority=attributes.get("taskPriority"), ) task = ActivityTask( activity_id=attributes["activityId"], activity_type=activity_type, input=attributes.get("input"), scheduled_event_id=evt.event_id, workflow_execution=self, timeouts=timeouts, ) self.domain.add_to_activity_task_list(task_list, task) self.open_counts["openActivityTasks"] += 1 self.latest_activity_task_timestamp = unix_time()
def __init__(self, partition_key, data, sequence_number, explicit_hash_key): self.partition_key = partition_key self.data = data self.sequence_number = sequence_number self.explicit_hash_key = explicit_hash_key self.create_at = unix_time()
def receive_message( self, queue_name, count, wait_seconds_timeout, visibility_timeout, message_attribute_names=None, ): # Attempt to retrieve visible messages from a queue. # If a message was read by client and not deleted it is considered to be # "inflight" and cannot be read. We make attempts to obtain ``count`` # messages but we may return less if messages are in-flight or there # are simple not enough messages in the queue. if message_attribute_names is None: message_attribute_names = [] queue = self.get_queue(queue_name) result = [] previous_result_count = len(result) polling_end = unix_time() + wait_seconds_timeout currently_pending_groups = deepcopy(queue.pending_message_groups) # queue.messages only contains visible messages while True: if result or (wait_seconds_timeout and unix_time() > polling_end): break messages_to_dlq = [] for message in queue.messages: if not message.visible: continue if message in queue.pending_messages: # The message is pending but is visible again, so the # consumer must have timed out. queue.pending_messages.remove(message) currently_pending_groups = deepcopy( queue.pending_message_groups) if message.group_id and queue.fifo_queue: if message.group_id in currently_pending_groups: # A previous call is still processing messages in this group, so we cannot deliver this one. continue if (queue.dead_letter_queue is not None and queue.redrive_policy and message.approximate_receive_count >= queue.redrive_policy["maxReceiveCount"]): messages_to_dlq.append(message) continue queue.pending_messages.add(message) message.mark_received(visibility_timeout=visibility_timeout) # Create deepcopy to not mutate the message state when filtering for attributes message_copy = deepcopy(message) _filter_message_attributes(message_copy, message_attribute_names) if not self.is_message_valid_based_on_retention_period( queue_name, message): break result.append(message_copy) if len(result) >= count: break for message in messages_to_dlq: queue._messages.remove(message) queue.dead_letter_queue.add_message(message) if previous_result_count == len(result): if wait_seconds_timeout == 0: # There is timeout and we have added no additional results, # so break to avoid an infinite loop. break import time time.sleep(0.01) continue previous_result_count = len(result) return result
def start(self, started_event_id, previous_started_event_id=None): self.state = "STARTED" self.started_timestamp = unix_time() self.started_event_id = started_event_id self.previous_started_event_id = previous_started_event_id
def __init__( self, name, region, authentication_strategy, auto_minor_version_upgrade, configuration, deployment_mode, encryption_options, engine_type, engine_version, host_instance_type, ldap_server_metadata, logs, maintenance_window_start_time, publicly_accessible, security_groups, storage_type, subnet_ids, users, ): self.name = name self.id = get_random_hex(6) self.arn = f"arn:aws:mq:{region}:{ACCOUNT_ID}:broker:{self.id}" self.state = "RUNNING" self.created = unix_time() self.authentication_strategy = authentication_strategy self.auto_minor_version_upgrade = auto_minor_version_upgrade self.deployment_mode = deployment_mode self.encryption_options = encryption_options if not self.encryption_options: self.encryption_options = {"useAwsOwnedKey": True} self.engine_type = engine_type self.engine_version = engine_version self.host_instance_type = host_instance_type self.ldap_server_metadata = ldap_server_metadata self.logs = logs if "general" not in self.logs: self.logs["general"] = False if "audit" not in self.logs: if self.engine_type.upper() == "ACTIVEMQ": self.logs["audit"] = False self.maintenance_window_start_time = maintenance_window_start_time if not self.maintenance_window_start_time: self.maintenance_window_start_time = { "dayOfWeek": "Sunday", "timeOfDay": "00:00", "timeZone": "UTC", } self.publicly_accessible = publicly_accessible self.security_groups = security_groups self.storage_type = storage_type self.subnet_ids = subnet_ids if not self.subnet_ids: if self.deployment_mode == "CLUSTER_MULTI_AZ": self.subnet_ids = [ "default-az1", "default-az2", "default-az3", "default-az4", ] elif self.deployment_mode == "ACTIVE_STANDBY_MULTI_AZ": self.subnet_ids = ["active-subnet", "standby-subnet"] else: self.subnet_ids = ["default-subnet"] self.users = dict() for user in users: self.create_user( username=user["username"], groups=user.get("groups", []), console_access=user.get("consoleAccess", False), ) if self.engine_type.upper() == "RABBITMQ": self.configurations = None else: current_config = configuration or { "id": f"c-{get_random_hex(6)}", "revision": 1, } self.configurations = { "current": current_config, "history": [], } if self.engine_type.upper() == "RABBITMQ": console_url = f"https://0000.mq.{region}.amazonaws.com" endpoints = ["amqps://mockmq:5671"] else: console_url = f"https://0000.mq.{region}.amazonaws.com:8162" endpoints = [ "ssl://mockmq:61617", "amqp+ssl://mockmq:5671", "stomp+ssl://mockmq:61614", "mqtt+ssl://mockmq:8883", "wss://mockmq:61619", ] self.instances = [{ "consoleURL": console_url, "endpoints": endpoints, "ipAddress": "192.168.0.1", }] if deployment_mode == "ACTIVE_STANDBY_MULTI_AZ": self.instances.append({ "consoleURL": console_url, "endpoints": endpoints, "ipAddress": "192.168.0.2", })
def test_unix_time(): unix_time().should.equal(1420113600.0)
def __init__(self): self.settings = dict() self.last_modified = unix_time()
def reached(self): return unix_time() >= self.timestamp
def start(self, started_event_id): self.state = "STARTED" self.started_timestamp = unix_time() self.started_event_id = started_event_id
def __init__(self, consumer_name, region_name, stream_arn): self.consumer_name = consumer_name self.created = unix_time() self.stream_arn = stream_arn stream_name = stream_arn.split("/")[-1] self.consumer_arn = f"arn:aws:kinesis:{region_name}:{ACCOUNT_ID}:stream/{stream_name}/consumer/{consumer_name}"
def describe(self): return { "ServicePrincipal": self.service_principal, "DateEnabled": unix_time(self.date_enabled), }
def describe(self): admin = self.account.describe() admin["DelegationEnabledDate"] = unix_time(self.enabled_date) return admin
def put_events(self, events): num_events = len(events) if num_events > 10: # the exact error text is longer, the Value list consists of all the put events raise ValidationException( "1 validation error detected: " "Value '[PutEventsRequestEntry]' at 'entries' failed to satisfy constraint: " "Member must have length less than or equal to 10") entries = [] for event in events: if "Source" not in event: entries.append({ "ErrorCode": "InvalidArgument", "ErrorMessage": "Parameter Source is not valid. Reason: Source is a required argument.", }) elif "DetailType" not in event: entries.append({ "ErrorCode": "InvalidArgument", "ErrorMessage": "Parameter DetailType is not valid. Reason: DetailType is a required argument.", }) elif "Detail" not in event: entries.append({ "ErrorCode": "InvalidArgument", "ErrorMessage": "Parameter Detail is not valid. Reason: Detail is a required argument.", }) else: try: json.loads(event["Detail"]) except ValueError: # json.JSONDecodeError exists since Python 3.5 entries.append({ "ErrorCode": "MalformedDetail", "ErrorMessage": "Detail is malformed.", }) continue event_id = str(uuid4()) entries.append({"EventId": event_id}) # if 'EventBusName' is not especially set, it will be sent to the default one event_bus_name = event.get("EventBusName", "default") for rule in self.rules.values(): rule.send_to_targets( event_bus_name, { "version": "0", "id": event_id, "detail-type": event["DetailType"], "source": event["Source"], "account": ACCOUNT_ID, "time": event.get("Time", unix_time(datetime.utcnow())), "region": self.region_name, "resources": event.get("Resources", []), "detail": json.loads(event["Detail"]), }, ) return entries
def schedule_activity_task(self, event_id, attributes): # Helper function to avoid repeating ourselves in the next sections def fail_schedule_activity_task(_type, _cause): # TODO: implement other possible failure mode: OPEN_ACTIVITIES_LIMIT_EXCEEDED # NB: some failure modes are not implemented and probably won't be implemented in # the future, such as ACTIVITY_CREATION_RATE_EXCEEDED or # OPERATION_NOT_PERMITTED self._add_event( "ScheduleActivityTaskFailed", activity_id=attributes["activityId"], activity_type=_type, cause=_cause, decision_task_completed_event_id=event_id, ) self.should_schedule_decision_next = True activity_type = self.domain.get_type( "activity", attributes["activityType"]["name"], attributes["activityType"]["version"], ignore_empty=True, ) if not activity_type: fake_type = ActivityType(attributes["activityType"]["name"], attributes["activityType"]["version"]) fail_schedule_activity_task(fake_type, "ACTIVITY_TYPE_DOES_NOT_EXIST") return if activity_type.status == "DEPRECATED": fail_schedule_activity_task(activity_type, "ACTIVITY_TYPE_DEPRECATED") return if any(at for at in self.activity_tasks if at.activity_id == attributes["activityId"]): fail_schedule_activity_task(activity_type, "ACTIVITY_ID_ALREADY_IN_USE") return # find task list or default task list, else fail task_list = attributes.get("taskList", {}).get("name") if not task_list and activity_type.task_list: task_list = activity_type.task_list if not task_list: fail_schedule_activity_task(activity_type, "DEFAULT_TASK_LIST_UNDEFINED") return # find timeouts or default timeout, else fail timeouts = {} for _type in ["scheduleToStartTimeout", "scheduleToCloseTimeout", "startToCloseTimeout", "heartbeatTimeout"]: default_key = "default_task_" + camelcase_to_underscores(_type) default_value = getattr(activity_type, default_key) timeouts[_type] = attributes.get(_type, default_value) if not timeouts[_type]: error_key = default_key.replace("default_task_", "default_") fail_schedule_activity_task(activity_type, "{0}_UNDEFINED".format(error_key.upper())) return # Only add event and increment counters now that nothing went wrong evt = self._add_event( "ActivityTaskScheduled", activity_id=attributes["activityId"], activity_type=activity_type, control=attributes.get("control"), decision_task_completed_event_id=event_id, heartbeat_timeout=attributes.get("heartbeatTimeout"), input=attributes.get("input"), schedule_to_close_timeout=attributes.get("scheduleToCloseTimeout"), schedule_to_start_timeout=attributes.get("scheduleToStartTimeout"), start_to_close_timeout=attributes.get("startToCloseTimeout"), task_list=task_list, task_priority=attributes.get("taskPriority"), ) task = ActivityTask( activity_id=attributes["activityId"], activity_type=activity_type, input=attributes.get("input"), scheduled_event_id=evt.event_id, workflow_execution=self, timeouts=timeouts, ) self.domain.add_to_activity_task_list(task_list, task) self.open_counts["openActivityTasks"] += 1 self.latest_activity_task_timestamp = unix_time()
def reset_heartbeat_clock(self): self.last_heartbeat_timestamp = unix_time()
def receive_messages(self, queue_name, count, wait_seconds_timeout, visibility_timeout): """ Attempt to retrieve visible messages from a queue. If a message was read by client and not deleted it is considered to be "inflight" and cannot be read. We make attempts to obtain ``count`` messages but we may return less if messages are in-flight or there are simple not enough messages in the queue. :param string queue_name: The name of the queue to read from. :param int count: The maximum amount of messages to retrieve. :param int visibility_timeout: The number of seconds the message should remain invisible to other queue readers. :param int wait_seconds_timeout: The duration (in seconds) for which the call waits for a message to arrive in the queue before returning. If a message is available, the call returns sooner than WaitTimeSeconds """ queue = self.get_queue(queue_name) result = [] previous_result_count = len(result) polling_end = unix_time() + wait_seconds_timeout # queue.messages only contains visible messages while True: if result or (wait_seconds_timeout and unix_time() > polling_end): break messages_to_dlq = [] for message in queue.messages: if not message.visible: continue if message in queue.pending_messages: # The message is pending but is visible again, so the # consumer must have timed out. queue.pending_messages.remove(message) if message.group_id and queue.fifo_queue: if message.group_id in queue.pending_message_groups: # There is already one active message with the same # group, so we cannot deliver this one. continue queue.pending_messages.add(message) if (queue.dead_letter_queue is not None and message.approximate_receive_count >= queue.redrive_policy["maxReceiveCount"]): messages_to_dlq.append(message) continue message.mark_received(visibility_timeout=visibility_timeout) result.append(message) if len(result) >= count: break for message in messages_to_dlq: queue._messages.remove(message) queue.dead_letter_queue.add_message(message) if previous_result_count == len(result): if wait_seconds_timeout == 0: # There is timeout and we have added no additional results, # so break to avoid an infinite loop. break import time time.sleep(0.01) continue previous_result_count = len(result) return result
def receive_messages(self, queue_name, count, wait_seconds_timeout, visibility_timeout): """ Attempt to retrieve visible messages from a queue. If a message was read by client and not deleted it is considered to be "inflight" and cannot be read. We make attempts to obtain ``count`` messages but we may return less if messages are in-flight or there are simple not enough messages in the queue. :param string queue_name: The name of the queue to read from. :param int count: The maximum amount of messages to retrieve. :param int visibility_timeout: The number of seconds the message should remain invisible to other queue readers. :param int wait_seconds_timeout: The duration (in seconds) for which the call waits for a message to arrive in the queue before returning. If a message is available, the call returns sooner than WaitTimeSeconds """ queue = self.get_queue(queue_name) result = [] previous_result_count = len(result) polling_end = unix_time() + wait_seconds_timeout # queue.messages only contains visible messages while True: if result or (wait_seconds_timeout and unix_time() > polling_end): break messages_to_dlq = [] for message in queue.messages: if not message.visible: continue if message in queue.pending_messages: # The message is pending but is visible again, so the # consumer must have timed out. queue.pending_messages.remove(message) if message.group_id and queue.fifo_queue: if message.group_id in queue.pending_message_groups: # There is already one active message with the same # group, so we cannot deliver this one. continue queue.pending_messages.add(message) if queue.dead_letter_queue is not None and message.approximate_receive_count >= queue.redrive_policy['maxReceiveCount']: messages_to_dlq.append(message) continue message.mark_received( visibility_timeout=visibility_timeout ) result.append(message) if len(result) >= count: break for message in messages_to_dlq: queue._messages.remove(message) queue.dead_letter_queue.add_message(message) if previous_result_count == len(result): if wait_seconds_timeout == 0: # There is timeout and we have added no additional results, # so break to avoid an infinite loop. break import time time.sleep(0.01) continue previous_result_count = len(result) return result