def test_query_all():
index_name = 'job_index'
secondary_key = 'job_id'
secondary_key_type = 'N'
table_name = id_generator.generate_job_item_id()[:10] + "_table"
conn = get_connection()
table = create_table_w_index(conn, table_name, 'job_item_id', 'N',
index_name, secondary_key, secondary_key_type)
table.meta.client.get_waiter('table_exists').wait(TableName=table_name)
test_job_id = 5
count = query_all(table, index_name, secondary_key, test_job_id, select='COUNT')
assert count == 0
record_id = random.randint(1, 1000)
item = {
"job_item_id" : record_id,
"job_id" : test_job_id,
"fourth_param" : "OINoin",
"third_param" : {
"mynested" : "value",
"nested2" : "value2"
}
}
insert(table, item)
count = query_all(table, index_name, secondary_key, test_job_id, select='COUNT')
assert count == 1
drop_table(conn, table_name)
def test_get_queue():
fake_queue = sqs.get_queue("fake-queue")
assert fake_queue is None
queue_name = id_generator.generate_job_item_id()
queue = sqs.create_queue(queue_name)
assert queue is not None
def test_get_queue():
fake_queue = sqs.get_queue("fake-queue")
assert fake_queue is None
queue_name = id_generator.generate_job_item_id()
queue = sqs.create_queue(queue_name)
assert queue is not None
def test_get_message_by_key():
TEST_JOB_ITEM_KEY = id_generator.generate_job_item_id()
queue_name = id_generator.generate_job_item_id()
queue = get_test_queue(queue_name)
message_id = sqs.send_message(queue, 'boto3', {
"job_item_key" : {
"StringValue" : TEST_JOB_ITEM_KEY,
"DataType" : "String"
},
"job_type" : {
"StringValue" : "process_job_item",
"DataType" : "String"
}
})
message = sqs.get_message_by_key(queue.url, TEST_JOB_ITEM_KEY)
assert message_id == message['MessageId']
assert message['MessageAttributes']['job_item_key']['StringValue'] == TEST_JOB_ITEM_KEY
sqs.delete_queue(queue)
def test_delete_queue(): """ SLOW! Takes 60 seconds to delete a queue """ queue_name = id_generator.generate_job_item_id() queue = get_test_queue(queue_name) resp = sqs.delete_queue(queue) time.sleep(90) queue = sqs.get_queue(queue_name) assert queue is None
def test_delete_queue():
"""
SLOW! Takes 60 seconds to delete a queue
"""
queue_name = id_generator.generate_job_item_id()
queue = get_test_queue(queue_name)
resp = sqs.delete_queue(queue)
time.sleep(90)
queue = sqs.get_queue(queue_name)
assert queue is None
def test_get_message_by_key():
TEST_JOB_ITEM_KEY = id_generator.generate_job_item_id()
queue_name = id_generator.generate_job_item_id()
queue = get_test_queue(queue_name)
message_id = sqs.send_message(
queue, 'boto3', {
"job_item_key": {
"StringValue": TEST_JOB_ITEM_KEY,
"DataType": "String"
},
"job_type": {
"StringValue": "process_job_item",
"DataType": "String"
}
})
message = sqs.get_message_by_key(queue.url, TEST_JOB_ITEM_KEY)
assert message_id == message['MessageId']
assert message['MessageAttributes']['job_item_key'][
'StringValue'] == TEST_JOB_ITEM_KEY
sqs.delete_queue(queue)
def test_create_table_w_index():
table_name = id_generator.generate_job_item_id()[:10] + "_table"
conn = get_connection()
table = create_table_w_index(conn, table_name, 'job_item_id', 'N',
'job_index', 'job_id', 'N')
table.meta.client.get_waiter('table_exists').wait(TableName=table_name)
print dir(table)
print table.key_schema
print table.item_count
print table.creation_date_time
drop_table(conn, table_name)
def test_create_queue():
queue_name = id_generator.generate_job_item_id()
dead_letter_queue_name = queue_name + "_dead"
dead_queue = sqs.create_queue(dead_letter_queue_name, 43200)
dead_arn = dead_queue.attributes['QueueArn']
redrive_policy = {"maxReceiveCount": 5, "deadLetterTargetArn": dead_arn}
new_queue = sqs.create_queue(queue_name, 30, redrive_policy)
sqs.delete_queue(dead_queue)
sqs.delete_queue(new_queue)
def test_retry_logic():
VISIBILITY_TIMEOUT = 2
MAX_RECEIVE_COUNT = 2
queue_name = id_generator.generate_job_item_id()
dead_letter_queue_name = queue_name + "_dead"
dead_queue = sqs.create_queue(dead_letter_queue_name, 43200)
dead_arn = dead_queue.attributes['QueueArn']
redrive_policy = {
"maxReceiveCount": MAX_RECEIVE_COUNT,
"deadLetterTargetArn": dead_arn
}
queue = sqs.create_queue(queue_name, VISIBILITY_TIMEOUT, redrive_policy)
# Create Message
expected_message_id = sqs.send_message(
queue, 'boto3',
{'Author': {
'StringValue': 'Daniel',
'DataType': 'String'
}})
# 1st message request succeeds
message = sqs.get_next_message(queue)
assert message['MessageId'] == expected_message_id
# Visibilty Timeout not exceeded
message = sqs.get_next_message(queue)
assert message is None
# Visibilty Timeout exceeded
time.sleep(VISIBILITY_TIMEOUT)
# 2nd message request succeeds
message = sqs.get_next_message(queue)
assert message['MessageId'] == expected_message_id
# Visibilty Timeout exceeded
time.sleep(VISIBILITY_TIMEOUT)
# 3rd message request fails - MAX_RECEIVE_COUNT exceeded
message = sqs.get_next_message(queue)
assert message is None
# 3rd message in Dead Letter Queue
message = sqs.get_next_message(dead_queue)
assert message['MessageId'] == expected_message_id
# Cleanup
time.sleep(VISIBILITY_TIMEOUT)
sqs.delete_queue(dead_queue)
sqs.delete_queue(queue)
def test_create_queue():
queue_name = id_generator.generate_job_item_id()
dead_letter_queue_name = queue_name + "_dead"
dead_queue = sqs.create_queue(dead_letter_queue_name, 43200)
dead_arn = dead_queue.attributes['QueueArn']
redrive_policy = {"maxReceiveCount":5, "deadLetterTargetArn":dead_arn}
new_queue = sqs.create_queue(queue_name, 30, redrive_policy)
sqs.delete_queue(dead_queue)
sqs.delete_queue(new_queue)
def test_update_msg_visibility_timeout():
job_item_key = id_generator.generate_job_item_id()
queue_name = id_generator.generate_job_item_id()
queue = get_test_queue(queue_name)
attribs = {}
attribs['VisibilityTimeout'] = str(120)
sqs.set_queue_attributes(queue, attribs)
message_id = sqs.send_message(
queue, 'boto3',
{"job_item_key": {
"StringValue": job_item_key,
"DataType": "String"
}})
# 1st Try
message = sqs.get_message_by_key(queue.url, job_item_key)
msg_receipt = message['ReceiptHandle']
assert message is not None
# 2nd - Visibilty Timeout not exceeded
message = sqs.get_message_by_key(queue.url, job_item_key)
assert message is None
new_timeout = 10
response = sqs.update_msg_visibility_timeout(queue.url, msg_receipt,
new_timeout)
# 3rd - Visibilty Timeout not exceeded
message = sqs.get_message_by_key(queue.url, job_item_key)
assert message is None
# Visibilty Timeout exceeded
time.sleep(new_timeout)
# 4th message request succeeds
message = sqs.get_next_message(queue)
assert message['MessageId'] == message_id
sqs.delete_queue(queue)
def test_update_msg_visibility_timeout():
job_item_key = id_generator.generate_job_item_id()
queue_name = id_generator.generate_job_item_id()
queue = get_test_queue(queue_name)
attribs = {}
attribs['VisibilityTimeout'] = str(120)
sqs.set_queue_attributes(queue, attribs)
message_id = sqs.send_message(queue, 'boto3', {
"job_item_key" : {
"StringValue" : job_item_key,
"DataType" : "String"
}
})
# 1st Try
message = sqs.get_message_by_key(queue.url, job_item_key)
msg_receipt = message['ReceiptHandle']
assert message is not None
# 2nd - Visibilty Timeout not exceeded
message = sqs.get_message_by_key(queue.url, job_item_key)
assert message is None
new_timeout = 10
response = sqs.update_msg_visibility_timeout(queue.url,
msg_receipt, new_timeout)
# 3rd - Visibilty Timeout not exceeded
message = sqs.get_message_by_key(queue.url, job_item_key)
assert message is None
# Visibilty Timeout exceeded
time.sleep(new_timeout)
# 4th message request succeeds
message = sqs.get_next_message(queue)
assert message['MessageId'] == message_id
sqs.delete_queue(queue)
def test_send_and_get_message():
queue_name = id_generator.generate_job_item_id()
queue = get_test_queue(queue_name)
message_id = sqs.send_message(queue, 'boto3', {
'Author': {
'StringValue': 'Daniel',
'DataType': 'String'
}
})
time.sleep(5) # Accounts for 5 second delivery delay
message = sqs.get_next_message(queue)
assert message_id == message['MessageId']
sqs.delete_queue(queue)
def test_send_and_get_message():
queue_name = id_generator.generate_job_item_id()
queue = get_test_queue(queue_name)
message_id = sqs.send_message(
queue, 'boto3',
{'Author': {
'StringValue': 'Daniel',
'DataType': 'String'
}})
time.sleep(5) # Accounts for 5 second delivery delay
message = sqs.get_next_message(queue)
assert message_id == message['MessageId']
sqs.delete_queue(queue)
def test_get_message_attributes():
queue_name = id_generator.generate_job_item_id()
queue = get_test_queue(queue_name)
expected_message_id = sqs.send_message(
queue, 'boto3',
{'Author': {
'StringValue': 'Daniel',
'DataType': 'String'
}})
message = sqs.get_next_message(queue)
assert message['Attributes'] is not None
print message['Attributes']['ApproximateReceiveCount']
print message['Attributes']['ApproximateFirstReceiveTimestamp']
def __init__(self, job_id, input_filename, output_filename, output_dir, plugin, method, channel=1, status_id=1, attempts=0): self.job_item_key = id_generator.generate_job_item_id() self.job_id = int(job_id) self.input_filename = input_filename self.output_filename = output_filename self.output_dir = output_dir self.plugin = plugin self.method = method self.channel = int(channel) self.status_id = int(status_id) self.attempts = int(attempts) self.creation_time = int(time.time()) #UTC = PST + 8
def test_get_queue_size():
queue_name = id_generator.generate_job_item_id()
queue = get_test_queue(queue_name)
expected_message_id = sqs.send_message(
queue, 'boto3',
{'Author': {
'StringValue': 'Daniel',
'DataType': 'String'
}})
size = sqs.get_queue_size(queue.url)
assert size == 1
sqs.clear_queue(queue)
sqs.delete_queue(queue)
def test_get_message_attributes():
queue_name = id_generator.generate_job_item_id()
queue = get_test_queue(queue_name)
expected_message_id = sqs.send_message(queue, 'boto3', {
'Author': {
'StringValue': 'Daniel',
'DataType': 'String'
}
})
message = sqs.get_next_message(queue)
assert message['Attributes'] is not None
print message['Attributes']['ApproximateReceiveCount']
print message['Attributes']['ApproximateFirstReceiveTimestamp']
def test_get_queue_size():
queue_name = id_generator.generate_job_item_id()
queue = get_test_queue(queue_name)
expected_message_id = sqs.send_message(queue, 'boto3', {
'Author': {
'StringValue': 'Daniel',
'DataType': 'String'
}
})
size = sqs.get_queue_size(queue.url)
assert size == 1
sqs.clear_queue(queue)
sqs.delete_queue(queue)
def test_clear_queue():
# Only one clear request is allowed every 60 seconds
queue_name = id_generator.generate_job_item_id()
queue = get_test_queue(queue_name)
expected_message_id = sqs.send_message(
queue, 'boto3',
{'Author': {
'StringValue': 'Daniel',
'DataType': 'String'
}})
msgs = sqs.get_messages(queue.url)
assert len(msgs) == 1
sqs.clear_queue(queue)
sqs.delete_queue(queue)
def test_clear_queue():
# Only one clear request is allowed every 60 seconds
queue_name = id_generator.generate_job_item_id()
queue = get_test_queue(queue_name)
expected_message_id = sqs.send_message(queue, 'boto3', {
'Author': {
'StringValue': 'Daniel',
'DataType': 'String'
}
})
msgs = sqs.get_messages(queue.url)
assert len(msgs) == 1
sqs.clear_queue(queue)
sqs.delete_queue(queue)
def test_retry_logic():
VISIBILITY_TIMEOUT = 2
MAX_RECEIVE_COUNT = 2
queue_name = id_generator.generate_job_item_id()
dead_letter_queue_name = queue_name + "_dead"
dead_queue = sqs.create_queue(dead_letter_queue_name, 43200)
dead_arn = dead_queue.attributes['QueueArn']
redrive_policy = {"maxReceiveCount":MAX_RECEIVE_COUNT, "deadLetterTargetArn":dead_arn}
queue = sqs.create_queue(queue_name, VISIBILITY_TIMEOUT, redrive_policy)
# Create Message
expected_message_id = sqs.send_message(queue, 'boto3', {
'Author': {
'StringValue': 'Daniel',
'DataType': 'String'
}
})
# 1st message request succeeds
message = sqs.get_next_message(queue)
assert message['MessageId'] == expected_message_id
# Visibilty Timeout not exceeded
message = sqs.get_next_message(queue)
assert message is None
# Visibilty Timeout exceeded
time.sleep(VISIBILITY_TIMEOUT)
# 2nd message request succeeds
message = sqs.get_next_message(queue)
assert message['MessageId'] == expected_message_id
# Visibilty Timeout exceeded
time.sleep(VISIBILITY_TIMEOUT)
# 3rd message request fails - MAX_RECEIVE_COUNT exceeded
message = sqs.get_next_message(queue)
assert message is None
# 3rd message in Dead Letter Queue
message = sqs.get_next_message(dead_queue)
assert message['MessageId'] == expected_message_id
# Cleanup
time.sleep(VISIBILITY_TIMEOUT)
sqs.delete_queue(dead_queue)
sqs.delete_queue(queue)
def test_all():
TABLE_NAME = id_generator.generate_job_item_id()[:10] + "_table"
conn = get_connection()
table = create_table(conn, TABLE_NAME, 'job_item_id', 'N')
table = get_table(conn, TABLE_NAME)
table.meta.client.get_waiter('table_exists').wait(TableName=TABLE_NAME)
print dir(table)
print table.key_schema
print table.item_count
print table.creation_date_time
record_id = random.randint(1, 1000)
item1 = {
"job_item_id" : record_id,
"job_id" : "9",
"fourth_param" : "OINoin",
"third_param" : {
"mynested" : "value",
"nested2" : "value2"
}
}
insert(table, item1)
i1 = get(table, record_id)
print i1['job_item_id'], i1['third_param']
print get_primary_key(table)
print scan_all(table, "fourth_param", "OINoin")
print query_first(table, "fourth_param", "OINoin")
delete(table, record_id)
print table_exists(conn, TABLE_NAME) == True
print table_exists(conn, "fake_table") == False
drop_table(conn, TABLE_NAME)
drop_table(conn, "fake_table")
def create_test_queue_w_dead_letter(self, timeout, max_receive): queue_name = id_generator.generate_job_item_id() dead_queue_name = queue_name + "_dead" return self.create_queue_w_dead_letter(queue_name, dead_queue_name, timeout, max_receive)
def test_get_queue_name(): expected_queue_name = id_generator.generate_job_item_id() queue = get_test_queue(expected_queue_name) queue_name = sqs.get_queue_name(queue) assert queue_name == expected_queue_name
def create_test_queue_w_dead_letter(self, timeout, max_receive):
queue_name = id_generator.generate_job_item_id()
dead_queue_name = queue_name + "_dead"
return self.create_queue_w_dead_letter(queue_name, dead_queue_name,
timeout, max_receive)
def test_generate_job_item_id(): job_item_id = generate_job_item_id() assert job_item_id is not None assert "-" not in job_item_id
def test_get_queue_name():
expected_queue_name = id_generator.generate_job_item_id()
queue = get_test_queue(expected_queue_name)
queue_name = sqs.get_queue_name(queue)
assert queue_name == expected_queue_name
