def test_get_queue():
fake_queue = sqs.get_queue("fake-queue")
assert fake_queue is None
queue_name = id_generator.generate_job_item_id()
queue = sqs.create_queue(queue_name)
assert queue is not None
def test_get_queue():
fake_queue = sqs.get_queue("fake-queue")
assert fake_queue is None
queue_name = id_generator.generate_job_item_id()
queue = sqs.create_queue(queue_name)
assert queue is not None
def test_delete_queue():
"""
SLOW! Takes 60 seconds to delete a queue
"""
queue_name = id_generator.generate_job_item_id()
queue = get_test_queue(queue_name)
resp = sqs.delete_queue(queue)
time.sleep(90)
queue = sqs.get_queue(queue_name)
assert queue is None
def test_delete_queue(): """ SLOW! Takes 60 seconds to delete a queue """ queue_name = id_generator.generate_job_item_id() queue = get_test_queue(queue_name) resp = sqs.delete_queue(queue) time.sleep(90) queue = sqs.get_queue(queue_name) assert queue is None
def create_job_items_from_directory(job_item, dir_path):
items_log.info("Creating job items from directory")
fileslist = []
for (dirpath, dirnames, filenames) in os.walk(dir_path):
for f in filenames:
fileslist.append({
"filename": f,
"file_path": os.path.join(dirpath,f),
})
for f in fileslist:
s3.upload_file(f['filename'], f['file_path'], S3_WORKING_INPUT_BUCKET)
create_job_item(job_item['job_id'], f['filename'], sqs.get_queue(SQS_JOB_ITEMS_QUEUE))
def create_job_items_from_directory(job_item, dir_path):
items_log.info("Creating job items from directory")
fileslist = []
for (dirpath, dirnames, filenames) in os.walk(dir_path):
for f in filenames:
fileslist.append({
"filename": f,
"file_path": os.path.join(dirpath, f),
})
for f in fileslist:
s3.upload_file(f['filename'], f['file_path'], S3_WORKING_INPUT_BUCKET)
create_job_item(job_item['job_id'], f['filename'],
sqs.get_queue(SQS_JOB_ITEMS_QUEUE))
def calculate_optimal_job_item_container_capacity(queue, itms_per_cntr):
"""
Checks size of JobItems queue and compares with constant JOB_ITEMS_PER_CONTAINER
"""
current_time = datetime.datetime.utcnow()
last_updated_job_item_time = datetime.datetime.utcnow() #job_manager.get_last_updated_job().last_updated
seconds_since_last_job_item = int((current_time - last_updated_job_item_time).total_seconds())
job_items_queue = sqs.get_queue(queue)
queue_size = sqs.get_queue_size(job_items_queue.url)
optimal_containers = calculate_job_item_containers(queue_size, itms_per_cntr)
msg = ("SITUATION:\nCurrentTime: %s\nLast_Updated: %s\nSecondsSinceLastUpdate: " +
"%s\nJobItemQueueSize: %s\nOptimalContainers: %s")
msg = msg % (current_time, last_updated_job_item_time, seconds_since_last_job_item,
queue_size, optimal_containers)
tasks_log.info(msg)
return optimal_containers
def create_job(user, data):
plugin_name = data['plugin']['name']
method = data['plugin']['method']
settings = data['plugin']['settings']
output_file_suffix = get_output_file_suffix(plugin_name, settings)
channel = data['plugin']['settings']['params']['channel']
job = Job(user.id, 1, data['outputDir'], plugin_name, method, channel,
output_file_suffix)
db.add(job)
db.commit()
job_item_queue = sqs.get_queue(SQS_JOB_ITEMS_QUEUE)
for f in data['filenames']:
job_item_doc = job_item_manager.build_job_item_doc(job, f)
job_item_manager.store_job_item_doc(job_item_doc)
job_item_manager.add_job_item_to_queue(job_item_doc.job_item_key,
job_item_queue)
return job.job_id
def process_next_job_item():
tasks_log.info("Getting next job_item from queue")
queue = sqs.get_queue(client_constants.SQS_JOB_ITEMS_QUEUE)
msg = sqs.get_next_message(queue)
if msg is None:
tasks_log.info("No job items found in Queue")
return
job_item_key = msg['MessageAttributes']['job_item_key']['StringValue']
tasks_log.info("Found new job_item " + job_item_key)
job_item = job_item_manager.get_job_item_doc(job_item_key)
job_item['attempts'] += 1
status = job_item_manager.process_job_item(job_item)
if status == "COMPLETE":
items_log.info("Deleting completed job_item from queue")
sqs.delete_message(queue, msg)
else:
# We are going to let SQS handle retries
items_log.info("Leaving job_item in queue")
def create_job(user, data): plugin_name = data['plugin']['name'] method = data['plugin']['method'] settings = data['plugin']['settings'] output_file_suffix = get_output_file_suffix(plugin_name, settings) channel = data['plugin']['settings']['params']['channel'] job = Job(user.id, 1, data['outputDir'], plugin_name, method, channel, output_file_suffix) db.add(job) db.commit() job_item_queue = sqs.get_queue(SQS_JOB_ITEMS_QUEUE) for f in data['filenames']: job_item_doc = job_item_manager.build_job_item_doc(job, f) job_item_manager.store_job_item_doc(job_item_doc) job_item_manager.add_job_item_to_queue(job_item_doc.job_item_key, job_item_queue) return job.job_id
def calculate_optimal_job_item_container_capacity(queue, itms_per_cntr):
"""
Checks size of JobItems queue and compares with constant JOB_ITEMS_PER_CONTAINER
"""
current_time = datetime.datetime.utcnow()
last_updated_job_item_time = datetime.datetime.utcnow(
) #job_manager.get_last_updated_job().last_updated
seconds_since_last_job_item = int(
(current_time - last_updated_job_item_time).total_seconds())
job_items_queue = sqs.get_queue(queue)
queue_size = sqs.get_queue_size(job_items_queue.url)
optimal_containers = calculate_job_item_containers(queue_size,
itms_per_cntr)
msg = (
"SITUATION:\nCurrentTime: %s\nLast_Updated: %s\nSecondsSinceLastUpdate: "
+ "%s\nJobItemQueueSize: %s\nOptimalContainers: %s")
msg = msg % (current_time, last_updated_job_item_time,
seconds_since_last_job_item, queue_size, optimal_containers)
tasks_log.info(msg)
return optimal_containers