def test_get_queue(): fake_queue = sqs.get_queue("fake-queue") assert fake_queue is None queue_name = id_generator.generate_job_item_id() queue = sqs.create_queue(queue_name) assert queue is not None
def test_get_queue(): fake_queue = sqs.get_queue("fake-queue") assert fake_queue is None queue_name = id_generator.generate_job_item_id() queue = sqs.create_queue(queue_name) assert queue is not None
def test_delete_queue(): """ SLOW! Takes 60 seconds to delete a queue """ queue_name = id_generator.generate_job_item_id() queue = get_test_queue(queue_name) resp = sqs.delete_queue(queue) time.sleep(90) queue = sqs.get_queue(queue_name) assert queue is None
def test_delete_queue(): """ SLOW! Takes 60 seconds to delete a queue """ queue_name = id_generator.generate_job_item_id() queue = get_test_queue(queue_name) resp = sqs.delete_queue(queue) time.sleep(90) queue = sqs.get_queue(queue_name) assert queue is None
def create_job_items_from_directory(job_item, dir_path): items_log.info("Creating job items from directory") fileslist = [] for (dirpath, dirnames, filenames) in os.walk(dir_path): for f in filenames: fileslist.append({ "filename": f, "file_path": os.path.join(dirpath,f), }) for f in fileslist: s3.upload_file(f['filename'], f['file_path'], S3_WORKING_INPUT_BUCKET) create_job_item(job_item['job_id'], f['filename'], sqs.get_queue(SQS_JOB_ITEMS_QUEUE))
def create_job_items_from_directory(job_item, dir_path): items_log.info("Creating job items from directory") fileslist = [] for (dirpath, dirnames, filenames) in os.walk(dir_path): for f in filenames: fileslist.append({ "filename": f, "file_path": os.path.join(dirpath, f), }) for f in fileslist: s3.upload_file(f['filename'], f['file_path'], S3_WORKING_INPUT_BUCKET) create_job_item(job_item['job_id'], f['filename'], sqs.get_queue(SQS_JOB_ITEMS_QUEUE))
def calculate_optimal_job_item_container_capacity(queue, itms_per_cntr): """ Checks size of JobItems queue and compares with constant JOB_ITEMS_PER_CONTAINER """ current_time = datetime.datetime.utcnow() last_updated_job_item_time = datetime.datetime.utcnow() #job_manager.get_last_updated_job().last_updated seconds_since_last_job_item = int((current_time - last_updated_job_item_time).total_seconds()) job_items_queue = sqs.get_queue(queue) queue_size = sqs.get_queue_size(job_items_queue.url) optimal_containers = calculate_job_item_containers(queue_size, itms_per_cntr) msg = ("SITUATION:\nCurrentTime: %s\nLast_Updated: %s\nSecondsSinceLastUpdate: " + "%s\nJobItemQueueSize: %s\nOptimalContainers: %s") msg = msg % (current_time, last_updated_job_item_time, seconds_since_last_job_item, queue_size, optimal_containers) tasks_log.info(msg) return optimal_containers
def create_job(user, data): plugin_name = data['plugin']['name'] method = data['plugin']['method'] settings = data['plugin']['settings'] output_file_suffix = get_output_file_suffix(plugin_name, settings) channel = data['plugin']['settings']['params']['channel'] job = Job(user.id, 1, data['outputDir'], plugin_name, method, channel, output_file_suffix) db.add(job) db.commit() job_item_queue = sqs.get_queue(SQS_JOB_ITEMS_QUEUE) for f in data['filenames']: job_item_doc = job_item_manager.build_job_item_doc(job, f) job_item_manager.store_job_item_doc(job_item_doc) job_item_manager.add_job_item_to_queue(job_item_doc.job_item_key, job_item_queue) return job.job_id
def process_next_job_item(): tasks_log.info("Getting next job_item from queue") queue = sqs.get_queue(client_constants.SQS_JOB_ITEMS_QUEUE) msg = sqs.get_next_message(queue) if msg is None: tasks_log.info("No job items found in Queue") return job_item_key = msg['MessageAttributes']['job_item_key']['StringValue'] tasks_log.info("Found new job_item " + job_item_key) job_item = job_item_manager.get_job_item_doc(job_item_key) job_item['attempts'] += 1 status = job_item_manager.process_job_item(job_item) if status == "COMPLETE": items_log.info("Deleting completed job_item from queue") sqs.delete_message(queue, msg) else: # We are going to let SQS handle retries items_log.info("Leaving job_item in queue")
def create_job(user, data): plugin_name = data['plugin']['name'] method = data['plugin']['method'] settings = data['plugin']['settings'] output_file_suffix = get_output_file_suffix(plugin_name, settings) channel = data['plugin']['settings']['params']['channel'] job = Job(user.id, 1, data['outputDir'], plugin_name, method, channel, output_file_suffix) db.add(job) db.commit() job_item_queue = sqs.get_queue(SQS_JOB_ITEMS_QUEUE) for f in data['filenames']: job_item_doc = job_item_manager.build_job_item_doc(job, f) job_item_manager.store_job_item_doc(job_item_doc) job_item_manager.add_job_item_to_queue(job_item_doc.job_item_key, job_item_queue) return job.job_id
def calculate_optimal_job_item_container_capacity(queue, itms_per_cntr): """ Checks size of JobItems queue and compares with constant JOB_ITEMS_PER_CONTAINER """ current_time = datetime.datetime.utcnow() last_updated_job_item_time = datetime.datetime.utcnow( ) #job_manager.get_last_updated_job().last_updated seconds_since_last_job_item = int( (current_time - last_updated_job_item_time).total_seconds()) job_items_queue = sqs.get_queue(queue) queue_size = sqs.get_queue_size(job_items_queue.url) optimal_containers = calculate_job_item_containers(queue_size, itms_per_cntr) msg = ( "SITUATION:\nCurrentTime: %s\nLast_Updated: %s\nSecondsSinceLastUpdate: " + "%s\nJobItemQueueSize: %s\nOptimalContainers: %s") msg = msg % (current_time, last_updated_job_item_time, seconds_since_last_job_item, queue_size, optimal_containers) tasks_log.info(msg) return optimal_containers