def create_job(user, data): plugin_name = data['plugin']['name'] method = data['plugin']['method'] settings = data['plugin']['settings'] output_file_suffix = get_output_file_suffix(plugin_name, settings) channel = data['plugin']['settings']['params']['channel'] job = Job(user.id, 1, data['outputDir'], plugin_name, method, channel, output_file_suffix) db.add(job) db.commit() job_item_queue = sqs.get_queue(SQS_JOB_ITEMS_QUEUE) for f in data['filenames']: job_item_doc = job_item_manager.build_job_item_doc(job, f) job_item_manager.store_job_item_doc(job_item_doc) job_item_manager.add_job_item_to_queue(job_item_doc.job_item_key, job_item_queue) return job.job_id
# Dynamo - Drop and recreate DB dynamo_conn = dynamo.get_connection() dynamo.drop_table(dynamo_conn, DYNAMO_JOB_ITEMS_TABLE) dynamo.create_table_w_index(dynamo_conn, DYNAMO_JOB_ITEMS_TABLE, "job_item_key", "S", "job_index", "job_id", "N") print "Created Job Items table" # Dynamo - Insert Test Data job_item_doc1 = job_item_manager.build_job_item_doc(job, VAA3D_TEST_INPUT_FILE_1) job_item_doc2 = job_item_manager.build_job_item_doc(job, VAA3D_TEST_INPUT_FILE_2) job_item_doc3 = job_item_manager.build_job_item_doc(job, VAA3D_TEST_INPUT_FILE_3) # job_item_doc4 = job_item_manager.build_job_item_doc(job, VAA3D_TEST_INPUT_FILE_4) job_item_manager.store_job_item_doc(job_item_doc1) job_item_manager.store_job_item_doc(job_item_doc2) job_item_manager.store_job_item_doc(job_item_doc3) # job_item_manager.store_job_item_doc(job_item_doc4) # Drop and Recreate SQS queues queue = sqs.drop_and_recreate_queue( SQS_JOB_ITEMS_QUEUE, SQS_JOB_ITEMS_DEAD_LETTER, SQS_VISIBILITY_TIMEOUT, SQS_MAX_RECEIVES ) # Add job_items to SQS job_item_manager.add_job_item_to_queue(job_item_doc1.job_item_key, queue) job_item_manager.add_job_item_to_queue(job_item_doc2.job_item_key, queue) job_item_manager.add_job_item_to_queue(job_item_doc3.job_item_key, queue) # job_item_manager.add_job_item_to_queue(job_item_doc4.job_item_key, queue) print "Loaded test job item data"
'job_item_key', 'S', 'job_index', 'job_id', 'N') print "Created Job Items table" # Dynamo - Insert Test Data job_item_doc1 = job_item_manager.build_job_item_doc(job, VAA3D_TEST_INPUT_FILE_1) job_item_doc2 = job_item_manager.build_job_item_doc(job, VAA3D_TEST_INPUT_FILE_2) job_item_doc3 = job_item_manager.build_job_item_doc(job, VAA3D_TEST_INPUT_FILE_3) #job_item_doc4 = job_item_manager.build_job_item_doc(job, VAA3D_TEST_INPUT_FILE_4) job_item_manager.store_job_item_doc(job_item_doc1) job_item_manager.store_job_item_doc(job_item_doc2) job_item_manager.store_job_item_doc(job_item_doc3) #job_item_manager.store_job_item_doc(job_item_doc4) # Drop and Recreate SQS queues queue = sqs.drop_and_recreate_queue(SQS_JOB_ITEMS_QUEUE, SQS_JOB_ITEMS_DEAD_LETTER, SQS_VISIBILITY_TIMEOUT, SQS_MAX_RECEIVES) # Add job_items to SQS job_item_manager.add_job_item_to_queue(job_item_doc1.job_item_key, queue) job_item_manager.add_job_item_to_queue(job_item_doc2.job_item_key, queue) job_item_manager.add_job_item_to_queue(job_item_doc3.job_item_key, queue) #job_item_manager.add_job_item_to_queue(job_item_doc4.job_item_key, queue) print "Loaded test job item data"