def get_job_items(job_id, include_zip=True): dynamo_conn = dynamo.get_connection() table = dynamo.get_table(dynamo_conn, DYNAMO_JOB_ITEMS_TABLE) job_items = dynamo.query_all(table, "job_index", "job_id", job_id) link_expiry_secs = 3600 # 1 hour s3_conn = s3.S3Connection(VAA3D_USER_AWS_ACCESS_KEY, VAA3D_USER_AWS_SECRET_KEY) job_items_list = [] for item in job_items: if include_zip or not zipper.is_compressed_filename(item['input_filename']): item_dict = job_item_manager.convert_dynamo_job_item_to_dict(item) item_dict['job_item_status'] = JobItemStatus.query.get(int(item_dict['status_id'])).status_name output_s3_key = item_dict['output_dir'] + "/" + item_dict['output_filename'] item_dict['download_url'] = s3.get_download_url(s3_conn, S3_OUTPUT_BUCKET, output_s3_key, link_expiry_secs) logs_s3_key = item_dict['output_dir'] + "/logs/" + item_dict['output_filename'] + USER_JOB_LOG_EXT item_dict['logs_download_url'] = s3.get_download_url(s3_conn, S3_OUTPUT_BUCKET, logs_s3_key, link_expiry_secs) job_items_list.append(item_dict) return job_items_list
def get_job_items(job_id, include_zip=True):
dynamo_conn = dynamo.get_connection()
table = dynamo.get_table(dynamo_conn, DYNAMO_JOB_ITEMS_TABLE)
job_items = dynamo.query_all(table, "job_index", "job_id", job_id)
link_expiry_secs = 3600 # 1 hour
s3_conn = s3.S3Connection(VAA3D_USER_AWS_ACCESS_KEY,
VAA3D_USER_AWS_SECRET_KEY)
job_items_list = []
for item in job_items:
if include_zip or not zipper.is_compressed_filename(
item['input_filename']):
item_dict = job_item_manager.convert_dynamo_job_item_to_dict(item)
item_dict['job_item_status'] = JobItemStatus.query.get(
int(item_dict['status_id'])).status_name
output_s3_key = item_dict['output_dir'] + "/" + item_dict[
'output_filename']
item_dict['download_url'] = s3.get_download_url(
s3_conn, S3_OUTPUT_BUCKET, output_s3_key, link_expiry_secs)
logs_s3_key = item_dict['output_dir'] + "/logs/" + item_dict[
'output_filename'] + USER_JOB_LOG_EXT
item_dict['logs_download_url'] = s3.get_download_url(
s3_conn, S3_OUTPUT_BUCKET, logs_s3_key, link_expiry_secs)
job_items_list.append(item_dict)
return job_items_list
"testjob1",
VAA3D_DEFAULT_PLUGIN,
VAA3D_DEFAULT_FUNC,
1,
OUTPUT_FILE_SUFFIXES[VAA3D_DEFAULT_PLUGIN],
)
db.add(job)
db.commit()
print "Loaded test job data"
# Insert Test Job Items
job = Job.query.first()
# Dynamo - Drop and recreate DB
dynamo_conn = dynamo.get_connection()
dynamo.drop_table(dynamo_conn, DYNAMO_JOB_ITEMS_TABLE)
dynamo.create_table_w_index(dynamo_conn, DYNAMO_JOB_ITEMS_TABLE, "job_item_key", "S", "job_index", "job_id", "N")
print "Created Job Items table"
# Dynamo - Insert Test Data
job_item_doc1 = job_item_manager.build_job_item_doc(job, VAA3D_TEST_INPUT_FILE_1)
job_item_doc2 = job_item_manager.build_job_item_doc(job, VAA3D_TEST_INPUT_FILE_2)
job_item_doc3 = job_item_manager.build_job_item_doc(job, VAA3D_TEST_INPUT_FILE_3)
# job_item_doc4 = job_item_manager.build_job_item_doc(job, VAA3D_TEST_INPUT_FILE_4)
job_item_manager.store_job_item_doc(job_item_doc1)
job_item_manager.store_job_item_doc(job_item_doc2)
job_item_manager.store_job_item_doc(job_item_doc3)
# job_item_manager.store_job_item_doc(job_item_doc4)
# Insert Test Job
job = Job(
User.query.filter_by(iam_username=DEFAULT_IAM_USER).first().id, 1,
'testjob1', VAA3D_DEFAULT_PLUGIN, VAA3D_DEFAULT_FUNC, 1,
OUTPUT_FILE_SUFFIXES[VAA3D_DEFAULT_PLUGIN])
db.add(job)
db.commit()
print "Loaded test job data"
# Insert Test Job Items
job = Job.query.first()
# Dynamo - Drop and recreate DB
dynamo_conn = dynamo.get_connection()
dynamo.drop_table(dynamo_conn, DYNAMO_JOB_ITEMS_TABLE)
dynamo.create_table_w_index(dynamo_conn, DYNAMO_JOB_ITEMS_TABLE,
'job_item_key', 'S', 'job_index', 'job_id', 'N')
print "Created Job Items table"
# Dynamo - Insert Test Data
job_item_doc1 = job_item_manager.build_job_item_doc(job,
VAA3D_TEST_INPUT_FILE_1)
job_item_doc2 = job_item_manager.build_job_item_doc(job,
VAA3D_TEST_INPUT_FILE_2)
job_item_doc3 = job_item_manager.build_job_item_doc(job,
VAA3D_TEST_INPUT_FILE_3)
#job_item_doc4 = job_item_manager.build_job_item_doc(job, VAA3D_TEST_INPUT_FILE_4)
def save_job_item(job_item):
conn = dynamo.get_connection()
table = dynamo.get_table(conn, DYNAMO_JOB_ITEMS_TABLE)
table.put_item(Item=job_item)
def get_job_item_doc(job_item_key):
# This can be modified like a dictionary, and saved to Dynamo
conn = dynamo.get_connection()
table = dynamo.get_table(conn, DYNAMO_JOB_ITEMS_TABLE)
return dynamo.get(table, job_item_key)
def store_job_item_doc(job_item_doc):
conn = dynamo.get_connection()
table = dynamo.get_table(conn, DYNAMO_JOB_ITEMS_TABLE)
dynamo.insert(table, job_item_doc.as_dict())
def save_job_item(job_item): conn = dynamo.get_connection() table = dynamo.get_table(conn, DYNAMO_JOB_ITEMS_TABLE) table.put_item(Item=job_item)
def get_job_item_doc(job_item_key): # This can be modified like a dictionary, and saved to Dynamo conn = dynamo.get_connection() table = dynamo.get_table(conn, DYNAMO_JOB_ITEMS_TABLE) return dynamo.get(table, job_item_key)
def store_job_item_doc(job_item_doc): conn = dynamo.get_connection() table = dynamo.get_table(conn, DYNAMO_JOB_ITEMS_TABLE) dynamo.insert(table, job_item_doc.as_dict())
