def reset(opts, args, conf):
q, conn = aws.get_sqs_conn_queue(conf)
if q:
if opts.hard:
conn.delete_queue(q)
else:
q.clear()
def reset(opts, args, conf):
q, conn = aws.get_sqs_conn_queue(conf)
if q:
if opts.hard:
conn.delete_queue(q)
else:
q.clear()
def reset(opts, args, conf):
q, conn = aws.get_sqs_conn_queue(conf)
if q:
if opts.hard:
logging.info('Deleting queue %s', aws.get_sqs_work_queue_name(conf))
conn.delete_queue(q)
else:
logging.info('Clearing queue %s', aws.get_sqs_work_queue_name(conf))
q.clear()
def task_loop():
try:
# reset tasks
local.task_render = None
local.task_upload = None
# get SQS work queue
q = aws.get_sqs_conn_queue(conf)[0]
# Loop over tasks. There are up to two different tasks at any
# given moment that we are processing concurrently:
#
# 1. Render task -- usually a render operation.
# 2. Upload task -- a task which uploads results to S3.
while True:
# reset render task
local.task_render = None
# initialize render task object
task = State()
task.msg = None
task.proc = None
task.retcode = None
task.outdir = None
task.id = 0
task.script_name = None
# Get a task from the SQS work queue. This is normally
# a short script that renders one or more frames.
task.msg = q.read(message_attributes=['All'])
# output some debug info
logging.debug('Reading work queue')
if local.task_upload:
logging.info("Running upload task #%d", local.task_upload.id)
logging.debug(local.task_upload.__dict__)
else:
logging.info('No upload task available')
# process task
if task.msg is not None:
# assign an ID to task
local.task_id_counter += 1
task.id = local.task_id_counter
task.script_name = task.msg.message_attributes['script_name']['string_value']
# register render task
local.task_render = task
# create output directory
task.outdir = os.path.join(work_dir, "{}_out_{}".format(task.script_name, task.id))
utils.rmtree(task.outdir)
utils.mkdir(task.outdir)
# get the task script
script = task.msg.get_body()
# cd to output directory, where we will run render task from
with utils.Cd(task.outdir):
# write script file and make it executable
script_fn = "./{}".format(task.script_name)
with open(script_fn, 'w') as f:
f.write(script)
st = os.stat(script_fn)
os.chmod(script_fn, st.st_mode | (stat.S_IEXEC|stat.S_IXGRP|stat.S_IXOTH))
# run the script
task.proc = Subprocess([script_fn])
logging.info('Running render task \"%s #%d\"', local.task_render.script_name, local.task_render.id)
logging.info(script.replace("\n"," "))
logging.debug(local.task_render.__dict__)
# Wait for render & upload tasks to complete, while periodically reasserting with SQS to
# acknowledge that tasks are still pending. (If we don't reassert with SQS frequently enough,
# it will assume we died, and put our tasks back in the queue. "frequently enough" means within
# visibility_timeout.)
count = 0
while True:
reassert = (count >= visibility_timeout_reassert)
for i, task in enumerate((local.task_render, local.task_upload)):
if task:
name = task_names[i]
if task.proc is not None:
# test if process has finished
task.retcode = task.proc.poll()
if task.retcode is not None:
# process has finished
task.proc = None
# did process finish with errors?
if task.retcode != 0:
if name == 'render':
errtxt = "Render task \"{} #{}\" exited with status code {}".format(
task.script_name, task.id, task.retcode)
raise error.ValueErrorRetry(errtxt)
else:
errtxt = "Upload task #{} exited with status code {}".format(
task.id, task.retcode)
raise ValueError(errtxt)
# Process finished successfully. If upload process,
# tell SQS that the task completed successfully.
if name == 'upload':
logging.info('Finished upload task #%d', task.id)
q.delete_message(task.msg)
task.msg = None
local.task_count += 1
task_complete_accounting(local.task_count)
# Render task completed?
if name == 'render':
logging.info('Finished render task \"%s #%d\"', task.script_name, task.id)
# tell SQS that we are still working on the task
if reassert and task.proc is not None:
logging.debug('Reasserting %s task %d with SQS', name, task.id)
task.msg.change_visibility(visibility_timeout)
# break out of loop only when no pending tasks remain
if ((not local.task_render or local.task_render.proc is None)
and (not local.task_upload or local.task_upload.proc is None)):
break
# setup for next process poll iteration
if reassert:
count = 0
time.sleep(1)
count += 1
# clean up the upload task
cleanup(local.task_upload, 'upload')
local.task_upload = None
# start a concurrent upload task to commit files generated by just-completed render task to S3
if local.task_render:
local.task_render.proc = start_upload_process(opts, args, conf, local.task_render)
local.task_upload = local.task_render
local.task_render = None
# if no render or upload task, we are done (unless DONE is set to "poll")
if not local.task_render and not local.task_upload:
if read_done_file() == "poll":
logging.info('Waiting for tasks...')
time.sleep(15)
else:
logging.info('Exiting')
break
finally:
cleanup_all()
def push(opts, args, conf):
# get task script
if not opts.task_script:
logging.error('-T, --task_script option is required')
try:
with open(opts.task_script) as f:
task_script = f.read()
except Exception:
logging.error("Could not read task_script file")
sys.exit(1)
# check for shebang
try:
task_script.startswith("#!")
except Exception:
logging.error('Shebang (#!) is missing from task script: %s', opts.task_script)
sys.exit(1)
# build tasklist
tasklist = []
for fnum in xrange(opts.start, opts.end+1, opts.step):
script = task_script
start = fnum
end = min(fnum + opts.step - 1, opts.end)
for key, value in (
("$JOB_NAME", conf.get("JOB_NAME", "NONE")),
("$JOB_URL", conf.get("JOB_URL", "NONE")),
("$START", "%d" % (start,)),
("$END", "%d" % (end,)),
("$STEP", "%d" % (opts.step,))
):
script = script.replace(key, value)
if subframe_iterator_defined(opts):
for macro_list in subframe_iterator(opts):
sf_script = script
for key, value in macro_list:
sf_script = sf_script.replace(key, value)
tasklist.append(sf_script)
else:
tasklist.append(script)
# possibly randomize the task list
if opts.randomize:
random.shuffle(tasklist)
# get work queue
q = None
if not opts.dry_run:
aws.create_sqs_queue(conf)
q, conn = aws.get_sqs_conn_queue(conf)
# push work queue to sqs
i = 0
j = 0
batch = []
tasklist_last_index = len(tasklist)-1
for task in tasklist:
i += 1
j += 1
logging.debug("Creating task #%04d: %s", j, task.replace("\n"," "))
attr = {"script_name": {"data_type": "String", "string_value": os.path.basename(opts.task_script)}}
batch.append((str(i), task, 0, attr))
# Deliver up to 10 messages in a single request
# http://boto.cloudhackers.com/en/latest/ref/sqs.html#boto.sqs.queue.Queue.write_batch
if i%10 == 0 or tasklist.index(task) == tasklist_last_index:
if q and not opts.dry_run:
logging.info('Queueing tasks %d of %d', j, tasklist_last_index+1)
aws.write_batch_sqs_queue(batch, q)
del batch[:]
i = 0
def status(opts, args, conf):
q, conn = aws.get_sqs_conn_queue(conf)
if q:
logging.info("%d tasks queued on %s", q.count(), aws.get_sqs_work_queue_name(conf))