def sqs_run_local(region_name,
sqs_queue_name,
job_num=1,
run_dir="/tmp/tasks"):
"""
Simple code to run jobs from SQS locally
USE ONLY FOR DEBUG
"""
sqs = boto3.resource('sqs', region_name=region_name)
queue = sqs.get_queue_by_name(QueueName=sqs_queue_name)
for job_i in range(job_num):
while True:
response = queue.receive_messages(WaitTimeSeconds=10,
MaxNumberOfMessages=1)
if len(response) > 0:
print("Dispatching")
#pool.apply_async(
m = response[0]
job = json.loads(m.body)
m.delete()
local.local_handler([job], run_dir, {
'invoker': 'SQSInvoker',
'job_i': job_i
})
print("done with invocation")
break
else:
print("no message, sleeping")
time.sleep(4)
print("run_local_done")
def run_jobs(self, MAXJOBS=-1, run_dir="/tmp/task"):
"""
run MAXJOBS in the queue
MAXJOBS = -1 to run all
# FIXME not multithreaded safe
"""
jobn = len(self.payloads)
if MAXJOBS != -1:
jobn = MAXJOBS
jobs = self.payloads[:jobn]
local.local_handler(jobs, run_dir, {'invoker': 'DummyInvoker'})
self.payloads = self.payloads[jobn:]
def _process_runner(self):
while True:
res = self.queue.get(block=True)
local.local_handler(res, self.run_dir,
{'invoker' : 'LocalInvoker'})
