def sqs_run_local(region_name, sqs_queue_name, job_num=1, run_dir="/tmp/tasks"): """ Simple code to run jobs from SQS locally USE ONLY FOR DEBUG """ sqs = boto3.resource('sqs', region_name=region_name) queue = sqs.get_queue_by_name(QueueName=sqs_queue_name) for job_i in range(job_num): while True: response = queue.receive_messages(WaitTimeSeconds=10, MaxNumberOfMessages=1) if len(response) > 0: print("Dispatching") #pool.apply_async( m = response[0] job = json.loads(m.body) m.delete() local.local_handler([job], run_dir, { 'invoker': 'SQSInvoker', 'job_i': job_i }) print("done with invocation") break else: print("no message, sleeping") time.sleep(4) print("run_local_done")
def run_jobs(self, MAXJOBS=-1, run_dir="/tmp/task"): """ run MAXJOBS in the queue MAXJOBS = -1 to run all # FIXME not multithreaded safe """ jobn = len(self.payloads) if MAXJOBS != -1: jobn = MAXJOBS jobs = self.payloads[:jobn] local.local_handler(jobs, run_dir, {'invoker': 'DummyInvoker'}) self.payloads = self.payloads[jobn:]
def _process_runner(self): while True: res = self.queue.get(block=True) local.local_handler(res, self.run_dir, {'invoker' : 'LocalInvoker'})