def test_live_queue(): live_queue = RedisQueue("test_queue", MockTask, namespace="pytest") assert live_queue.connected is False task = MockTask() task2 = MockTask() task.uri = 'thisIsUnique' task2.uri = 'thisIsUnique' assert live_queue.connect(host=live_host, port=live_port, password=live_pass) is True assert live_queue.connected is True live_queue.clear() assert live_queue.qsize == 0 live_queue.put(task) assert live_queue.qsize == 1 live_queue.put(task2) assert live_queue.qsize == 2 new_task = live_queue.get() assert isinstance(new_task, MockTask) assert new_task.uid == task.uid assert new_task.uri == 'thisIsUnique' live_queue.clear() task.unique = True task2.unique = True assert task.unique_hash() == task2.unique_hash() live_queue.put(task) with pytest.raises(TaskAlreadyInQueueException): live_queue.put(task2) assert live_queue.qsize == 1 live_queue.clear() # test getting and putting the same task into the queue assert live_queue.qsize == 0 live_queue.put(task) my_task = live_queue.get() live_queue.put(my_task) assert live_queue.qsize == 1
def start(): round_robin = 0 while True: try: manager_queue = RedisQueue('manager', host=settings.REDIS_HOST, port=settings.REDIS_PORT) # get next message - first find an available queue keys = RedisKeys(pattern='rendertaskqueue:*', host=settings.REDIS_HOST, port=settings.REDIS_PORT).keys if keys is not None and len(keys) > 0: round_robin = BlendClusterClient.next_queue(round_robin, keys) queue_name = keys[round_robin] # just grab the first queue name print 'round robin: ' + queue_name queue = RedisQueue('namechangedbelow', host=settings.REDIS_HOST, port=settings.REDIS_PORT) queue.key = queue_name # override key name (queue name) here as we have the qualified name print 'getting render task message' render_task_json = queue.get() print render_task_json render_task_obj = RenderTaskMessage.json_to_object(render_task_json) hg_url = render_task_obj.hg_url blend_file = render_task_obj.blend_file # notify manager we are working msg = RenderTaskStateChangeMessage.json_from_object({ 'id': render_task_obj.id, 'state': 1}) # set working state manager_queue.put(msg) print 'Set task to WORKING state: ' + str(render_task_obj.id) print '-------------------------' print 'WORKING' print '-------------------------' # so some work print 'About to prepare hg' hg = HgUtils(url=hg_url) print 'about to clone or pull hg' hg.clone_or_pull() full_blend_filename = hg.local_project_path + '/' + render_task_obj.blend_file short_blend_filename = ntpath.basename(full_blend_filename).split('.')[0] render_dir = hg.local_project_path + '/renders/' + short_blend_filename + '/'\ + render_task_obj.scene_name + '/' print 'Rendering in dir: ' + render_dir if not os.path.exists(render_dir): os.makedirs(render_dir) hg.push_all_changes() blender = Blender(blend_file=full_blend_filename, scene_name=render_task_obj.scene_name, start_frame=render_task_obj.frame, end_frame=render_task_obj.frame, output_dir=render_dir) blender.run() # before pushing changes update local repo hg.get_latest_changes() hg.push_all_changes() print 'done hg' msg = RenderTaskStateChangeMessage.json_from_object({ 'id': render_task_obj.id, 'state': 4}) # set success state manager_queue.put(msg) print '-------------------------' print 'SUCCESSFUL' print '-------------------------' else: print('No messages available right now, will try again in 5 seconds...') time.sleep(5) # wait 5 secs before trying again except Exception as e: print('Error raised: ' + e.message) try: msg = RenderTaskStateChangeMessage.json_from_object({ 'id': render_task_obj.id, 'state': 3}) # set working state failed manager_queue.put(msg) print '-------------------------' print 'FAILED' print '-------------------------' except: print('Trying to respond with a failure status but could not!') print('Will try again in 5 seconds...') time.sleep(5) # wait 5 secs before trying again
if ex[0]==17: # directory already exists pass else: raise def makeprocessqentry(config, jobid, path, imgname, timestamp, size): return { "action": "process_file", "jobid": jobid, "path": path, "name": imgname, "changed": timestamp, "filesize": size } if __name__=='__main__': config= json.load(open("config.json")) makedatadir(config) downloadq= RedisQueue(host=config["redis-host"], namespace=config["redis-namespace"], name=config["redis-download-queue"]) processq= RedisQueue(host=config["redis-host"], namespace=config["redis-namespace"], name=config["redis-process-queue"]) session= requests.Session() processq.clear() while True: row= json.loads(downloadq.get()) print "%s => " % row['name'].encode('utf-8'), r= session.get(row['url']) print(r.status_code) if r.status_code!=200: raise RuntimeError("requests.get(%s) returned %s" % (row['url'], r.status_code)) outputpath= os.path.join(os.path.expanduser(config["download-dir"]), row["name"]) with open(outputpath, "w") as f: f.write(r.content) processq.put(json.dumps(makeprocessqentry(config, row["jobid"], outputpath, row["name"], row["changed"], row["filesize"])))
from redisqueue import RedisQueue from redishash import RedisHash def sqlconn(config): conn= MySQLdb.connect( read_default_file=os.path.expanduser('~/replica.my.cnf'), host='tools.labsdb', use_unicode=True, cursorclass=MySQLdb.cursors.DictCursor ) cursor= conn.cursor() return conn,cursor def writeresumefile(config, job): dir= config["download-dir"] tmpfile= os.path.join(dir, "resume.tmp") resumefile= os.path.join(dir, "resume.json") with open(tmpfile, "w") as f: f.write(json.dumps(job)) os.rename(tmpfile, resumefile) if __name__=='__main__': config= json.load(open("config.json")) joborderq= RedisQueue(host=config["redis-host"], namespace=config["redis-namespace"], name=config["redis-job-order-queue"]) resulthash= RedisHash(host=config["redis-host"], namespace=config["redis-namespace"], name=config["redis-result-hash"]) conn,cursor= sqlconn(config) while True: nextjob= json.loads(joborderq.get()) while not nextjob["jobid"] in resulthash: time.sleep(1.0) result= resulthash[nextjob["jobid"]] print("got result: %s" % json.dumps(result)) # xxx write result to db writeresumefile(config, nextjob)
class Worker(Process): def __init__(self, event_kill, name, concurrency, ip, max_time=-1): self.queue_name = name self.concurrency = concurrency self.max_time = max_time self.event_kill = event_kill self.conn = RedisQueue(Redis(host=ip), self.queue_name) super(Worker, self).__init__() def safe_worker(self, func, return_dict, apply_max_time, body): try: return_dict['result'] = call(func, apply_max_time, *body['args'], **body['kwargs']) return_dict['success'] = True except: return_dict['result'] = traceback.format_exc() return_dict['success'] = False logging.error(return_dict['result']) def safe_call(self, func, apply_max_time, body): # os.setpgrp() # kill non propagate if 'gevent' not in sys.modules: return_dict = Manager().dict() p = Process(target=self.safe_worker, args=(func, return_dict, apply_max_time, body)) p.start() p.join() else: return_dict = {} self.safe_worker(func, return_dict, apply_max_time, body) return return_dict def callback(self, body): logging.info("execute %s" % body['event']) _id = body['args'][0] if settings.ROCKETTM_CALLBACK: send('results', {'_id': _id, 'status': 'processing'}) if not body['event'] in tasks.subs: if settings.ROCKETTM_CALLBACK: send('results', {'_id': _id, 'result': 'task not defined', 'status': 'finished', 'success': False}) return False result = [] for func, max_time2 in tasks.subs[body['event']]: logging.info("exec func: %s, timeout: %s" % (func, max_time2)) if max_time2 != -1: apply_max_time = max_time2 else: apply_max_time = self.max_time result.append(dict(self.safe_call(func, apply_max_time, body))) success = not any(r['success'] is False for r in result) send('results', {'_id': _id, 'status': 'finished', 'success': success, 'result': result}) return True def run(self): while not self.event_kill.is_set(): try: task = self.conn.get(timeout=20) if task: self.callback(task) except: logging.error(traceback.format_exc()) logging.error("connection loss, try reconnect") time.sleep(5)
# You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. import os, sys import time import json import redis from redisqueue import RedisQueue from redishash import RedisHash def process(job): if job["action"]=="process_file": # { "jobid", "path", "name", "changed", "filesize" } # ... do stuff ... os.remove(job["path"]) # return fake entry return { "status": "OK", "metadata": { "foo": 47 } } else: raise runtime_error("unknown job action '%s'" % job["action"]) if __name__=='__main__': config= json.load(open("config.json")) processq= RedisQueue(host=config["redis-host"], namespace=config["redis-namespace"], name=config["redis-process-queue"]) resulthash= RedisHash(host=config["redis-host"], namespace=config["redis-namespace"], name=config["redis-result-hash"]) resulthash.clear() while True: job= json.loads(processq.get()) print("processing job %s..." % job["jobid"]) result= process(job) resulthash[job["jobid"]]= result