def test_live_queue():
live_queue = RedisQueue("test_queue", MockTask, namespace="pytest")
assert live_queue.connected is False
task = MockTask()
task2 = MockTask()
task.uri = 'thisIsUnique'
task2.uri = 'thisIsUnique'
assert live_queue.connect(host=live_host, port=live_port,
password=live_pass) is True
assert live_queue.connected is True
live_queue.clear()
assert live_queue.qsize == 0
live_queue.put(task)
assert live_queue.qsize == 1
live_queue.put(task2)
assert live_queue.qsize == 2
new_task = live_queue.get()
assert isinstance(new_task, MockTask)
assert new_task.uid == task.uid
assert new_task.uri == 'thisIsUnique'
live_queue.clear()
task.unique = True
task2.unique = True
assert task.unique_hash() == task2.unique_hash()
live_queue.put(task)
with pytest.raises(TaskAlreadyInQueueException):
live_queue.put(task2)
assert live_queue.qsize == 1
live_queue.clear()
# test getting and putting the same task into the queue
assert live_queue.qsize == 0
live_queue.put(task)
my_task = live_queue.get()
live_queue.put(my_task)
assert live_queue.qsize == 1
def start():
round_robin = 0
while True:
try:
manager_queue = RedisQueue('manager', host=settings.REDIS_HOST, port=settings.REDIS_PORT)
# get next message - first find an available queue
keys = RedisKeys(pattern='rendertaskqueue:*', host=settings.REDIS_HOST, port=settings.REDIS_PORT).keys
if keys is not None and len(keys) > 0:
round_robin = BlendClusterClient.next_queue(round_robin, keys)
queue_name = keys[round_robin] # just grab the first queue name
print 'round robin: ' + queue_name
queue = RedisQueue('namechangedbelow', host=settings.REDIS_HOST, port=settings.REDIS_PORT)
queue.key = queue_name # override key name (queue name) here as we have the qualified name
print 'getting render task message'
render_task_json = queue.get()
print render_task_json
render_task_obj = RenderTaskMessage.json_to_object(render_task_json)
hg_url = render_task_obj.hg_url
blend_file = render_task_obj.blend_file
# notify manager we are working
msg = RenderTaskStateChangeMessage.json_from_object({ 'id': render_task_obj.id,
'state': 1}) # set working state
manager_queue.put(msg)
print 'Set task to WORKING state: ' + str(render_task_obj.id)
print '-------------------------'
print 'WORKING'
print '-------------------------'
# so some work
print 'About to prepare hg'
hg = HgUtils(url=hg_url)
print 'about to clone or pull hg'
hg.clone_or_pull()
full_blend_filename = hg.local_project_path + '/' + render_task_obj.blend_file
short_blend_filename = ntpath.basename(full_blend_filename).split('.')[0]
render_dir = hg.local_project_path + '/renders/' + short_blend_filename + '/'\
+ render_task_obj.scene_name + '/'
print 'Rendering in dir: ' + render_dir
if not os.path.exists(render_dir):
os.makedirs(render_dir)
hg.push_all_changes()
blender = Blender(blend_file=full_blend_filename, scene_name=render_task_obj.scene_name,
start_frame=render_task_obj.frame, end_frame=render_task_obj.frame,
output_dir=render_dir)
blender.run()
# before pushing changes update local repo
hg.get_latest_changes()
hg.push_all_changes()
print 'done hg'
msg = RenderTaskStateChangeMessage.json_from_object({ 'id': render_task_obj.id,
'state': 4}) # set success state
manager_queue.put(msg)
print '-------------------------'
print 'SUCCESSFUL'
print '-------------------------'
else:
print('No messages available right now, will try again in 5 seconds...')
time.sleep(5) # wait 5 secs before trying again
except Exception as e:
print('Error raised: ' + e.message)
try:
msg = RenderTaskStateChangeMessage.json_from_object({ 'id': render_task_obj.id,
'state': 3}) # set working state failed
manager_queue.put(msg)
print '-------------------------'
print 'FAILED'
print '-------------------------'
except:
print('Trying to respond with a failure status but could not!')
print('Will try again in 5 seconds...')
time.sleep(5) # wait 5 secs before trying again
if ex[0]==17: # directory already exists
pass
else:
raise
def makeprocessqentry(config, jobid, path, imgname, timestamp, size):
return { "action": "process_file", "jobid": jobid, "path": path, "name": imgname, "changed": timestamp, "filesize": size }
if __name__=='__main__':
config= json.load(open("config.json"))
makedatadir(config)
downloadq= RedisQueue(host=config["redis-host"], namespace=config["redis-namespace"], name=config["redis-download-queue"])
processq= RedisQueue(host=config["redis-host"], namespace=config["redis-namespace"], name=config["redis-process-queue"])
session= requests.Session()
processq.clear()
while True:
row= json.loads(downloadq.get())
print "%s => " % row['name'].encode('utf-8'),
r= session.get(row['url'])
print(r.status_code)
if r.status_code!=200:
raise RuntimeError("requests.get(%s) returned %s" % (row['url'], r.status_code))
outputpath= os.path.join(os.path.expanduser(config["download-dir"]), row["name"])
with open(outputpath, "w") as f:
f.write(r.content)
processq.put(json.dumps(makeprocessqentry(config, row["jobid"], outputpath, row["name"], row["changed"], row["filesize"])))
from redisqueue import RedisQueue
from redishash import RedisHash
def sqlconn(config):
conn= MySQLdb.connect( read_default_file=os.path.expanduser('~/replica.my.cnf'), host='tools.labsdb', use_unicode=True, cursorclass=MySQLdb.cursors.DictCursor )
cursor= conn.cursor()
return conn,cursor
def writeresumefile(config, job):
dir= config["download-dir"]
tmpfile= os.path.join(dir, "resume.tmp")
resumefile= os.path.join(dir, "resume.json")
with open(tmpfile, "w") as f:
f.write(json.dumps(job))
os.rename(tmpfile, resumefile)
if __name__=='__main__':
config= json.load(open("config.json"))
joborderq= RedisQueue(host=config["redis-host"], namespace=config["redis-namespace"], name=config["redis-job-order-queue"])
resulthash= RedisHash(host=config["redis-host"], namespace=config["redis-namespace"], name=config["redis-result-hash"])
conn,cursor= sqlconn(config)
while True:
nextjob= json.loads(joborderq.get())
while not nextjob["jobid"] in resulthash:
time.sleep(1.0)
result= resulthash[nextjob["jobid"]]
print("got result: %s" % json.dumps(result))
# xxx write result to db
writeresumefile(config, nextjob)
class Worker(Process):
def __init__(self, event_kill, name, concurrency, ip, max_time=-1):
self.queue_name = name
self.concurrency = concurrency
self.max_time = max_time
self.event_kill = event_kill
self.conn = RedisQueue(Redis(host=ip), self.queue_name)
super(Worker, self).__init__()
def safe_worker(self, func, return_dict, apply_max_time, body):
try:
return_dict['result'] = call(func, apply_max_time,
*body['args'], **body['kwargs'])
return_dict['success'] = True
except:
return_dict['result'] = traceback.format_exc()
return_dict['success'] = False
logging.error(return_dict['result'])
def safe_call(self, func, apply_max_time, body):
# os.setpgrp() # kill non propagate
if 'gevent' not in sys.modules:
return_dict = Manager().dict()
p = Process(target=self.safe_worker, args=(func, return_dict,
apply_max_time, body))
p.start()
p.join()
else:
return_dict = {}
self.safe_worker(func, return_dict, apply_max_time, body)
return return_dict
def callback(self, body):
logging.info("execute %s" % body['event'])
_id = body['args'][0]
if settings.ROCKETTM_CALLBACK:
send('results', {'_id': _id, 'status': 'processing'})
if not body['event'] in tasks.subs:
if settings.ROCKETTM_CALLBACK:
send('results', {'_id': _id,
'result': 'task not defined',
'status': 'finished',
'success': False})
return False
result = []
for func, max_time2 in tasks.subs[body['event']]:
logging.info("exec func: %s, timeout: %s" % (func, max_time2))
if max_time2 != -1:
apply_max_time = max_time2
else:
apply_max_time = self.max_time
result.append(dict(self.safe_call(func, apply_max_time, body)))
success = not any(r['success'] is False for r in result)
send('results', {'_id': _id, 'status': 'finished',
'success': success, 'result': result})
return True
def run(self):
while not self.event_kill.is_set():
try:
task = self.conn.get(timeout=20)
if task:
self.callback(task)
except:
logging.error(traceback.format_exc())
logging.error("connection loss, try reconnect")
time.sleep(5)
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import os, sys
import time
import json
import redis
from redisqueue import RedisQueue
from redishash import RedisHash
def process(job):
if job["action"]=="process_file":
# { "jobid", "path", "name", "changed", "filesize" }
# ... do stuff ...
os.remove(job["path"])
# return fake entry
return { "status": "OK", "metadata": { "foo": 47 } }
else:
raise runtime_error("unknown job action '%s'" % job["action"])
if __name__=='__main__':
config= json.load(open("config.json"))
processq= RedisQueue(host=config["redis-host"], namespace=config["redis-namespace"], name=config["redis-process-queue"])
resulthash= RedisHash(host=config["redis-host"], namespace=config["redis-namespace"], name=config["redis-result-hash"])
resulthash.clear()
while True:
job= json.loads(processq.get())
print("processing job %s..." % job["jobid"])
result= process(job)
resulthash[job["jobid"]]= result
