def backoff_request(self, url, method, payload=None, num_attempts=1, initial_wait=0, need_result=True):
initial_wait = 5
for _ in range(0, num_attempts):
if self.stop_event.is_set():
break
try:
try:
if method == "POST":
if need_result or num_attempts > 1:
content = post_string(url, payload)
else:
post_string_noreturn(url, payload, result_callback=self.master_post_result_callback)
return
elif method == "GET":
content = get_string(url)
else:
raise Exception("Invalid method %s" % method)
return 200, content
except Exception as e:
ciel.log("Backoff-request failed with exception %s; re-raising MasterNotResponding" % e, "MASTER_PROXY", logging.ERROR)
raise MasterNotRespondingException()
except:
ciel.log.error("Error contacting master", "MSTRPRXY", logging.WARN, True)
self.stop_event.wait(initial_wait)
initial_wait += initial_wait * random.uniform(0.5, 1.5)
ciel.log.error("Given up trying to contact master", "MSTRPRXY", logging.ERROR, True)
if self.stop_event.is_set():
raise WorkerShutdownException()
else:
raise MasterNotRespondingException()
def execute_task_on_worker(self, worker, task):
try:
message = simplejson.dumps(task.as_descriptor(), cls=SWReferenceJSONEncoder)
if worker.communication_mechanism == Worker.C2DM_PUSH:
# magic push notification
self.c2dm_mgr.send_message(worker, ",".join([task.job.id, task.task_id]))
else:
post_string_noreturn("http://%s/control/task/" % (worker.netloc), message, result_callback=self.worker_post_result_callback)
except:
self.worker_failed(worker)
def backoff_request(self, url, method, payload=None, need_result=True, callback=None):
if self.stop_event.is_set():
return
try:
if method == "POST":
if need_result:
content = post_string(url, payload)
else:
if callback is None:
callback = self.master_post_result_callback
post_string_noreturn(url, payload, result_callback=callback)
return
elif method == "GET":
content = get_string(url)
else:
raise Exception("Invalid method %s" % method)
return 200, content
except:
ciel.log("Error attempting to contact master, aborting", "MSTRPRXY", logging.WARNING, True)
raise
def subscribe_output(otherend_netloc, chunk_size, id):
post = None
with module_lock:
try:
producer = get_producer_for_id(id)
if producer is not None:
try:
remote_stream_subscribers[(id, otherend_netloc)].set_chunk_size(chunk_size)
ciel.log("Remote %s changed chunk size for %s to %d" % (otherend_netloc, id, chunk_size), "BLOCKSTORE", logging.INFO)
except KeyError:
new_subscriber = RemoteOutputSubscriber(producer, otherend_netloc, chunk_size)
producer.subscribe(new_subscriber, try_direct=False)
ciel.log("Remote %s subscribed to output %s (chunk size %d)" % (otherend_netloc, id, chunk_size), "BLOCKSTORE", logging.INFO)
else:
try:
st = os.stat(filename(id))
post = simplejson.dumps({"bytes": st.st_size, "done": True})
except OSError:
post = simplejson.dumps({"absent": True})
except Exception as e:
ciel.log("Subscription to %s failed with exception %s; reporting absent" % (id, e), "BLOCKSTORE", logging.WARNING)
post = simplejson.dumps({"absent": True})
if post is not None:
post_string_noreturn("http://%s/control/streamstat/%s/advert" % (otherend_netloc, id), post)
def abort_task_on_worker(self, task, worker):
try:
ciel.log("Aborting task %s on worker %s" % (task.task_id, worker), "WORKER_POOL", logging.WARNING)
post_string_noreturn('http://%s/control/abort/%s/%s' % (worker.netloc, task.job.id, task.task_id), "", result_callback=self.worker_post_result_callback)
except:
self.worker_failed(worker)
def execute_task_on_worker(self, worker, task):
try:
message = simplejson.dumps(task.as_descriptor(), cls=SWReferenceJSONEncoder)
post_string_noreturn("http://%s/control/task/" % (worker.netloc), message, result_callback=self.worker_post_result_callback)
except:
self.worker_failed(worker)
def post(self, message):
post_string_noreturn("http://%s/control/streamstat/%s/advert" % (self.netloc, self.file_output.refid), message)
def unsubscribe_remote_output(refid):
unsubscribe_remote_output_nopost(refid)
netloc = get_own_netloc()
post_data = simplejson.dumps({"netloc": netloc})
post_string_noreturn("http://%s/control/streamstat/%s/unsubscribe"
% (netloc, refid), post_data)
def subscribe_remote_output(refid, remote_netloc, chunk_size, subscriber):
subscribe_remote_output_nopost(refid, subscriber)
post_data = simplejson.dumps({"netloc": get_own_netloc(), "chunk_size": chunk_size})
post_string_noreturn("http://%s/control/streamstat/%s/subscribe" % (remote_netloc, refid), post_data, result_callback=(lambda success, url: subscribe_result(refid, success, url)))
