def unsubscribe_remote_output(refid):
unsubscribe_remote_output_nopost(refid)
netloc = get_own_netloc()
post_data = simplejson.dumps({"netloc": netloc})
post_string_noreturn(
"http://%s/control/streamstat/%s/unsubscribe" % (netloc, refid),
post_data)
def _backoff_request(self, url, method, payload=None, num_attempts=1, initial_wait=0, need_result=True, callback=None):
initial_wait = 5
for _ in range(0, num_attempts):
if self.stop_event.is_set():
break
try:
try:
if method == "POST":
if need_result or num_attempts > 1:
content = post_string(url, payload)
else:
if callback is None:
callback = self.master_post_result_callback
post_string_noreturn(url, payload, result_callback=callback)
return
elif method == "GET":
content = get_string(url)
else:
raise Exception("Invalid method %s" % method)
return 200, content
except Exception as e:
ciel.log("Backoff-request failed with exception %s; re-raising MasterNotResponding" % e, "MASTER_PROXY", logging.ERROR)
raise MasterNotRespondingException()
except:
ciel.log.error("Error contacting master", "MSTRPRXY", logging.WARN, True)
self.stop_event.wait(initial_wait)
initial_wait += initial_wait * random.uniform(0.5, 1.5)
ciel.log.error("Given up trying to contact master", "MSTRPRXY", logging.ERROR, True)
if self.stop_event.is_set():
raise WorkerShutdownException()
else:
raise MasterNotRespondingException()
def backoff_request(self,
url,
method,
payload=None,
need_result=True,
callback=None):
if self.stop_event.is_set():
return
try:
if method == "POST":
if need_result:
content = post_string(url, payload)
else:
if callback is None:
callback = self.master_post_result_callback
post_string_noreturn(url,
payload,
result_callback=callback)
return
elif method == "GET":
content = get_string(url)
else:
raise Exception("Invalid method %s" % method)
return 200, content
except:
ciel.log("Error attempting to contact master, aborting",
"MSTRPRXY", logging.WARNING, True)
raise
def execute_task_on_worker(self, worker, task):
try:
ciel.stopwatch.stop("master_task")
message = simplejson.dumps(task.as_descriptor(), cls=SWReferenceJSONEncoder)
post_string_noreturn("http://%s/control/task/" % (worker.netloc), message, result_callback=self.worker_post_result_callback)
except:
self.worker_failed(worker)
def subscribe_remote_output(refid, remote_netloc, chunk_size, subscriber):
subscribe_remote_output_nopost(refid, subscriber)
post_data = simplejson.dumps({
"netloc": get_own_netloc(),
"chunk_size": chunk_size
})
post_string_noreturn(
"http://%s/control/streamstat/%s/subscribe" % (remote_netloc, refid),
post_data,
result_callback=(
lambda success, url: subscribe_result(refid, success, url)))
def abort_task_on_worker(self, task, worker):
try:
ciel.log("Aborting task %s on worker %s" % (task.task_id, worker),
"WORKER_POOL", logging.WARNING)
post_string_noreturn(
'http://%s/control/abort/%s/%s' %
(worker.netloc, task.job.id, task.task_id),
"",
result_callback=self.worker_post_result_callback)
except:
self.worker_failed(worker)
def abort_task_on_worker(self, task, worker):
"""
Abort the given task, which must be running on the given
worker. This only does the RPC, and doesn't attempt to update
the master state at all. Note that the arguments are
the opposite way around to execute_task_on_worker()!
Can aquire the lock if we hit an error in the RPC operation.
"""
try:
ciel.log("Aborting task %s on worker %s" % (task.task_id, worker), "WORKER_POOL", logging.WARNING)
post_string_noreturn('http://%s/control/abort/%s/%s' % (worker.netloc, task.job.id, task.task_id), "", result_callback=self._worker_post_result_callback)
except:
self.worker_failed(worker)
def _backoff_request(self,
url,
method,
payload=None,
num_attempts=1,
initial_wait=0,
need_result=True,
callback=None):
initial_wait = 5
for _ in range(0, num_attempts):
if self.stop_event.is_set():
break
try:
try:
if method == "POST":
if need_result or num_attempts > 1:
content = post_string(url, payload)
else:
if callback is None:
callback = self.master_post_result_callback
post_string_noreturn(url,
payload,
result_callback=callback)
return
elif method == "GET":
content = get_string(url)
else:
raise Exception("Invalid method %s" % method)
return 200, content
except Exception as e:
ciel.log(
"Backoff-request failed with exception %s; re-raising MasterNotResponding"
% e, "MASTER_PROXY", logging.ERROR)
raise MasterNotRespondingException()
except:
ciel.log.error("Error contacting master", "MSTRPRXY",
logging.WARN, True)
self.stop_event.wait(initial_wait)
initial_wait += initial_wait * random.uniform(0.5, 1.5)
ciel.log.error("Given up trying to contact master", "MSTRPRXY",
logging.ERROR, True)
if self.stop_event.is_set():
raise WorkerShutdownException()
else:
raise MasterNotRespondingException()
def execute_task_on_worker(self, worker, task):
"""
Actually run the given task on the given worker. This only
does the RPC, and doesn't attempt to update the master state
at all.
Can aquire the lock if we hit an error in the RPC operation.
"""
if worker.failed or self.workers.get(worker.id) != worker or self.netlocs.get(worker.netloc) != worker:
ciel.log.error("Executing task %s on worker %s which doesn't seem to be properly registered?" % (str(task), str(worker)),
"WORKER_POOL",
logging.WARNING)
try:
ciel.stopwatch.stop("master_task")
message = simplejson.dumps(task.as_descriptor(), cls=SWReferenceJSONEncoder)
post_string_noreturn("http://%s/control/task/" % (worker.netloc), message, result_callback=self._worker_post_result_callback)
except:
self.worker_failed(worker)
def backoff_request(self, url, method, payload=None, need_result=True, callback=None):
if self.stop_event.is_set():
return
try:
if method == "POST":
if need_result:
content = post_string(url, payload)
else:
if callback is None:
callback = self.master_post_result_callback
post_string_noreturn(url, payload, result_callback=callback)
return
elif method == "GET":
content = get_string(url)
else:
raise Exception("Invalid method %s" % method)
return 200, content
except:
ciel.log("Error attempting to contact master, aborting", "MSTRPRXY", logging.WARNING, True)
raise
def subscribe_output(otherend_netloc, chunk_size, id):
post = None
with module_lock:
try:
producer = get_producer_for_id(id)
if producer is not None:
try:
remote_stream_subscribers[(
id, otherend_netloc)].set_chunk_size(chunk_size)
ciel.log(
"Remote %s changed chunk size for %s to %d" %
(otherend_netloc, id, chunk_size), "BLOCKSTORE",
logging.DEBUG)
except KeyError:
new_subscriber = RemoteOutputSubscriber(
producer, otherend_netloc, chunk_size)
producer.subscribe(new_subscriber, try_direct=False)
ciel.log(
"Remote %s subscribed to output %s (chunk size %d)" %
(otherend_netloc, id, chunk_size), "BLOCKSTORE",
logging.DEBUG)
else:
try:
st = os.stat(filename(id))
post = simplejson.dumps({
"bytes": st.st_size,
"done": True
})
except OSError:
post = simplejson.dumps({"absent": True})
except Exception as e:
ciel.log(
"Subscription to %s failed with exception %s; reporting absent"
% (id, e), "BLOCKSTORE", logging.WARNING)
post = simplejson.dumps({"absent": True})
if post is not None:
post_string_noreturn(
"http://%s/control/streamstat/%s/advert" % (otherend_netloc, id),
post)
def subscribe_output(otherend_netloc, chunk_size, id):
post = None
with module_lock:
try:
producer = get_producer_for_id(id)
if producer is not None:
try:
remote_stream_subscribers[(id, otherend_netloc)].set_chunk_size(chunk_size)
ciel.log("Remote %s changed chunk size for %s to %d" % (otherend_netloc, id, chunk_size), "BLOCKSTORE", logging.DEBUG)
except KeyError:
new_subscriber = RemoteOutputSubscriber(producer, otherend_netloc, chunk_size)
producer.subscribe(new_subscriber, try_direct=False)
ciel.log("Remote %s subscribed to output %s (chunk size %d)" % (otherend_netloc, id, chunk_size), "BLOCKSTORE", logging.DEBUG)
else:
try:
st = os.stat(filename(id))
post = simplejson.dumps({"bytes": st.st_size, "done": True})
except OSError:
post = simplejson.dumps({"absent": True})
except Exception as e:
ciel.log("Subscription to %s failed with exception %s; reporting absent" % (id, e), "BLOCKSTORE", logging.WARNING)
post = simplejson.dumps({"absent": True})
if post is not None:
post_string_noreturn("http://%s/control/streamstat/%s/advert" % (otherend_netloc, id), post)
def abort_task_on_worker(self, task, worker):
try:
ciel.log("Aborting task %s on worker %s" % (task.task_id, worker), "WORKER_POOL", logging.WARNING)
post_string_noreturn('http://%s/control/abort/%s/%s' % (worker.netloc, task.job.id, task.task_id), "", result_callback=self.worker_post_result_callback)
except:
self.worker_failed(worker)
def post(self, message):
post_string_noreturn("http://%s/control/streamstat/%s/advert" % (self.netloc, self.file_output.refid), message)
def post(self, message):
post_string_noreturn(
"http://%s/control/streamstat/%s/advert" %
(self.netloc, self.file_output.refid), message)
def unsubscribe_remote_output(refid):
unsubscribe_remote_output_nopost(refid)
netloc = get_own_netloc()
post_data = simplejson.dumps({"netloc": netloc})
post_string_noreturn("http://%s/control/streamstat/%s/unsubscribe"
% (netloc, refid), post_data)
def subscribe_remote_output(refid, remote_netloc, chunk_size, subscriber):
subscribe_remote_output_nopost(refid, subscriber)
post_data = simplejson.dumps({"netloc": get_own_netloc(), "chunk_size": chunk_size})
post_string_noreturn("http://%s/control/streamstat/%s/subscribe" % (remote_netloc, refid), post_data, result_callback=(lambda success, url: subscribe_result(refid, success, url)))
