def handle_recv(self, data):
topic, msg = data
topic_parts = s(topic).split(".")
watcher = topic_parts[1]
action = topic_parts[2]
with open(self.config['file'], 'a+') as f:
f.write('%s:%s' % (watcher, action))
def handle_recv(self, data):
"""called each time circusd sends an event"""
# maintains a periodic callback to compute mem and cpu consumption for
# each pid.
logger.debug('Received an event from circusd: %s' % str(data))
topic, msg = data
try:
topic = s(topic)
watcher = topic.split('.')[1:-1][0]
action = topic.split('.')[-1]
msg = json.loads(msg)
if action in ('reap', 'kill'):
# a process was reaped
pid = msg['process_pid']
self.remove_pid(watcher, pid)
elif action == 'spawn':
# a process was added
pid = msg['process_pid']
self._append_pid(watcher, pid)
elif action == 'stop':
# the whole watcher was stopped.
self.stop_watcher(watcher)
else:
logger.debug('Unknown action: %r' % action)
logger.debug(msg)
except Exception:
logger.exception('Failed to handle %r' % msg)
def write_data(self, data):
# data to write on file
file_data = s(data["data"])
# If we want to prefix the stream with the current datetime
if self._time_format is not None:
if "timestamp" in data:
time = self.fromtimestamp(data["timestamp"])
else:
time = self.now()
time = time.strftime(self._time_format)
prefix = "{time} [{pid}] | ".format(time=time, pid=data["pid"])
file_data = prefix + file_data.rstrip("\n")
file_data = file_data.replace("\n", "\n" + prefix)
file_data += "\n"
# writing into the file
try:
self._file.write(file_data)
except Exception:
# we can strip the string down on Py3 but not on Py2
if not PY2:
file_data = file_data.encode("latin-1", errors="replace")
file_data = file_data.decode("latin-1")
self._file.write(file_data)
else:
raise
self._file.flush()
def write_data(self, data):
# data to write on file
file_data = s(data['data'])
# If we want to prefix the stream with the current datetime
if self._time_format is not None:
time = self.now().strftime(self._time_format)
prefix = '{time} [{pid}] | '.format(time=time, pid=data['pid'])
file_data = prefix + file_data.rstrip('\n')
file_data = file_data.replace('\n', '\n' + prefix)
file_data += '\n'
# writing into the file
try:
self._file.write(file_data)
except Exception:
# we can strip the string down on Py3 but not on Py2
if not PY2:
file_data = file_data.encode('latin-1', errors='replace')
file_data = file_data.decode('latin-1')
self._file.write(file_data)
else:
raise
self._file.flush()
def write_data(self, data):
# data to write on file
file_data = s(data['data'])
# If we want to prefix the stream with the current datetime
if self._time_format is not None:
if 'timestamp' in data:
time = self.fromtimestamp(data['timestamp'])
else:
time = self.now()
time = time.strftime(self._time_format)
prefix = '{time} [{pid}] | '.format(time=time, pid=data['pid'])
file_data = prefix + file_data.rstrip('\n')
file_data = file_data.replace('\n', '\n' + prefix)
file_data += '\n'
# writing into the file
try:
self._file.write(file_data)
except Exception:
# we can strip the string down on Py3 but not on Py2
if not PY2:
file_data = file_data.encode('latin-1', errors='replace')
file_data = file_data.decode('latin-1')
self._file.write(file_data)
else:
raise
self._file.flush()
def iter_messages(self):
""" Yields tuples of (watcher, subtopic, stat)"""
recv = self.pubsub_socket.recv_multipart
with self:
while True:
try:
events = dict(self.poller.poll(self.timeout * 1000))
except zmq.ZMQError as e:
if e.errno == errno.EINTR:
continue
raise
if len(events) == 0:
continue
try:
topic, stat = recv()
except zmq.core.error.ZMQError as e:
if e.errno != errno.EINTR:
raise
else:
try:
sys.exc_clear()
except Exception:
pass
continue
topic = s(topic).split('.')
if len(topic) == 3:
__, watcher, subtopic = topic
yield watcher, subtopic, json.loads(stat)
elif len(topic) == 2:
__, watcher = topic
yield watcher, None, json.loads(stat)
def handle_recv(self, data):
topic, msg = data
topic_parts = s(topic).split(".")
watcher = topic_parts[1]
action = topic_parts[2]
with open(self.config['file'], 'a+') as f:
f.write('%s:%s' % (watcher, action))
def handle_recv(self, data):
"""called each time circusd sends an event"""
# maintains a periodic callback to compute mem and cpu consumption for
# each pid.
logger.debug('Received an event from circusd: %s' % str(data))
topic, msg = data
try:
topic = s(topic)
watcher = topic.split('.')[1:-1][0]
action = topic.split('.')[-1]
msg = json.loads(msg)
if action in ('reap', 'kill'):
# a process was reaped
pid = msg['process_pid']
self.remove_pid(watcher, pid)
elif action == 'spawn':
# a process was added
pid = msg['process_pid']
self._append_pid(watcher, pid)
elif action == 'stop':
# the whole watcher was stopped.
self.stop_watcher(watcher)
else:
logger.debug('Unknown action: %r' % action)
logger.debug(msg)
except Exception:
logger.exception('Failed to handle %r' % msg)
def iter_messages(self):
""" Yields tuples of (watcher, subtopic, stat)"""
recv = self.pubsub_socket.recv_multipart
with self:
while True:
try:
events = dict(self.poller.poll(self.timeout * 1000))
except zmq.ZMQError as e:
if e.errno == errno.EINTR:
continue
raise
if len(events) == 0:
continue
try:
topic, stat = recv()
except zmq.core.error.ZMQError as e:
if e.errno != errno.EINTR:
raise
else:
try:
sys.exc_clear()
except Exception:
pass
continue
topic = s(topic).split('.')
if len(topic) == 3:
__, watcher, subtopic = topic
yield watcher, subtopic, json.loads(stat)
elif len(topic) == 2:
__, watcher = topic
yield watcher, None, json.loads(stat)
def __call__(self, data):
for line in s(data['data']).split('\n'):
if line:
self.out.write(self.prefix(data))
self.out.write(line)
# stop coloring
self.out.write('\033[0m\n')
self.out.flush()
def __call__(self, data):
for line in s(data['data']).split('\n'):
if line:
self.out.write(self.prefix(data['pid']))
self.out.write(line)
# stop coloring
self.out.write('\033[0m\n')
self.out.flush()
def run_ctl(args, queue=None, stdin=''):
cmd = '%s -m circus.circusctl' % sys.executable
proc = subprocess.Popen(cmd.split() + shlex.split(args),
stdin=subprocess.PIPE if stdin else None,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stdout, stderr = proc.communicate(b(stdin) if stdin else None)
stdout = s(stdout)
stderr = s(stderr)
if queue:
queue.put(stderr)
queue.put(stdout)
try:
import gevent
if hasattr(gevent, 'shutdown'):
gevent.shutdown()
except ImportError:
pass
return stdout, stderr
def run_ctl(args, queue=None, stdin=''):
cmd = '%s -m circus.circusctl' % sys.executable
proc = subprocess.Popen(cmd.split() + shlex.split(args),
stdin=subprocess.PIPE if stdin else None,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stdout, stderr = proc.communicate(b(stdin) if stdin else None)
stdout = s(stdout)
stderr = s(stderr)
if queue:
queue.put(stderr)
queue.put(stdout)
try:
import gevent
if hasattr(gevent, 'shutdown'):
gevent.shutdown()
except ImportError:
pass
return stdout, stderr
def run_ctl(args, queue=None, stdin='', endpoint=DEFAULT_ENDPOINT_DEALER):
cmd = '%s -m circus.circusctl' % PYTHON
if '--endpoint' not in args:
args = '--endpoint %s ' % endpoint + args
proc = subprocess.Popen(cmd.split() + shlex.split(args),
stdin=subprocess.PIPE if stdin else None,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stdout, stderr = proc.communicate(b(stdin) if stdin else None)
stdout = s(stdout)
stderr = s(stderr)
if queue:
queue.put(stderr)
queue.put(stdout)
try:
import gevent
if hasattr(gevent, 'shutdown'):
gevent.shutdown()
except ImportError:
pass
return stdout, stderr
def run_ctl(args, queue=None, stdin='', endpoint=DEFAULT_ENDPOINT_DEALER):
cmd = '%s -m circus.circusctl' % PYTHON
if '--endpoint' not in args:
args = '--endpoint %s ' % endpoint + args
proc = subprocess.Popen(cmd.split() + shlex.split(args),
stdin=subprocess.PIPE if stdin else None,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stdout, stderr = proc.communicate(b(stdin) if stdin else None)
stdout = s(stdout)
stderr = s(stderr)
if queue:
queue.put(stderr)
queue.put(stdout)
try:
import gevent
if hasattr(gevent, 'shutdown'):
gevent.shutdown()
except ImportError:
pass
return stdout, stderr
def read(self, timeout=None):
timeout = timeout or self._timeout
if self._buffer:
raise tornado.gen.Return(self._buffer.pop(0))
start = time.time()
while time.time() - start < timeout:
try:
msg = self._stream.get_nowait()
lines = [l for l in s(msg["data"]).split("\n") if l]
self._buffer.extend(lines)
raise tornado.gen.Return(self._buffer.pop(0))
except Empty:
yield tornado_sleep(0.1)
raise TimeoutException("Timeout reading queue")
def read(self, timeout=None):
timeout = timeout or self._timeout
if self._buffer:
raise tornado.gen.Return(self._buffer.pop(0))
start = time.time()
while time.time() - start < timeout:
try:
msg = self._stream.get_nowait()
lines = [l for l in s(msg['data']).split('\n') if l]
self._buffer.extend(lines)
raise tornado.gen.Return(self._buffer.pop(0))
except Empty:
yield tornado_sleep(0.1)
raise TimeoutException('Timeout reading queue')
def read_from_stream(stream, desired_channel, timeout=5):
start = time.time()
accumulator = ''
while not channels[desired_channel] and time.time() - start < timeout:
try:
data = stream.get_nowait()
data = s(data['data']).split('\n')
accumulator += data.pop(0)
if data:
data.insert(0, accumulator)
accumulator = data.pop()
for line in data:
if len(line) > 1 and line[1] == ':':
channel, string = line.partition(':')[::2]
channels[int(channel)].append(string)
except Empty:
yield tornado_sleep(0.1)
if channels[desired_channel]:
raise tornado.gen.Return(channels[desired_channel].pop(0))
raise TimeoutException('Timeout reading queue')
def read_from_stream(stream, desired_channel, timeout=5):
start = time.time()
accumulator = ''
while not channels[desired_channel] and time.time() - start < timeout:
try:
data = stream.get_nowait()
data = s(data['data']).split('\n')
accumulator += data.pop(0)
if data:
data.insert(0, accumulator)
accumulator = data.pop()
for line in data:
if len(line) > 1 and line[1] == ':':
channel, string = line.partition(':')[::2]
channels[int(channel)].append(string)
except Empty:
yield tornado_sleep(0.1)
if channels[desired_channel]:
raise tornado.gen.Return(channels[desired_channel].pop(0))
raise TimeoutException('Timeout reading queue')
def __call__(self, data):
if self._should_rollover(data["data"]):
self._do_rollover()
# If we want to prefix the stream with the current datetime
for line in s(data["data"]).split("\n"):
if not line:
continue
if self.time_format is not None:
self._file.write(
"{time} [{pid}] | ".format(time=self.now().strftime(self.time_format), pid=data["pid"])
)
try:
self._file.write(line)
except Exception:
# we can strip the string down on Py3 but not on Py2
if not PY2:
self._file.write(line.encode("latin-1", errors="replace").decode("latin-1"))
self._file.write("\n")
self._file.flush()
def __call__(self, data):
if self._should_rollover(data['data']):
self._do_rollover()
# If we want to prefix the stream with the current datetime
for line in s(data['data']).split('\n'):
if not line:
continue
if self.time_format is not None:
self._file.write('{time} [{pid}] | '.format(
time=self.now().strftime(self.time_format),
pid=data['pid']))
try:
self._file.write(line)
except Exception:
# we can strip the string down on Py3 but not on Py2
if not PY2:
self._file.write(
line.encode('latin-1',
errors='replace').decode('latin-1'))
self._file.write('\n')
self._file.flush()
def test_copy_path(self):
watcher = SomeWatcher(stream=True)
yield watcher.run()
# wait for watcher data at most 5s
messages = []
resp = False
start_time = time.time()
while (time.time() - start_time) <= 5:
yield tornado_sleep(0.5)
# More than one Queue.get call is needed to get full
# output from a watcher in an environment with rich sys.path.
try:
m = watcher.stream.get(block=False)
messages.append(m)
except Queue.Empty:
pass
data = ''.join(s(m['data']) for m in messages)
if 'XYZ' in data:
resp = True
break
self.assertTrue(resp)
yield watcher.stop()
def __call__(self, data):
sys.stdout.write(s(data['data']))
sys.stdout.flush()
def split_data(data):
topic, msg = data
topic_parts = s(topic).split(".")
return topic_parts[1], topic_parts[2], msg
def __call__(self, data):
sys.stdout.write(s(data['data']))
sys.stdout.flush()
def split_data(data):
topic, msg = data
topic_parts = s(topic).split(".")
return topic_parts[1], topic_parts[2], msg
