def __init__(self, loop, pipe, protocol, waiter=None, extra=None):
super().__init__(extra)
self._extra['pipe'] = pipe
self._loop = loop
self._pipe = pipe
self._fileno = pipe.fileno()
self._protocol = protocol
self._closing = False
mode = os.fstat(self._fileno).st_mode
if not (stat.S_ISFIFO(mode) or
stat.S_ISSOCK(mode) or
stat.S_ISCHR(mode)):
self._pipe = None
self._fileno = None
self._protocol = None
raise ValueError("Pipe transport is for pipes/sockets only.")
os.set_blocking(self._fileno, False)
self._loop.call_soon(self._protocol.connection_made, self)
# only start reading when connection_made() has been called
self._loop.call_soon(self._loop._add_reader,
self._fileno, self._read_ready)
if waiter is not None:
# only wake up the waiter when connection_made() has been called
self._loop.call_soon(futures._set_result_unless_cancelled,
waiter, None)
def __enter__(self):
# prepare standard file descriptors for raw manipulation
self.was_blocking = os.get_blocking(0)
os.set_blocking(0, False)
try:
self.terminal_attr_stdin = termios.tcgetattr(0)
self.terminal_attr_stdout = termios.tcgetattr(1)
self.terminal_attr_stderr = termios.tcgetattr(2)
tty.setraw(0)
tty.setraw(1)
tty.setraw(2)
except termios.error: # probably redirected
self.terminal_attr_stdin = None
# redirect standard file descriptors to new PTY
master, slave = pty.openpty()
os.set_blocking(master, False)
self.real_stdin = os.dup(0)
self.real_stdout = os.dup(1)
self.real_stderr = os.dup(2)
os.close(0)
os.close(1)
os.close(2)
os.dup2(slave, 0)
os.dup2(slave, 1)
os.dup2(slave, 2)
os.close(slave)
self.terminal_pipe = master
# start REPL in separate thread
threading.Thread(target=repl, args=(self,), daemon=True).start()
return self
def __init__(self, fileobj):
assert not isinstance(fileobj, io.TextIOBase), 'Only binary mode files allowed'
self._file = fileobj
self._fileno = fileobj.fileno()
os.set_blocking(self._fileno, False)
self._linebuffer = bytearray()
self._timeout = None
def __init__(self, fileobj):
assert not isinstance(fileobj, io.TextIOBase), 'Only binary mode files allowed'
super().__init__(fileobj)
os.set_blocking(self._fileno, False)
# Common bound methods
self._file_read = fileobj.read
self._file_write = fileobj.write
def set_blocking(fd, blocking):
if hasattr(os, 'set_blocking'):
os.set_blocking(fd, blocking)
else:
flags = fcntl.fcntl(fd, fcntl.F_GETFL)
flags = (flags & ~os.O_NONBLOCK if blocking else
flags | os.O_NONBLOCK)
fcntl.fcntl(fd, fcntl.F_SETFL, flags)
def set_nonblocking(fd): # pragma: no cover
if PY3 and sys.version_info[1] >= 5:
os.set_blocking(fd, False)
elif fcntl is None:
raise RuntimeError('no fcntl module present')
else:
flags = fcntl.fcntl(fd, fcntl.F_GETFL, 0)
flags = flags | os.O_NONBLOCK
fcntl.fcntl(fd, fcntl.F_SETFL, flags)
def blocking(self):
'''
Allow temporary access to the underlying socket in blocking mode
'''
try:
os.set_blocking(self._fileno, True)
yield self._file
finally:
os.set_blocking(self._fileno, False)
def __init__(self, fileobj):
assert not isinstance(fileobj, io.TextIOBase), 'Only binary mode files allowed'
super().__init__(fileobj)
os.set_blocking(int(self._fileno), False)
# Common bound methods
self._file_read = fileobj.read
self._readinto_impl = getattr(fileobj, 'readinto', None)
self._file_write = fileobj.write
def __init__(self, fd, map=None):
dispatcher.__init__(self, None, map)
self.connected = True
try:
fd = fd.fileno()
except AttributeError:
pass
self.set_file(fd)
# set it to non-blocking mode
os.set_blocking(fd, False)
def blocking(self):
'''
Allow temporary access to the underlying file in blocking mode
'''
if self._buffer:
raise IOError('There is unread buffered data.')
try:
os.set_blocking(int(self._fileno), True)
yield self._file
finally:
os.set_blocking(int(self._fileno), False)
def run_pty(script, input=b"dummy input\r", env=None):
pty = import_module('pty')
output = bytearray()
[master, slave] = pty.openpty()
args = (sys.executable, '-c', script)
proc = subprocess.Popen(args, stdin=slave, stdout=slave, stderr=slave, env=env)
os.close(slave)
with ExitStack() as cleanup:
cleanup.enter_context(proc)
def terminate(proc):
try:
proc.terminate()
except ProcessLookupError:
# Workaround for Open/Net BSD bug (Issue 16762)
pass
cleanup.callback(terminate, proc)
cleanup.callback(os.close, master)
# Avoid using DefaultSelector and PollSelector. Kqueue() does not
# work with pseudo-terminals on OS X < 10.9 (Issue 20365) and Open
# BSD (Issue 20667). Poll() does not work with OS X 10.6 or 10.4
# either (Issue 20472). Hopefully the file descriptor is low enough
# to use with select().
sel = cleanup.enter_context(selectors.SelectSelector())
sel.register(master, selectors.EVENT_READ | selectors.EVENT_WRITE)
os.set_blocking(master, False)
while True:
for [_, events] in sel.select():
if events & selectors.EVENT_READ:
try:
chunk = os.read(master, 0x10000)
except OSError as err:
# Linux raises EIO when slave is closed (Issue 5380)
if err.errno != EIO:
raise
chunk = b""
if not chunk:
return output
output.extend(chunk)
if events & selectors.EVENT_WRITE:
try:
input = input[os.write(master, input):]
except OSError as err:
# Apparently EIO means the slave was closed
if err.errno != EIO:
raise
input = b"" # Stop writing
if not input:
sel.modify(master, selectors.EVENT_READ)
def test_set_wakeup_fd_blocking(self):
rfd, wfd = os.pipe()
self.addCleanup(os.close, rfd)
self.addCleanup(os.close, wfd)
# fd must be non-blocking
os.set_blocking(wfd, True)
with self.assertRaises(ValueError) as cm:
signal.set_wakeup_fd(wfd)
self.assertEqual(str(cm.exception),
"the fd %s must be in non-blocking mode" % wfd)
# non-blocking is ok
os.set_blocking(wfd, False)
signal.set_wakeup_fd(wfd)
signal.set_wakeup_fd(-1)
def test_set_wakeup_fd_result(self):
r1, w1 = os.pipe()
self.addCleanup(os.close, r1)
self.addCleanup(os.close, w1)
r2, w2 = os.pipe()
self.addCleanup(os.close, r2)
self.addCleanup(os.close, w2)
if hasattr(os, 'set_blocking'):
os.set_blocking(w1, False)
os.set_blocking(w2, False)
signal.set_wakeup_fd(w1)
self.assertEqual(signal.set_wakeup_fd(w2), w1)
self.assertEqual(signal.set_wakeup_fd(-1), w2)
self.assertEqual(signal.set_wakeup_fd(-1), -1)
def test_basic(self):
try:
debug("Calling master_open()")
master_fd, slave_name = pty.master_open()
debug("Got master_fd '%d', slave_name '%s'" %
(master_fd, slave_name))
debug("Calling slave_open(%r)" % (slave_name,))
slave_fd = pty.slave_open(slave_name)
debug("Got slave_fd '%d'" % slave_fd)
except OSError:
# " An optional feature could not be imported " ... ?
raise unittest.SkipTest("Pseudo-terminals (seemingly) not functional.")
self.assertTrue(os.isatty(slave_fd), 'slave_fd is not a tty')
# Solaris requires reading the fd before anything is returned.
# My guess is that since we open and close the slave fd
# in master_open(), we need to read the EOF.
# Ensure the fd is non-blocking in case there's nothing to read.
blocking = os.get_blocking(master_fd)
try:
os.set_blocking(master_fd, False)
try:
s1 = os.read(master_fd, 1024)
self.assertEqual(b'', s1)
except OSError as e:
if e.errno != errno.EAGAIN:
raise
finally:
# Restore the original flags.
os.set_blocking(master_fd, blocking)
debug("Writing to slave_fd")
os.write(slave_fd, TEST_STRING_1)
s1 = _readline(master_fd)
self.assertEqual(b'I wish to buy a fish license.\n',
normalize_output(s1))
debug("Writing chunked output")
os.write(slave_fd, TEST_STRING_2[:5])
os.write(slave_fd, TEST_STRING_2[5:])
s2 = _readline(master_fd)
self.assertEqual(b'For my pet fish, Eric.\n', normalize_output(s2))
os.close(slave_fd)
os.close(master_fd)
def __init__(self, tool, prompt, exit_cmd, args=[], encoding='UTF-8'):
"""Create an interactive PTY session
tool - full name of the executable to run
prompt - token to treat as the interpreter's prompt
exit_cmd - command to tell the interpreter to exit cleanly
args - extra arguments (list) to pass to launcher (def: [])
encoding - downstream interpreter's encoding (def: UTF-8)
"""
if not os.path.exists(tool):
raise ValueError("'{}' is not found".format(tool))
if prompt == '':
raise ValueError("prompt must be a valid string")
self._tool = tool
self._tool_args = args
self._prompt = prompt
self._encoding = encoding
self._exit_cmd = exit_cmd
# where to accumulate incoming data
self._raw_buffer = b''
# cook up a PTY
self.master, self.slave = pty.openpty()
# hook it up to a subprocess
args = [self._tool] + self._tool_args
self.proc = subprocess.Popen(args,
stdin=self.slave,
stdout=self.slave,
stderr=self.slave)
self.proc_rc = None
# register a selector manager
self.sel = selectors.SelectSelector()
self.sel.register(self.master,
selectors.EVENT_READ | selectors.EVENT_WRITE)
# try to avoid os interference
if sys.version_info[0] >= 3 and sys.version_info[1] >= 5:
os.set_blocking(self.master, False)
else:
import fcntl
flag = fcntl.fcntl(self.master, fcntl.F_GETFL)
fcntl.fcntl(self.master, fcntl.F_SETFL, flag | os.O_NONBLOCK)
def test_write_pty(self):
master, slave = os.openpty()
os.set_blocking(master, False)
slave_write_obj = io.open(slave, 'wb', 0)
proto = MyWritePipeProto(loop=self.loop)
connect = self.loop.connect_write_pipe(lambda: proto, slave_write_obj)
transport, p = self.loop.run_until_complete(connect)
self.assertIs(p, proto)
self.assertIs(transport, proto.transport)
self.assertEqual('CONNECTED', proto.state)
transport.write(b'1')
data = bytearray()
def reader(data):
try:
chunk = os.read(master, 1024)
except BlockingIOError:
return len(data)
data += chunk
return len(data)
tb.run_until(self.loop, lambda: reader(data) >= 1,
timeout=10)
self.assertEqual(b'1', data)
transport.write(b'2345')
tb.run_until(self.loop, lambda: reader(data) >= 5,
timeout=10)
self.assertEqual(b'12345', data)
self.assertEqual('CONNECTED', proto.state)
os.close(master)
# extra info is available
self.assertIsNotNone(proto.transport.get_extra_info('pipe'))
# close connection
proto.transport.close()
self.loop.run_until_complete(proto.done)
self.assertEqual('CLOSED', proto.state)
def __exit__(self, exc_type, exc_value, traceback):
self.flush_pipes()
# should kill thread because of readline, but cannot do this in Python
# restore redirections
os.close(self.terminal_pipe)
# we should close file descriptors 0, 1 and 2 at this point, but the
# REPL thread might still be using them...
os.dup2(self.real_stderr, 2)
os.dup2(self.real_stdout, 1)
os.dup2(self.real_stdin, 0)
self.terminal_pipe = None
# restore standard file descriptors attributes
if self.terminal_attr_stdin is not None:
termios.tcsetattr(2, termios.TCSADRAIN, self.terminal_attr_stderr)
termios.tcsetattr(1, termios.TCSADRAIN, self.terminal_attr_stdout)
termios.tcsetattr(0, termios.TCSADRAIN, self.terminal_attr_stdin)
os.set_blocking(0, self.was_blocking)
def test_write_pipe(self):
rpipe, wpipe = os.pipe()
os.set_blocking(rpipe, False)
pipeobj = io.open(wpipe, "wb", 1024)
proto = MyWritePipeProto(loop=self.loop)
connect = self.loop.connect_write_pipe(lambda: proto, pipeobj)
transport, p = self.loop.run_until_complete(connect)
self.assertIs(p, proto)
self.assertIs(transport, proto.transport)
self.assertEqual("CONNECTED", proto.state)
transport.write(b"1")
data = bytearray()
def reader(data):
try:
chunk = os.read(rpipe, 1024)
except BlockingIOError:
return len(data)
data += chunk
return len(data)
test_utils.run_until(self.loop, lambda: reader(data) >= 1)
self.assertEqual(b"1", data)
transport.write(b"2345")
test_utils.run_until(self.loop, lambda: reader(data) >= 5)
self.assertEqual(b"12345", data)
self.assertEqual("CONNECTED", proto.state)
os.close(rpipe)
# extra info is available
self.assertIsNotNone(proto.transport.get_extra_info("pipe"))
# close connection
proto.transport.close()
self.loop.run_until_complete(proto.done)
self.assertEqual("CLOSED", proto.state)
def __init__(self, args, **kwargs):
if "universal_newlines" in kwargs:
raise RuntimeError("universal_newlines argument not supported")
# If stdin has been given and it's set to a curio FileStream object,
# then we need to flip it to blocking.
if "stdin" in kwargs:
stdin = kwargs["stdin"]
if isinstance(stdin, FileStream):
# At hell's heart I stab thy coroutine attempting to read from a stream
# that's been used as a pipe input to a subprocess. Must set back to
# blocking or all hell breaks loose in the child.
os.set_blocking(stdin.fileno(), True)
self._popen = subprocess.Popen(args, **kwargs)
if self._popen.stdin:
self.stdin = FileStream(self._popen.stdin)
if self._popen.stdout:
self.stdout = FileStream(self._popen.stdout)
if self._popen.stderr:
self.stderr = FileStream(self._popen.stderr)
def check_reopen(r1, w):
try:
print("Reopening read end")
r2 = os.open("/proc/self/fd/{}".format(r1), os.O_RDONLY)
print("r1 is {}, r2 is {}".format(r1, r2))
print("checking they both can receive from w...")
os.write(w, b"a")
assert os.read(r1, 1) == b"a"
os.write(w, b"b")
assert os.read(r2, 1) == b"b"
print("...ok")
print("setting r2 to non-blocking")
os.set_blocking(r2, False)
print("os.get_blocking(r1) ==", os.get_blocking(r1))
print("os.get_blocking(r2) ==", os.get_blocking(r2))
# Check r2 is really truly non-blocking
try:
os.read(r2, 1)
except BlockingIOError:
print("r2 definitely seems to be in non-blocking mode")
# Check that r1 is really truly still in blocking mode
def sleep_then_write():
time.sleep(1)
os.write(w, b"c")
threading.Thread(target=sleep_then_write, daemon=True).start()
assert os.read(r1, 1) == b"c"
print("r1 definitely seems to be in blocking mode")
except Exception as exc:
print("ERROR: {!r}".format(exc))
def __init__(self, loop, pipe, protocol, waiter=None, extra=None):
super().__init__(extra, loop)
self._extra['pipe'] = pipe
self._pipe = pipe
self._fileno = pipe.fileno()
self._protocol = protocol
self._buffer = bytearray()
self._conn_lost = 0
self._closing = False # Set when close() or write_eof() called.
mode = os.fstat(self._fileno).st_mode
is_char = stat.S_ISCHR(mode)
is_fifo = stat.S_ISFIFO(mode)
is_socket = stat.S_ISSOCK(mode)
if not (is_char or is_fifo or is_socket):
self._pipe = None
self._fileno = None
self._protocol = None
raise ValueError("Pipe transport is only for "
"pipes, sockets and character devices")
os.set_blocking(self._fileno, False)
self._loop.call_soon(self._protocol.connection_made, self)
# On AIX, the reader trick (to be notified when the read end of the
# socket is closed) only works for sockets. On other platforms it
# works for pipes and sockets. (Exception: OS X 10.4? Issue #19294.)
if is_socket or (is_fifo and not sys.platform.startswith("aix")):
# only start reading when connection_made() has been called
self._loop.call_soon(self._loop._add_reader,
self._fileno, self._read_ready)
if waiter is not None:
# only wake up the waiter when connection_made() has been called
self._loop.call_soon(futures._set_result_unless_cancelled,
waiter, None)
def client(args: argparse.Namespace) -> None:
tokens = args.address.rsplit(":", 1)
host = tokens[0] if len(tokens) > 0 else ""
port = tokens[1] if len(tokens) > 1 else ""
if not args.command:
args.command = ["sh", "-i"]
with socket.create_connection((host or None, port or DEFAULT_PORT)) as sock:
peername = sock.getpeername()
if args.verbose:
print(f"Connected to {peername} from {sock.getsockname()}", file=sys.stderr)
sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)
sock.setblocking(False)
pid, pty_fd = pty.fork()
if pid == 0:
os.execvp(args.command[0], args.command)
try:
os.set_blocking(pty_fd, False)
with Multiplexer() as multiplexer:
pty_file = multiplexer.open(pty_fd)
rrsh = Rrsh()
# Data read from the socket is written to the pseudoterminal.
rrsh.on_data = pty_file.write
rrsh.on_winch = lambda columns, lines: fcntl.ioctl(
pty_fd,
termios.TIOCSWINSZ,
struct.pack("HHHH", lines, columns, 0, 0),
)
def read_sock(buf: bytes) -> None:
if buf:
rrsh.feed(buf)
else:
if args.verbose:
log(f"{peername} disconnected")
multiplexer.done = True
sock_file = multiplexer.open(sock)
sock_file.read_cb = read_sock
# Data read from the pseudoterminal is written to the socket.
pty_file.read_cb = partial(Rrsh.write_data, sock_file)
exit_status = 1
def send_exit(signalnum: int) -> None:
while True:
try:
wpid, wstatus = os.waitpid(-1, os.WNOHANG)
except ChildProcessError:
break
if not wpid:
break
elif wpid == pid:
nonlocal exit_status
exit_status = decode_wstatus(wstatus, args.verbose)
sock_file.write(
struct.pack("!BBH", 0, Rrsh.Event.EXIT, wstatus)
)
multiplexer.done = True
multiplexer.signal(signal.SIGCHLD, send_exit)
multiplexer.run()
finally:
os.close(pty_fd)
if args.verbose:
log(f"Disconnected from {peername}")
sys.exit(exit_status)
def run(self):
self.started_event()
mutex.lock()
log_entry = EventLogModel(category='borg-run',
subcommand=self.cmd[1],
profile=self.params.get(
'profile_name', None))
log_entry.save()
logger.info('Running command %s', ' '.join(self.cmd))
p = Popen(self.cmd,
stdout=PIPE,
stderr=PIPE,
bufsize=1,
universal_newlines=True,
env=self.env,
cwd=self.cwd,
start_new_session=True)
self.process = p
# Prevent blocking of stdout/err. Via https://stackoverflow.com/a/7730201/3983708
os.set_blocking(p.stdout.fileno(), False)
os.set_blocking(p.stderr.fileno(), False)
def read_async(fd):
try:
return fd.read()
except (IOError, TypeError):
return ''
stdout = []
while True:
# Wait for new output
select.select([p.stdout, p.stderr], [], [], 0.1)
stdout.append(read_async(p.stdout))
stderr = read_async(p.stderr)
if stderr:
for line in stderr.split('\n'):
try:
parsed = json.loads(line)
if parsed['type'] == 'log_message':
self.app.backup_log_event.emit(
f'{parsed["levelname"]}: {parsed["message"]}')
level_int = getattr(logging, parsed["levelname"])
logger.log(level_int, parsed["message"])
elif parsed['type'] == 'file_status':
self.app.backup_log_event.emit(
f'{parsed["path"]} ({parsed["status"]})')
elif parsed['type'] == 'archive_progress':
msg = (
f"Files: {parsed['nfiles']}, "
f"Original: {pretty_bytes(parsed['original_size'])}, "
f"Deduplicated: {pretty_bytes(parsed['deduplicated_size'])}, "
f"Compressed: {pretty_bytes(parsed['compressed_size'])}"
)
self.app.backup_progress_event.emit(msg)
except json.decoder.JSONDecodeError:
msg = line.strip()
if msg: # Log only if there is something to log.
self.app.backup_log_event.emit(msg)
logger.warning(msg)
if p.poll() is not None:
time.sleep(0.1)
stdout.append(read_async(p.stdout))
break
result = {
'params': self.params,
'returncode': self.process.returncode,
'cmd': self.cmd,
}
stdout = ''.join(stdout)
try:
result['data'] = json.loads(stdout)
except ValueError:
result['data'] = stdout
log_entry.returncode = p.returncode
log_entry.repo_url = self.params.get('repo_url', None)
log_entry.save()
self.process_result(result)
self.finished_event(result)
mutex.unlock()
assert not os.get_inheritable(rfd)
assert not os.get_inheritable(wfd)
os.set_inheritable(rfd, True)
os.set_inheritable(wfd, True)
assert os.get_inheritable(rfd)
assert os.get_inheritable(wfd)
os.set_inheritable(rfd, True)
os.set_inheritable(wfd, True)
os.set_inheritable(rfd, True)
os.set_inheritable(wfd, True)
assert os.get_inheritable(rfd)
assert os.get_inheritable(wfd)
assert os.get_blocking(rfd)
assert os.get_blocking(wfd)
os.set_blocking(rfd, False)
os.set_blocking(wfd, False)
assert not os.get_blocking(rfd)
assert not os.get_blocking(wfd)
os.set_blocking(rfd, True)
os.set_blocking(wfd, True)
os.set_blocking(rfd, True)
os.set_blocking(wfd, True)
assert os.get_blocking(rfd)
assert os.get_blocking(wfd)
finally:
os.close(rfd)
os.close(wfd)
# os.pipe2
if sys.platform.startswith('linux') or sys.platform.startswith('freebsd'):
def invoke_run(manifest):
def get_qualified_output_file(ext):
if ext not in manifest.output_extensions:
raise RuntimeError(
f"Unexpected output extension for experiment '{manifest.experiment}': .{ext}\n"
)
return manifest.output_file_path(ext)
(stdout_pipe, stdout) = (None, None)
if manifest.stdout is not None:
stdout_path = get_qualified_output_file(manifest.stdout)
# We do not actually need to write anything to the output file.
# However, we might want to pipe experimental output to it.
with open(stdout_path, 'w') as f:
stdout = os.dup(f.fileno())
else:
stdout_path = manifest.aux_file_path('stdout')
(stdout_pipe, stdout) = os.pipe()
os.set_blocking(stdout_pipe, False)
# Create all the other the output files.
# The creation of the output file with extension '.out' signals that the run has been started.
diff_set = {manifest.stdout} if manifest.stdout is not None else set()
for ext in manifest.output_extensions - diff_set:
util.touch(manifest.output_file_path(ext))
# Create the error file.
(stderr_pipe, stderr) = os.pipe()
os.set_blocking(stderr_pipe, False)
def get_qualified_filename(identifier):
if identifier.isdigit():
identifier = int(identifier)
if manifest.instance_files is None:
raise RuntimeError(
f"Instance '{manifest.instance}' does not have any files specified in the experiments.yml"
) from None
if len(manifest.instance_files) <= identifier:
raise IndexError('File index out of range: {}'.format(identifier))
return ''.join([manifest.instance_dir, '/', manifest.instance_files[identifier]])
else:
if manifest.instance_extensions is None:
raise RuntimeError(
f"Instance '{manifest.instance}' does not have any extensions specified in the experiments.yml"
) from None
if identifier not in manifest.instance_extensions:
raise RuntimeError(
f"Unexpected file extension for instance '{manifest.instance}': .{identifier}"
) from None
return ''.join([manifest.instance_dir, '/', manifest.instance_yml_name, '.', identifier])
def substitute(p):
if p == 'INSTANCE':
return manifest.instance_dir + '/' + manifest.instance_yml_name
elif p.startswith('INSTANCE:'):
return get_qualified_filename(p.split(':')[1])
elif p == 'REPETITION':
return str(manifest.repetition)
elif p == 'OUTPUT':
return stdout_path
elif p.startswith('OUTPUT:'):
return get_qualified_output_file(p.split(':')[1])
elif p.startswith('SOURCE_DIR_FOR:'):
return manifest.get_source_dir_for(p.split(':')[1])
elif p.startswith('COMPILE_DIR_FOR:'):
return manifest.get_compile_dir_for(p.split(':')[1])
elif p.startswith('PREFIX_DIR_FOR:'):
return manifest.get_prefix_dir_for(p.split(':')[1])
elif p == 'OUTPUT_SUBDIR':
return manifest.output_subdir
else:
return None
def substitute_list(p):
if p == 'EXTRA_ARGS':
return manifest.get_extra_args()
else:
return None
cmd = util.expand_at_params(manifest.args, substitute, listfn=substitute_list)
# Build the environment.
def prepend_env(var, items):
if var in os.environ:
return ':'.join(items) + ':' + os.environ[var]
return ':'.join(items)
def substitute_extra_paths(p):
if p in ['THIS_CLONE_DIR', 'THIS_SOURCE_DIR']:
return build_yml['source']
elif p == 'THIS_COMPILE_DIR':
return build_yml['compile']
elif p == 'THIS_PREFIX_DIR':
return build_yml['prefix']
elif p.startswith('SOURCE_DIR_FOR:'):
return manifest.get_source_dir_for(p.split(':')[1])
elif p.startswith('COMPILE_DIR_FOR:'):
return manifest.get_compile_dir_for(p.split(':')[1])
elif p.startswith('PREFIX_DIR_FOR:'):
return manifest.get_prefix_dir_for(p.split(':')[1])
else:
return None
environ = os.environ.copy()
extra_paths = manifest.get_paths()
for build_yml in manifest.builds.values():
extra_paths.extend(util.expand_at_params(build_yml['extra_paths'], substitute_extra_paths))
environ['PATH'] = prepend_env('PATH', extra_paths)
environ['LD_LIBRARY_PATH'] = prepend_env('LD_LIBRARY_PATH', manifest.get_ldso_paths())
environ['PYTHONPATH'] = prepend_env('PYTHONPATH', manifest.get_python_paths())
environ.update(manifest.environ)
# Dumps data from an FD to the FS.
# Creates the output file only if something is written.
class LazyWriter:
def __init__(self, fd, path):
self._fd = fd
self._path = path
self._out = None
def progress(self):
# Specify some chunk size to avoid reading the whole pipe at once.
chunk = os.read(self._fd, 16 * 1024)
if not len(chunk):
return False
if self._out is None:
self._out = open(self._path, "wb")
self._out.write(chunk)
self._out.flush()
return True
def close(self):
if self._out is not None:
self._out.close()
start = time.perf_counter()
cwd = (util.expand_at_params(manifest.workdir, substitute)
if manifest.workdir is not None else manifest.base_dir)
child = subprocess.Popen(cmd, cwd=cwd, env=environ,
stdout=stdout, stderr=stderr)
os.close(stderr)
sel = selectors.DefaultSelector()
if manifest.stdout is None:
os.close(stdout)
stdout_writer = LazyWriter(stdout_pipe, manifest.aux_file_path('stdout'))
sel.register(stdout_pipe, selectors.EVENT_READ, stdout_writer)
stderr_writer = LazyWriter(stderr_pipe, manifest.aux_file_path('stderr'))
sel.register(stderr_pipe, selectors.EVENT_READ, stderr_writer)
# Wait until the run program finishes.
while True:
if child.poll() is not None:
break
elapsed = time.perf_counter() - start
if manifest.timeout is not None and elapsed > manifest.timeout:
# Programs can catch the SIGXCPU signal and keep running.
# Thus, the kill command ensures that the time limit is respected (with a grace period).
if elapsed > manifest.timeout + base.TIMEOUT_GRACE_PERIOD:
child.kill()
else:
child.send_signal(signal.SIGXCPU)
# Consume any output that might be ready.
events = sel.select(timeout=1)
for (sk, mask) in events:
if not sk.data.progress():
sel.unregister(sk.fd)
# Consume all remaining output.
while True:
events = sel.select(timeout=0)
for (sk, mask) in events:
if not sk.data.progress():
sel.unregister(sk.fd)
if not events:
break
if manifest.stdout is None:
stdout_writer.close()
os.close(stdout_pipe)
stderr_writer.close()
os.close(stderr_pipe)
runtime = time.perf_counter() - start
# Collect the status information.
status = None
sigcode = None
if child.returncode < 0: # Killed by a signal?
sigcode = signal.Signals(-child.returncode).name
else:
status = child.returncode
did_timeout = manifest.timeout is not None and runtime > manifest.timeout
# Create the status file to signal that we are finished.
status_dict = {'timeout': did_timeout, 'walltime': runtime,
'status': status, 'signal': sigcode}
with open(manifest.output_file_path('status.tmp'), "w") as f:
yaml.dump(status_dict, f)
os.rename(manifest.output_file_path('status.tmp'), manifest.output_file_path('status'))
def __iter__(self):
os.set_blocking(self.stdout.fileno(), False)
return self
def __init__(
self,
dag_directory: Union[str, "pathlib.Path"],
max_runs: int,
processor_factory: Callable[[str, List[CallbackRequest]], AbstractDagFileProcessorProcess],
processor_timeout: timedelta,
signal_conn: MultiprocessingConnection,
dag_ids: Optional[List[str]],
pickle_dags: bool,
async_mode: bool = True,
):
super().__init__()
self._file_paths: List[str] = []
self._file_path_queue: List[str] = []
self._dag_directory = dag_directory
self._max_runs = max_runs
self._processor_factory = processor_factory
self._signal_conn = signal_conn
self._pickle_dags = pickle_dags
self._dag_ids = dag_ids
self._async_mode = async_mode
self._parsing_start_time: Optional[int] = None
# Set the signal conn in to non-blocking mode, so that attempting to
# send when the buffer is full errors, rather than hangs for-ever
# attempting to send (this is to avoid deadlocks!)
#
# Don't do this in sync_mode, as we _need_ the DagParsingStat sent to
# continue the scheduler
if self._async_mode:
os.set_blocking(self._signal_conn.fileno(), False)
self._parallelism = conf.getint('scheduler', 'parsing_processes')
if 'sqlite' in conf.get('core', 'sql_alchemy_conn') and self._parallelism > 1:
self.log.warning(
"Because we cannot use more than 1 thread (parsing_processes = "
"%d ) when using sqlite. So we set parallelism to 1.",
self._parallelism,
)
self._parallelism = 1
# Parse and schedule each file no faster than this interval.
self._file_process_interval = conf.getint('scheduler', 'min_file_process_interval')
# How often to print out DAG file processing stats to the log. Default to
# 30 seconds.
self.print_stats_interval = conf.getint('scheduler', 'print_stats_interval')
# How many seconds do we wait for tasks to heartbeat before mark them as zombies.
self._zombie_threshold_secs = conf.getint('scheduler', 'scheduler_zombie_task_threshold')
# Should store dag file source in a database?
self.store_dag_code = STORE_DAG_CODE
# Map from file path to the processor
self._processors: Dict[str, AbstractDagFileProcessorProcess] = {}
self._num_run = 0
# Map from file path to stats about the file
self._file_stats: Dict[str, DagFileStat] = {}
self._last_zombie_query_time = None
# Last time that the DAG dir was traversed to look for files
self.last_dag_dir_refresh_time = timezone.make_aware(datetime.fromtimestamp(0))
# Last time stats were printed
self.last_stat_print_time = 0
# TODO: Remove magic number
self._zombie_query_interval = 10
# How long to wait before timing out a process to parse a DAG file
self._processor_timeout = processor_timeout
# How often to scan the DAGs directory for new files. Default to 5 minutes.
self.dag_dir_list_interval = conf.getint('scheduler', 'dag_dir_list_interval')
# Mapping file name and callbacks requests
self._callback_to_execute: Dict[str, List[CallbackRequest]] = defaultdict(list)
self._log = logging.getLogger('airflow.processor_manager')
self.waitables: Dict[Any, Union[MultiprocessingConnection, AbstractDagFileProcessorProcess]] = {
self._signal_conn: self._signal_conn,
}
from pytz import timezone
from time import strftime, sleep
from base64 import b64encode
import fitbit
from fitbit.api import Fitbit
from oauthlib.oauth2.rfc6749.errors import MismatchingStateError, MissingTokenError
import matplotlib
matplotlib.use('Agg')
import pylab as pl
#from garmin_app import garmin_utils, garmin_parse, garmin_report
#from garmin_app.util import utc, est
os.set_blocking(0, True)
utc = timezone('UTC')
est = timezone(strftime("%Z").replace('CST', 'CST6CDT').replace('EDT', 'EST5EDT'))
def read_config_env():
with open('config.env', 'r') as f:
for l in f:
(key, val) = l.strip().split('=')[:2]
os.environ[key] = val
read_config_env()
client_id = os.environ['FITBIT_CLIENTID']
args = parse_args()
done = False
signal.signal(signal.SIGALRM, handler)
signal.signal(signal.SIGINT, handler)
s = socket.socket(socket.AF_PACKET, socket.SOCK_RAW, socket.htons(3))
if_name = args.interface.encode('utf-8')
ifr = if_name + b'\x00' * (32 - len(if_name))
ifs = fcntl.ioctl(s, 0x8921, ifr)
mtu = int.from_bytes(ifs[16:18], byteorder='little')
MAXLINE = mtu if args.interface != 'lo' else 2**16
r, w = os.pipe()
os.set_blocking(w, False)
signal.set_wakeup_fd(w)
s.bind((args.interface, 0))
if args.timeout:
signal.alarm(args.timeout)
f = None
if args.output:
f = open(args.output, 'wb')
f.write(shb.build(SHB) + idb.build(IDB))
packets = 0
while not done:
def start(self):
raven_command = [self.raven_path] + self.raven_args
self.raven_proc = subprocess.Popen(raven_command,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE)
os.set_blocking(self.raven_proc.stdout.fileno(), False)
def sshUtil(self, command):
"""
Run the given command on the cluster.
Raise subprocess.CalledProcessError if it fails.
"""
cmd = [
'toil', 'ssh-cluster', '--insecure', '-p=aws', '-z', self.zone,
self.clusterName
] + command
log.info("Running %s.", str(cmd))
p = subprocess.Popen(cmd,
stderr=subprocess.PIPE,
stdout=subprocess.PIPE)
# Put in non-blocking mode. See https://stackoverflow.com/a/59291466
os.set_blocking(p.stdout.fileno(), False)
os.set_blocking(p.stderr.fileno(), False)
out_buffer = b''
err_buffer = b''
loops_since_line = 0
running = True
while running:
# While the process is running, see if it stopped
running = (p.poll() is None)
# Also collect its output
out_data = p.stdout.read()
if out_data:
out_buffer += out_data
while out_buffer.find(b'\n') != -1:
# And log every full line
cut = out_buffer.find(b'\n')
log.info('STDOUT: %s',
out_buffer[0:cut].decode('utf-8', errors='ignore'))
loops_since_line = 0
out_buffer = out_buffer[cut + 1:]
# Same for the error
err_data = p.stderr.read()
if err_data:
err_buffer += err_data
while err_buffer.find(b'\n') != -1:
cut = err_buffer.find(b'\n')
log.info('STDERR: %s',
err_buffer[0:cut].decode('utf-8', errors='ignore'))
loops_since_line = 0
err_buffer = err_buffer[cut + 1:]
loops_since_line += 1
if loops_since_line > 60:
log.debug('...waiting...')
loops_since_line = 0
time.sleep(1)
# At the end, log the last lines
if out_buffer:
log.info('STDOUT: %s', out_buffer.decode('utf-8', errors='ignore'))
if err_buffer:
log.info('STDERR: %s', err_buffer.decode('utf-8', errors='ignore'))
if p.returncode != 0:
# It failed
log.error("Failed to run %s.", str(cmd))
raise subprocess.CalledProcessError(p.returncode, ' '.join(cmd))
def _set_nonblocking(fd):
os.set_blocking(fd, False)
threading.Thread(target=sleep_then_write, daemon=True).start()
assert os.read(r1, 1) == b"c"
print("r1 definitely seems to be in blocking mode")
except Exception as exc:
print("ERROR: {!r}".format(exc))
print("-- testing anonymous pipe --")
check_reopen(*os.pipe())
print("-- testing FIFO --")
with tempfile.TemporaryDirectory() as tmpdir:
fifo = tmpdir + "/" + "myfifo"
os.mkfifo(fifo)
# "A process can open a FIFO in nonblocking mode. In this case, opening
# for read-only will succeed even if no-one has opened on the write side
# yet and opening for write-only will fail with ENXIO (no such device or
# address) unless the other end has already been opened." -- Linux fifo(7)
r = os.open(fifo, os.O_RDONLY | os.O_NONBLOCK)
assert not os.get_blocking(r)
os.set_blocking(r, True)
assert os.get_blocking(r)
w = os.open(fifo, os.O_WRONLY)
check_reopen(r, w)
print("-- testing socketpair --")
import socket
rs, ws = socket.socketpair()
check_reopen(rs.fileno(), ws.fileno())
async def monrun(kernel):
stdin = Stream(sys.stdin.buffer.raw)
try:
print('\nCurio Monitor: %d tasks running' % len(kernel._tasks))
print('Type help for commands')
while True:
print('curio > ', end='', flush=True)
resp = await stdin.readline()
if not resp or resp.startswith(b'q'):
print('Leaving monitor')
return
elif resp.startswith(b'p'):
ps(kernel)
elif resp.startswith(b'exit'):
raise SystemExit()
elif resp.startswith(b'cancel'):
try:
_, taskid_s = resp.split()
taskid = int(taskid_s)
if taskid in kernel._tasks:
print('Cancelling task', taskid)
await kernel._tasks[taskid].cancel()
else:
print('Bad task id')
except Exception as e:
print('Bad command')
elif resp.startswith(b'signal'):
try:
_, signame = resp.split()
signame = signame.decode('ascii').strip()
if hasattr(signal, signame):
os.kill(os.getpid(), getattr(signal, signame))
except Exception as e:
print('Bad command',e )
elif resp.startswith(b'w'):
try:
_, taskid_s = resp.split()
taskid = int(taskid_s)
if taskid in kernel._tasks:
print_stack(kernel._tasks[taskid])
print()
except Exception as e:
print('Bad command', e)
elif resp.startswith(b'h'):
print(
'''Commands:
ps : Show task table
where taskid : Show stack frames for a task
cancel taskid : Cancel a task
signal signame : Send a Unix signal
exit : Raise SystemExit and terminate
quit : Leave the monitor
''')
elif resp == b'\n':
pass
else:
print('Unknown command. Type help.')
finally:
os.set_blocking(stdin.fileno(), True)
def nonblocking(fd):
os.set_blocking(fd, False)
try:
yield
finally:
os.set_blocking(fd, True)
def main(listener_fd, alive_r, preload, main_path=None, sys_path=None):
'''Run forkserver.'''
if preload:
if '__main__' in preload and main_path is not None:
process.current_process()._inheriting = True
try:
spawn.import_main_path(main_path)
finally:
del process.current_process()._inheriting
for modname in preload:
try:
__import__(modname)
except ImportError:
pass
util._close_stdin()
sig_r, sig_w = os.pipe()
os.set_blocking(sig_r, False)
os.set_blocking(sig_w, False)
def sigchld_handler(*_unused):
# Dummy signal handler, doesn't do anything
pass
# letting SIGINT through avoids KeyboardInterrupt tracebacks
# unblocking SIGCHLD allows the wakeup fd to notify our event loop
handlers = {
signal.SIGCHLD: sigchld_handler,
signal.SIGINT: signal.SIG_DFL,
}
old_handlers = {sig: signal.signal(sig, val)
for (sig, val) in handlers.items()}
# calling os.write() in the Python signal handler is racy
signal.set_wakeup_fd(sig_w)
# map child pids to client fds
pid_to_fd = {}
with socket.socket(socket.AF_UNIX, fileno=listener_fd) as listener, \
selectors.DefaultSelector() as selector:
_forkserver._forkserver_address = listener.getsockname()
selector.register(listener, selectors.EVENT_READ)
selector.register(alive_r, selectors.EVENT_READ)
selector.register(sig_r, selectors.EVENT_READ)
while True:
try:
while True:
rfds = [key.fileobj for (key, events) in selector.select()]
if rfds:
break
if alive_r in rfds:
# EOF because no more client processes left
assert os.read(alive_r, 1) == b''
raise SystemExit
if sig_r in rfds:
# Got SIGCHLD
os.read(sig_r, 65536) # exhaust
while True:
# Scan for child processes
try:
pid, sts = os.waitpid(-1, os.WNOHANG)
except ChildProcessError:
break
if pid == 0:
break
child_w = pid_to_fd.pop(pid, None)
if child_w is not None:
if os.WIFSIGNALED(sts):
returncode = -os.WTERMSIG(sts)
else:
assert os.WIFEXITED(sts)
returncode = os.WEXITSTATUS(sts)
# Send exit code to client process
try:
write_signed(child_w, returncode)
except BrokenPipeError:
# client vanished
pass
os.close(child_w)
else:
# This shouldn't happen really
warnings.warn('forkserver: waitpid returned '
'unexpected pid %d' % pid)
if listener in rfds:
# Incoming fork request
with listener.accept()[0] as s:
# Receive fds from client
fds = reduction.recvfds(s, MAXFDS_TO_SEND + 1)
assert len(fds) <= MAXFDS_TO_SEND
child_r, child_w, *fds = fds
s.close()
pid = os.fork()
if pid == 0:
# Child
code = 1
try:
listener.close()
code = _serve_one(child_r, fds,
(alive_r, child_w, sig_r, sig_w),
old_handlers)
except Exception:
sys.excepthook(*sys.exc_info())
sys.stderr.flush()
finally:
os._exit(code)
else:
# Send pid to client process
try:
write_signed(child_w, pid)
except BrokenPipeError:
# client vanished
pass
pid_to_fd[pid] = child_w
os.close(child_r)
for fd in fds:
os.close(fd)
except OSError as e:
if e.errno != errno.ECONNABORTED:
raise
from pytz import timezone
from time import strftime, sleep
from base64 import b64encode
import fitbit
from fitbit.api import Fitbit
import matplotlib
matplotlib.use('Agg')
import pylab as pl
#from garmin_app import garmin_utils, garmin_parse, garmin_report
#from garmin_app.util import utc, est
os.set_blocking(0, True)
utc = timezone('UTC')
est = timezone(
strftime("%Z").replace('CST', 'CST6CDT').replace('EDT', 'EST5EDT'))
hostname = socket.gethostname()
HOME = os.environ['HOME']
DEFAULT_HOST = 'www.ddboline.net' if hostname == 'dilepton-tower' else 'cloud.ddboline.net'
def read_config_env():
with open('config.env', 'r') as f:
for l in f:
(key, val) = l.strip().split('=')[:2]
os.environ[key] = val
def _set_nonblocking(fd):
os.set_blocking(fd, False)
message = "001000emptyB#"
rpath = "000001"
wpath = "001000"
atexit.register(stopSch)
sched = Scheduler()
sched.start()
sched.add_interval_job(sendMessage, seconds=.1)
fd = os.open(
rpath, os.O_RDONLY) #C type open a file and return an int file descriptor
os.set_blocking(
fd, False
) # setting the reader to NON_BLOCKING so if it reads from an empty pipe it does not yield until it receives data (continues its own code)
while True:
doneReading = False
readSomething = False
string = ""
if fd >= 0: #check for error on opening the file
i = 0
while not doneReading:
try: #using a try catch block for if the reader opens the fifo before the writer and trys reading, the program wont crash due to an unneccessary error
buffer = os.read(
fd, 1
) #C type read of read from int file descriptor and read a number of bytes i.e 1
except OSError as err:
if err.errno == os.errno.EAGAIN or err.errno == os.errno.EWOULDBLOCK: #If one of these errors is detected treat it as nothing and reset our read variable to nothing