def test_fd_leak():
base_count = count_fds()
with pipes():
print('ok')
assert count_fds() == base_count
for i in range(10):
with pipes():
print('ok')
assert count_fds() == base_count
async def dispatch(self, request: Request, call_next):
"""Add VSI stats in headers."""
rio_stream = StringIO()
logger = logging.getLogger("rasterio")
logger.setLevel(logging.DEBUG)
handler = logging.StreamHandler(rio_stream)
logger.addHandler(handler)
gdal_config = {"CPL_DEBUG": "ON", "CPL_CURL_VERBOSE": "TRUE"}
with pipes() as (_, curl_stream):
with rasterio.Env(**gdal_config, **self.config):
response = await call_next(request)
logger.removeHandler(handler)
handler.close()
if rio_stream or curl_stream:
rio_lines = rio_stream.getvalue().splitlines()
curl_lines = curl_stream.read().splitlines()
results = analyse_logs(rio_lines, curl_lines)
head_results = "head;count={count}".format(**results["HEAD"])
list_results = "list;count={count}".format(**results["LIST"])
get_results = "get;count={count};size={bytes}".format(
**results["GET"])
ranges_results = "ranges; values={}".format("|".join(
results["GET"]["ranges"]))
response.headers[
"VSI-Stats"] = f"{list_results}, {head_results}, {get_results}, {ranges_results}"
return response
def sys_pipes():
'''
We can use the Wurlitzer package to redirect stdout and stderr
from the command line into a Jupyter notebook. But if we're not
in a notebook, this isn't safe because we can't redirect stdout
to itself. This function is a thin wrapper that checks if the
stdout/err streams are TTYs and enables output redirection
based on that.
Basic usage is:
>>> with sys_pipes(): # doctest: +SKIP
... call_some_c_function()
See the Wurlitzer package for usage of `wurlitzer.pipes()`;
see also https://github.com/manodeep/Corrfunc/issues/157.
'''
kwargs = {'stdout':None if sys.stdout.isatty() else sys.stdout,
'stderr':None if sys.stderr.isatty() else sys.stderr }
# Redirection might break for any number of reasons, like
# stdout/err already being closed/redirected. We probably
# prefer not to crash in that case and instead continue
# without any redirection.
try:
with wurlitzer.pipes(**kwargs):
yield
except:
yield
def test_skips_when_reaches_continue(self, evaluator):
expr = f"""
a_list = [1, 2, 3 ,4, 5]
for item in a_list
if item == 3
print("skipped")
continue
print("never here")
end
print(item)
end
print("out")
"""
with pipes() as (out, _):
evaluator.evaluate(expr)
out = out.read()
out.should.contain('1')
out.should.contain('2')
out.should.contain('skipped')
out.should.contain('4')
out.should.contain('5')
out.should.contain('out')
out.should_not.contain('never here')
def test_generates_a_function(self, evaluator):
expr = f"""
class Object
end
class MyClass
def say_42()
return 42
end
end
mc = MyClass()
forty_two = mc.say_42()
print(forty_two)
"""
with pipes() as (out, _):
evaluator.evaluate(expr)
out = out.read()
out.should.contain('42')
def get_context():
""":class:`contextlib.GeneratorContextManager`: Silence all C-level stdout messages."""
with wurlitzer.pipes() as (stdout, stderr):
yield (stdout, stderr)
stdout.close()
stderr.close()
def _assertAwsCliReturns(args, calls):
t_code = None
from wurlitzer import pipes
try:
with pipes() as (out, err):
try:
with patch('builtins.input', return_value='') as mock:
exec_awscli(*args)
except SystemExit as e:
t_code = e.code
else:
# This should never happen...
raise Exception("exec_awscli: Failed to raise SystemExit!")
import sys
cmd = ' '.join(sys.argv)
t_out = out.read()
t_err = err.read()
# Must be closed explicitly otherwise we receive warnings
finally:
out.close()
err.close()
if calls:
mock.assert_has_calls(calls)
assert len(mock.call_args_list) == len(calls)
return t_out, t_err, t_code, cmd
def test_pipes():
with pipes(stdout=PIPE, stderr=PIPE) as (stdout, stderr):
printf(u"Hellø")
printf_err(u"Hi, stdérr")
assert stdout.read() == u"Hellø\n"
assert stderr.read() == u"Hi, stdérr\n"
def test_pipe_bytes():
with pipes(encoding=None) as (stdout, stderr):
printf(u"Hellø")
printf_err(u"Hi, stdérr")
assert stdout.read() == u"Hellø\n".encode('utf8')
assert stderr.read() == u"Hi, stdérr\n".encode('utf8')
def test_pipe_bytes():
with pipes(encoding=None) as (stdout, stderr):
printf(u"Hellø")
printf_err(u"Hi, stdérr")
assert stdout.read() == u"Hellø\n".encode('utf8')
assert stderr.read() == u"Hi, stdérr\n".encode('utf8')
def _dispatch(self, opcode, index=0, value=0, ptr=None, opt=0.):
if ptr is None:
ptr = c_void_p()
with pipes() if not self.verbose else contextlib.suppress():
output = self._effect.dispatcher(byref(self._effect), c_int32(opcode), c_int32(index),
vst_int_ptr(value), ptr, c_float(opt))
return output
async def compile_c(code):
string = code.encode('utf-8')
with wurlitzer.pipes() as (out, err):
result = mcu_fn.mcu_load_c(string)
data = out.read()
if data:
await web_console(data)
def instantiate(config, weights, metadata):
'''Load darknet
Inputs
------
config: str
path to config filename
weights: str
path to weights file
metadata: str
path to metadata
Outputs
-------
network: pointer to loaded network
metadata: pointer to loaded metadata
'''
logger.info('Loading network and metadata')
with pipes() as (out, err):
net = load_network(config.encode('ascii'),
weights.encode('ascii'), 0)
meta = load_metadata(metadata.encode('ascii'))
logger.info(err.read())
return net, meta
def _cdt_exists(cdt_meta_node, channel_index):
import conda_build.api
try:
f = io.StringIO()
with redirect_stdout(f), redirect_stderr(f), pipes(stdout=f, stderr=f):
metas = conda_build.api.render(
cdt_meta_node["recipe_path"],
variant_config_files=["conda_build_config.yaml"],
bypass_env_check=True,
)
except Exception as e:
print(f.getvalue())
raise e
dist_fnames = [
path for m, _, _ in metas
for path in conda_build.api.get_output_file_paths(m) if not m.skip()
]
on_channel = True
for dist_fname in dist_fnames:
fname = os.path.basename(dist_fname)
on_channel &= (fname in channel_index)
return on_channel
def main(args):
device = torch.device(args.device)
with pipes():
loaders, in_channels, out_classes = get_dataset(
args.dataset, args.data_root, args.batch_size)
model_class = get_arch(args.arch)
warp_class = get_warp(args.warp)
model = model_class(in_channels, out_classes, warp_class)
model.to(device)
optimizer = Adam(model.parameters())
for epoch in range(args.num_epochs):
t_accs = []
v_accs = []
for is_train, (x, y) in each_batch(loaders, device, args.tqdm):
if is_train:
model.train()
optimizer.zero_grad()
logits = model(x)
loss = F.cross_entropy(logits, y)
loss.backward()
optimizer.step()
else:
model.eval()
with torch.no_grad():
logits = model(x)
acc = (logits.argmax(1) == y).type(torch.float32).mean()
accs = t_accs if is_train else v_accs
accs.append(acc)
train_acc = 100 * sum(t_accs) / len(t_accs)
val_acc = 100 * sum(v_accs) / len(v_accs)
print('%6d %5.1f %5.1f' % (epoch, train_acc, val_acc))
def _compiles_ok(codestring):
from distutils.ccompiler import new_compiler
from distutils.sysconfig import customize_compiler
from distutils.errors import CompileError
with TemporaryDirectory() as folder:
source_path = os.path.join(folder, 'compiler_test_source.cpp')
with open(source_path, 'wt') as ofh:
ofh.write(codestring)
compiler = new_compiler()
customize_compiler(compiler)
out = ''
try:
if pipes is None:
compiler.compile([source_path])
else:
with pipes() as out_err:
compiler.compile([source_path])
out = '\n'.join([p.read() for p in out_err])
except CompileError:
_ok = False
except Exception:
_ok = False
_warn("Failed test compilation of '%s':\n %s" % (codestring, out))
else:
_ok = True
return _ok, out
def _make_temporary_build(package):
working_directory = os.path.dirname(package.filepath)
system = build_system.create_build_system(working_directory,
package=package,
verbose=True)
# create and execute build process
builder = build_process_.create_build_process(
"local", # See :func:`rez.build_process_.get_build_process_types` for possible values
working_directory,
build_system=system,
verbose=True,
)
install_path = tempfile.mkdtemp(suffix="package_{package.name}".format(
package=package))
_LOGGER.debug('Building to "%s" path.', install_path)
with wurlitzer.pipes() as (stdout, stderr):
builder.build(clean=True, install=True, install_path=install_path)
stdout.close()
stderr.close()
return install_path
def test_pipes():
with pipes(stdout=PIPE, stderr=PIPE) as (stdout, stderr):
printf(u"Hellø")
printf_err(u"Hi, stdérr")
assert stdout.read() == u"Hellø\n"
assert stderr.read() == u"Hi, stdérr\n"
def test_buffer_full():
with pipes(stdout=None, stderr=io.StringIO()) as (stdout, stderr):
long_string = "x" * 1000000 # create a very long string
printf_err(long_string)
# Test never reaches here as the process hangs.
assert stderr.getvalue() == long_string + "\n"
def assertAwsCliReturns(self, *args, stdout='', stderr='', code=0):
"""Runs awscli and tests the output.
This test runs the awscli command with `*args`. It tests
that the exit code and stdout and stderr match the given
expect values.
Example:
`self.assertAwsCliReturns('logout')` is equivalent to running:
$ aws logout
The test in this example will pass if `aws logout` returns
exit code 0 and emits no output to stdout or stderr.
Otherwise an AssertionError is raised.
Args:
*args (str): Arguments to be passed to the AWS command
line utility.
stdout (str): Expected output to stdout.
stderr (str): Expected output to stderr.
code (int): Expected return code.
Raises:
AssertionError
"""
try:
with pipes() as (out, err):
with self.assertRaises(SystemExit) as e:
exec_awscli(*args)
import sys
cmd = ' '.join(sys.argv)
mesg = "Error: ran '%s', on %s expected output: %s"
test_out = out.read()
test_err = err.read()
self.assertEqual(
test_out,
stdout,
mesg % (cmd, 'stdout', stdout)
)
self.assertEqual(
test_err,
stderr,
mesg % (cmd, 'stderr', stderr)
)
self.assertEqual(
e.exception.code,
code,
"Error: '%s' returned %d, expected: %d" %
(cmd, e.exception.code, code)
)
# Must be closed explicitly otherwise we receive warnings
finally:
out.close()
err.close()
async def interrupt_mcu(vect):
await log('Received interrupt (' + str(vect) + ')\n')
with wurlitzer.pipes() as (out, err):
mcu_fn.mcu_send_interrupt(vect)
data = out.read()
await log('MCU interrupted\n')
if data:
await web_console(data)
def _execute_async(self, code):
try:
with pipes(stdout=_PseudoStream(partial(self.Print, end="")),
stderr=_PseudoStream(partial(self.Error, end=""))):
future = self._matlab.eval(code, **self._async_kwargs)
future.result()
except (SyntaxError, MatlabExecutionError, KeyboardInterrupt) as exc:
pass
def test_works_with_expressions(self, evaluator):
expr = f"""
if 1 + 2 * 3
print("seven")
end
if 1 - 1
print("weird")
else
print("zero")
end
if 1.3 + 3.7
print("5.0")
end
if 2 > 1
print("2 gt 1")
end
if 3 < 4
print("3 lt 4")
end
if 3 == 3
print("3 eq 3")
end
if 4 != 4
print("4 neq 4")
else
print("4 eq 4")
end
if 2 + 3 == 5
print("2 + 3 == 5")
end
if (2 + 3) * 4 - 2 == 9 * 2
print("eighteen")
end
"""
with pipes() as (out, _):
evaluator.evaluate(expr)
out = out.read()
out.should.contain('seven')
out.should.contain('zero')
out.should.contain('5.0')
out.should.contain('2 gt 1')
out.should.contain('3 lt 4')
out.should.contain('3 eq 3')
out.should.contain('4 eq 4')
out.should.contain('2 + 3 == 5')
out.should.contain('eighteen')
def extract(self, sess, filename):
symbol = {};
print ('[*] Extracting Feature From Video => {}'.format(filename))
symbol['raw'] = tf.placeholder(tf.float32, shape = [None, self.height, self.width, 3], name = 'input')
stdout = sys.stdout
sys.stdout = open(os.devnull, 'w')
#vggnet = vgg16(symbol['raw'], 'vgg16_weights.npz', sess)
sys.stdout = stdout
capture = cv2.VideoCapture(filename)
self.features = None
fidx = -1
self.max_nframes = 0
with pipes() as (out, err):
while True:
ret, frame = capture.read()
fidx += 1
#if fidx % self.skip != 0: # skip
#continue
if frame is None or ret is None:
break
resized = cv2.resize(frame, (self.width, self.height)).reshape([1, self.width, self.height, 3])
self.max_nframes += 1
feature = sess.run(vggnet.pool5, feed_dict = {symbol['raw'] : resized})
if self.features is None:
self.features = feature
else:
self.features = np.vstack((self.features, feature))
self.features = self.features.reshape([-1, self.max_nframes, 25088])
self.nframes = [self.max_nframes]
def _execute_async(self, code):
try:
with pipes(stdout=_PseudoStream(partial(self.Print, end="")),
stderr=_PseudoStream(partial(self.Error, end=""))):
future = self._matlab.eval(code, nargout=0, async=True)
future.result()
except (SyntaxError, MatlabExecutionError, KeyboardInterrupt) as exc:
stdout = exc.args[0]
return ExceptionWrapper("Error", -1, stdout)
async def do_test(data):
global mcu
print('Test data = ', data)
string = 'This is a test'.encode('utf-8')
with wurlitzer.pipes() as (out, err):
mcu_fn.mcu_load_asm(string)
data = out.read()
if data:
await web_console(data)
def getCutout(
self,
data_set='isotropic1024coarse',
field='u',
time_step=int(0),
start=np.array([1, 1, 1], dtype=np.int),
end=np.array([8, 8, 8], dtype=np.int),
step=np.array([1, 1, 1], dtype=np.int),
filter_width=1):
if not self.connection_on:
print('you didn\'t connect to the database')
sys.exit()
time_step=int(time_step)
if field in ['u', 'a', 'b']:
result_dim = 3
elif field in ['p', 'd', 't']:
result_dim = 1
else:
print(('wrong result type requested in getCutout\n'
+ 'maybe it\'s just missing from the list?'))
sys.exit()
return None
tempa=np.arange(start[0], end[0]+1, step[0])
tempb=np.arange(start[1], end[1]+1, step[1])
tempc=np.arange(start[2], end[2]+1, step[2])
real_size=np.array([np.size(tempa), np.size(tempb), np.size(tempc)], dtype=np.int)
getFunction = 'getCutout'
get_data = getattr(self.lib, getFunction)
result_array = np.empty(tuple(list(real_size[::-1]) + [result_dim]), dtype=np.float32)
stdout = io.StringIO()
stderr = io.StringIO()
with pipes(stdout=stdout, stderr=stderr):
get_data(self.authToken,
ctypes.c_char_p(data_set.encode('ascii')),
ctypes.c_char_p(field.encode('ascii')),
ctypes.c_int(time_step),
ctypes.c_int(start[0]),
ctypes.c_int(start[1]),
ctypes.c_int(start[2]),
ctypes.c_int(end[0]),
ctypes.c_int(end[1]),
ctypes.c_int(end[2]),
ctypes.c_int(step[0]),
ctypes.c_int(step[1]),
ctypes.c_int(step[2]),
ctypes.c_int(filter_width),
result_array.ctypes.data_as(ctypes.POINTER(ctypes.c_char)))
if (len(stderr.getvalue())>0):
print(stderr.getvalue(), file=sys.stderr)
return None
return result_array
def _run_quartus_docker(self, quartus_command):
logger.info(f'Running {quartus_command}...')
cmd = f"docker run " \
f"-u `id -u` " \
f"-v /sys:/sys:ro " \
f"-v {self.project_path}:/simulation " \
f"gasparka/quartus {quartus_command}"
with pipes(stdout=sys.stdout if self.verbose else None, stderr=sys.stderr):
subprocess.run(cmd, shell=True)
def __init__(self, ids, vectors, post=0, m=16):
super().__init__(ids, vectors)
with pipes(stdout=None):
self.index = nmslib.init(method='hnsw', space='cosinesimil')
self.index.addDataPointBatch(vectors)
self.index.createIndex({
'post': post,
'M': m
},
print_progress=False)
def test_pipes_stderr():
stdout = io.StringIO()
with pipes(stdout=stdout, stderr=STDOUT) as (_stdout, _stderr):
printf(u"Hellø")
libc.fflush(c_stdout_p)
printf_err(u"Hi, stdérr")
assert _stdout is stdout
assert _stderr is None
assert stdout.getvalue() == u"Hellø\nHi, stdérr\n"
def test_forward():
stdout = io.StringIO()
stderr = io.StringIO()
with pipes(stdout=stdout, stderr=stderr) as (_stdout, _stderr):
printf(u"Hellø")
printf_err(u"Hi, stdérr")
assert _stdout is stdout
assert _stderr is stderr
assert stdout.getvalue() == u"Hellø\n"
assert stderr.getvalue() == u"Hi, stdérr\n"
def test_items_can_be_printed(self, evaluator):
expr = f"""
print([11, 22, 33][1])
"""
with pipes() as (out, _):
evaluator.evaluate(expr)
out = out.read()
out.should.contain('22')
def test_pipes_stderr():
stdout = io.StringIO()
with pipes(stdout=stdout, stderr=STDOUT) as (_stdout, _stderr):
printf(u"Hellø")
libc.fflush(c_stdout_p)
time.sleep(0.1)
printf_err(u"Hi, stdérr")
assert _stdout is stdout
assert _stderr is None
assert stdout.getvalue() == u"Hellø\nHi, stdérr\n"
def test_forward():
stdout = io.StringIO()
stderr = io.StringIO()
with pipes(stdout=stdout, stderr=stderr) as (_stdout, _stderr):
printf(u"Hellø")
printf_err(u"Hi, stdérr")
assert _stdout is stdout
assert _stderr is stderr
assert stdout.getvalue() == u"Hellø\n"
assert stderr.getvalue() == u"Hi, stdérr\n"
with_strides = ("contiguous" not in form or not form["contiguous"])
array_struct = build_array_struct(prop, value, with_strides)
args.append(array_struct)
else:
args.append(value)
simple_result = True
if result_schema["type"] == "object":
simple_result = False
result_arg = build_result_struct(result_schema)
args.append(result_arg)
elif result_schema["type"] == "array":
simple_result = False
raise NotImplementedError
def run():
if simple_result:
return binary_module.lib.transform(*args)
else:
binary_module.lib.transform(*args)
return unpack_result_struct(args[-1], result_schema)
if wurlitzer is not None:
with wurlitzer.pipes() as (stdout, stderr):
translator_result_ = run()
sys.stderr.write(stderr.read())
sys.stdout.write(stdout.read())
else:
translator_result_ = run()
ARRAYS.clear()