def test_run(capsys):
entrypoint(['run', '--quiet', get_examples_path('types/strings.py')])
out, err = capsys.readouterr()
out = out.split('\n')
assert out[0].startswith('Foo ')
assert out[1].startswith('Bar ')
assert out[2].startswith('Baz ')
def get_services():
from ._services import get_services
return {
**get_services(), 'fs':
TarFS(bonobo.get_examples_path('datasets/spam.tgz'))
}
def test_run_path(runner, capsys):
runner('run', '--quiet', get_examples_path('types'))
out, err = capsys.readouterr()
out = out.split('\n')
assert out[0].startswith('Foo ')
assert out[1].startswith('Bar ')
assert out[2].startswith('Baz ')
def test_run_with_env(runner, capsys):
runner('run', '--quiet', get_examples_path('env_vars/get_passed_env.py'),
'--env', 'ENV_TEST_NUMBER=123', '--env', 'ENV_TEST_USER=cwandrews',
'--env', "ENV_TEST_STRING='my_test_string'")
out, err = capsys.readouterr()
out = out.split('\n')
assert out[0] == 'cwandrews'
assert out[1] == '123'
assert out[2] == 'my_test_string'
def handle(self, *, path, **options):
if not path.startswith('examples'):
raise ValueError('Download command currently supports examples only')
examples_path = re.sub('^examples/', '', path)
output_path = bonobo.get_examples_path(examples_path)
with _open_url(EXAMPLES_BASE_URL + examples_path) as response, open(output_path, 'wb') as fout:
for chunk in response.iter_content(io.DEFAULT_BUFFER_SIZE):
fout.write(chunk)
self.logger.info('Download saved to {}'.format(output_path))
def handle(self, *, path, **options):
if not path.startswith('examples'):
raise ValueError(
'Download command currently supports examples only')
examples_path = re.sub('^examples/', '', path)
output_path = bonobo.get_examples_path(examples_path)
with _open_url(EXAMPLES_BASE_URL + examples_path) as response, open(
output_path, 'wb') as fout:
for chunk in response.iter_content(io.DEFAULT_BUFFER_SIZE):
fout.write(chunk)
self.logger.info('Download saved to {}'.format(output_path))
def runner_entrypoint(args):
"""Run bonobo using the python command entrypoint directly (bonobo.commands.entrypoint). """
return entrypoint(args)
@runner
def runner_module(args):
"""Run bonobo using the bonobo.__main__ file, which is equivalent as doing "python -m bonobo ..."."""
with patch.object(sys, "argv", ["bonobo", *args]):
return runpy.run_path(__main__.__file__, run_name="__main__")
all_runners = pytest.mark.parametrize("runner",
[runner_entrypoint, runner_module])
all_environ_targets = pytest.mark.parametrize(
"target", [(get_examples_path("environ.py"), ),
("-m", "bonobo.examples.environ")])
@all_runners
@all_environ_targets
class EnvironmentTestCase:
def run_quiet(self, runner, *args):
return runner("run", "--quiet", *args)
def run_environ(self, runner, *args, environ=None):
_environ = {"PATH": "/usr/bin"}
if environ:
_environ.update(environ)
with patch.dict("os.environ", _environ, clear=True):
def test_run(runner):
out, err = runner('run', '--quiet', get_examples_path('types/strings.py'))
out = out.split('\n')
assert out[0].startswith('Foo ')
assert out[1].startswith('Bar ')
assert out[2].startswith('Baz ')
def test_run(runner):
out, err = runner('run', '--quiet', get_examples_path('types/strings.py'))
out = out.split('\n')
assert out[0].startswith('Foo ')
assert out[1].startswith('Bar ')
assert out[2].startswith('Baz ')
def test_run(runner):
out, err = runner("run", "--quiet", get_examples_path("types/strings.py"))
out = out.split("\n")
assert out[0].startswith("Foo ")
assert out[1].startswith("Bar ")
assert out[2].startswith("Baz ")
from bonobo import FileReader, Graph, get_examples_path
def skip_comments(line):
if not line.startswith('#'):
yield line
graph = Graph(
FileReader(path=get_examples_path('datasets/passwd.txt')),
skip_comments,
lambda s: s.split(':'),
lambda l: l[0],
print,
)
if __name__ == '__main__':
import bonobo
bonobo.run(graph)
def get_services():
from ._services import get_services
return {
**get_services(), 'fs':
TarFS(bonobo.get_examples_path('datasets/spam.tgz'))
}
@runner
def runner_entrypoint(args):
""" Run bonobo using the python command entrypoint directly (bonobo.commands.entrypoint). """
return entrypoint(args)
@runner
def runner_module(args):
""" Run bonobo using the bonobo.__main__ file, which is equivalent as doing "python -m bonobo ..."."""
with patch.object(sys, 'argv', ['bonobo', *args]):
return runpy.run_path(__main__.__file__, run_name='__main__')
all_runners = pytest.mark.parametrize('runner', [runner_entrypoint, runner_module])
all_environ_targets = pytest.mark.parametrize(
'target', [(get_examples_path('environ.py'),), ('-m', 'bonobo.examples.environ')]
)
@all_runners
@all_environ_targets
class EnvironmentTestCase:
def run_quiet(self, runner, *args):
return runner('run', '--quiet', *args)
def run_environ(self, runner, *args, environ=None):
_environ = {'PATH': '/usr/bin'}
if environ:
_environ.update(environ)
with patch.dict('os.environ', _environ, clear=True):
def runner_entrypoint(args):
""" Run bonobo using the python command entrypoint directly (bonobo.commands.entrypoint). """
return entrypoint(args)
@runner
def runner_module(args):
""" Run bonobo using the bonobo.__main__ file, which is equivalent as doing "python -m bonobo ..."."""
with patch.object(sys, 'argv', ['bonobo', *args]):
return runpy.run_path(__main__.__file__, run_name='__main__')
all_runners = pytest.mark.parametrize('runner', [runner_entrypoint, runner_module])
all_environ_targets = pytest.mark.parametrize(
'target', [
(get_examples_path('environ.py'), ),
(
'-m',
'bonobo.examples.environ',
),
]
)
@all_runners
@all_environ_targets
class EnvironmentTestCase():
def run_quiet(self, runner, *args):
return runner('run', '--quiet', *args)
def run_environ(self, runner, *args, environ=None):
def get_services():
return {
'fs': open_fs(get_examples_path()),
'fs.output': open_fs(),
}
def get_services():
return {'fs': TarFS(bonobo.get_examples_path('datasets/spam.tgz'))}
def test_install_requirements_for_file(runner):
dirname = get_examples_path('types')
with patch('bonobo.commands.run._install_requirements') as install_mock:
runner('run', '--install', os.path.join(dirname, 'strings.py'))
install_mock.assert_called_once_with(
os.path.join(dirname, 'requirements.txt'))
def get_services():
return {
**examples.get_services(), "fs":
TarFS(bonobo.get_examples_path("datasets", "static", "spam.tgz"))
}
def test_install_requirements_for_file(runner):
dirname = get_examples_path("types")
with patch("bonobo.commands.run._install_requirements") as install_mock:
runner("run", "--install", os.path.join(dirname, "strings.py"))
install_mock.assert_called_once_with(
os.path.join(dirname, "requirements.txt"))
' '.join(
filter(None,
(row.get('postal_code', None), row.get('city', None)))),
row.get('county', None),
row.get('country'),
)))
print(' - {}: {address}'.format(t.blue('address'),
address=', '.join(address)))
print(' - {}: {links}'.format(t.blue('links'),
links=', '.join(row['links'])))
print(' - {}: {geometry}'.format(t.blue('geometry'), **row))
print(' - {}: {source}'.format(t.blue('source'),
source='datanova/' + API_DATASET))
graph = Graph(
OpenDataSoftAPI(dataset=API_DATASET,
netloc=API_NETLOC,
timezone='Europe/Paris'),
normalize,
filter_france,
Tee(display),
JsonWriter(path=get_examples_path('datasets/fablabs.txt')),
)
if __name__ == '__main__':
from bonobo import run
run(graph)
from bonobo import CsvReader, Graph, get_examples_path
graph = Graph(
CsvReader(path=get_examples_path('datasets/coffeeshops.txt')),
print,
)
if __name__ == '__main__':
import bonobo
bonobo.run(graph)
def test_install_requirements_for_file(runner):
dirname = get_examples_path('types')
with patch('bonobo.commands.run._install_requirements') as install_mock:
runner('run', '--install', os.path.join(dirname, 'strings.py'))
install_mock.assert_called_once_with(os.path.join(dirname, 'requirements.txt'))
def get_services():
return {'fs': open_fs(get_examples_path()), 'fs.output': open_fs()}
def get_services():
return {'fs': open_fs(get_examples_path())}
