def run(self, *args, **options):
results = []
with bonobo.parse_args(options) as options:
services = self.get_services()
strategy = self.get_strategy()
graph_coll = self.get_graph(*args, **options)
if not isinstance(graph_coll, GeneratorType):
graph_coll = (graph_coll, )
for i, graph in enumerate(graph_coll):
if not isinstance(graph, bonobo.Graph):
raise ValueError(
"Expected a Graph instance, got {!r}.".format(graph))
print(
term.lightwhite("{}. {}".format(
i + 1, graph.name or repr(graph).strip("<>"))))
result = bonobo.run(graph,
services=services,
strategy=strategy)
results.append(result)
for node in result.nodes:
print(node.get_statistics_as_string(),
node.get_flags_as_string())
print(term.lightblack(" ... return value: " + str(result)))
return results
def main(setting, ckan_portal, dataset_id, ressource, namespace, filename):
parser = bonobo.get_argument_parser()
with bonobo.parse_args(parser) as options:
bonobo.run(
get_graph(ckan_portal, dataset_id, ressource, namespace, filename, **options),
services=get_services(setting, **options)
)
def handle(self, *args, **options):
_stdout_backup, _stderr_backup = self.stdout, self.stderr
self.stdout = OutputWrapper(ConsoleOutputPlugin._stdout,
ending=CLEAR_EOL + '\n')
self.stderr = OutputWrapper(ConsoleOutputPlugin._stderr,
ending=CLEAR_EOL + '\n')
self.stderr.style_func = lambda x: Fore.LIGHTRED_EX + Back.RED + '!' + Style.RESET_ALL + ' ' + x
with bonobo.parse_args(options) as options:
services = self.get_services()
graph_coll = self.get_graph(*args, **options)
if not isinstance(graph_coll, GeneratorType):
graph_coll = (graph_coll, )
for i, graph in enumerate(graph_coll):
assert isinstance(graph,
bonobo.Graph), 'Invalid graph provided.'
print(term.lightwhite('{}. {}'.format(i + 1, graph.name)))
result = bonobo.run(graph, services=services)
print(term.lightblack(' ... return value: ' + str(result)))
print()
self.stdout, self.stderr = _stdout_backup, _stderr_backup
def run_etl(input_file_path, config_file, output_cube_path=None, cube_config=None):
"""
Run ETl Process for passed excel file.
:param input_file_path: excel file path
:param config_file: config file path
example of config::
# in the config file you specify for each table, columns associate with it.
Facts: [Price, Quantity]
Accounts: ['Source Account', 'Destination Account']
Client: ['Client Activity', 'Client Role']
:param output_cube_path: cube folder path
:param cube_config: if you want to call run_etl as function, you can pass dict config directly as param,
there an example::
@click.command()
@click.pass_context
def myETL(ctx):
# demo run_etl as function with config as dict
config = {
'Facts': ['Amount', 'Count'],
'Geography': ['Continent', 'Country', 'City'],
'Product': ['Company', 'Article', 'Licence'],
'Date': ['Year', 'Quarter', 'Month', 'Day']
}
ctx.invoke(run_etl, input_file_path='sales.xlsx', cube_config=config, output_cube_path='cube2')
"""
parser = bonobo.get_argument_parser()
parser.add_argument("-in", "--input_file_path", help="Input file")
parser.add_argument("-cf", "--config_file", help="Configuration file path")
parser.add_argument("-out", "--output_cube_path", help="Cube export path")
with bonobo.parse_args(parser) as options:
if cube_config:
options["cube_config"] = cube_config
elif config_file:
with open(config_file) as config_file:
options["cube_config"] = yaml.load(config_file)
else:
raise Exception("Config file is not specified")
if input_file_path:
options["input_file_path"] = input_file_path
else:
raise Exception("Excel file is not specified")
if output_cube_path:
options["output_cube_path"] = output_cube_path
else:
options["output_cube_path"] = os.path.join(
expanduser("~"), "olapy-data", "cubes", Path(input_file_path).stem
)
bonobo.run(get_graph(**options), services=get_services(**options))
def execute_pipeline(self):
self.bonobo_parser = bonobo.get_argument_parser()
with bonobo.parse_args(self.bonobo_parser) as options:
bonobo.run(
self.build_graph(**options),
services=self.get_services(**options))
def main():
"""Execute the pipeline graph"""
# logfilename = "wh.log"
# logger = logging.getLogger()
# ch = logging.FileHandler(logfilename)
# formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
# ch.setFormatter(formatter)
# logger.addHandler(ch)
parser = bonobo.get_argument_parser()
with bonobo.parse_args(parser) as options:
bonobo.run(get_graph(**options), services=get_services(**options))
def run(get_graph, get_services, *, parser=None):
parser = parser or get_argument_parser()
with bonobo.parse_args(parser) as options:
with Timer() as timer:
print(
"Options:", " ".join("{}={}".format(k, v)
for k, v in sorted(options.items())))
retval = bonobo.run(get_graph(**get_graph_options(options)),
services=get_services(),
strategy=options["strategy"])
print("Execution time:", timer)
print("Return value:", retval)
print("XStatus:", retval.xstatus)
return retval.xstatus
def handle(self, *args, **options):
_stdout_backup, _stderr_backup = self.stdout, self.stderr
self.stdout = OutputWrapper(ConsoleOutputPlugin._stdout,
ending=CLEAR_EOL + '\n')
self.stderr = OutputWrapper(ConsoleOutputPlugin._stderr,
ending=CLEAR_EOL + '\n')
self.stderr.style_func = lambda x: Fore.LIGHTRED_EX + Back.RED + '!' + Style.RESET_ALL + ' ' + x
with bonobo.parse_args(options) as options:
result = bonobo.run(
self.get_graph(*args, **options),
services=self.get_services(),
)
self.stdout, self.stderr = _stdout_backup, _stderr_backup
return '\nReturn Value: ' + str(result)
def run(self, *args, **options):
results = []
with bonobo.parse_args(options) as options:
services = self.get_services()
strategy = self.get_strategy()
graph_coll = self.get_graph(*args, **options)
if not isinstance(graph_coll, GeneratorType):
graph_coll = (graph_coll, )
for i, graph in enumerate(graph_coll):
assert isinstance(graph, bonobo.Graph), 'Invalid graph provided.'
print(term.lightwhite('{}. {}'.format(i + 1, graph.name)))
result = bonobo.run(graph, services=services, strategy=strategy)
results.append(result)
print(term.lightblack(' ... return value: ' + str(result)))
print()
return results
def parse_args(parser=None):
parser = parser or bonobo.get_argument_parser()
parser.add_argument('--drop', '-D', action='store_true')
parser.add_argument('--create', '-C', action='store_true')
parser.add_argument('--echo', action='store_true')
with bonobo.parse_args(parser) as options:
import models
import settings
import services
if options['echo']:
logging.getLogger('sqlalchemy.engine').setLevel(logging.INFO)
if options['drop'] or options['create']:
root_engine = services.create_engine(superuser=True)
if options['drop']:
# drop database/role with super user privileges
_execute_sql(root_engine,
"DROP DATABASE {}".format(settings.DATABASE_NAME))
_execute_sql(root_engine,
"DROP ROLE {}".format(settings.DATABASE_USERNAME))
if options['create']:
# create database/role with super user privileges
_execute_sql(
root_engine,
'CREATE ROLE {} WITH LOGIN PASSWORD \'{}\';'.format(
settings.DATABASE_USERNAME,
settings.DATABASE_PASSWORD))
_execute_sql(
root_engine,
'CREATE DATABASE {} WITH OWNER={} TEMPLATE=template0 ENCODING="utf-8";'
.format(settings.DATABASE_NAME,
settings.DATABASE_USERNAME))
# create tables in userland
engine = services.create_engine()
models.metadata.create_all(engine)
yield options
def run(self, *args, **options):
results = []
with bonobo.parse_args(options) as options:
services = self.get_services()
strategy = self.get_strategy()
graph_coll = self.get_graph(*args, **options)
if not isinstance(graph_coll, GeneratorType):
graph_coll = (graph_coll,)
for i, graph in enumerate(graph_coll):
if not isinstance(graph, bonobo.Graph):
raise ValueError('Expected a Graph instance, got {!r}.'.format(graph))
print(term.lightwhite('{}. {}'.format(i + 1, graph.name)))
result = bonobo.run(graph, services=services, strategy=strategy)
results.append(result)
print(term.lightblack(' ... return value: ' + str(result)))
print()
return results
bonobo.CsvReader('datasets/coffeeshops.txt'),
*((bonobo.Limit(_limit), ) if _limit else ()),
*((bonobo.PrettyPrinter(), ) if _print else ()),
bonobo.CsvWriter('coffeeshops.csv', fs='fs.output')
)
if __name__ == '__main__':
parser = bonobo.get_argument_parser()
parser.add_argument(
'--limit',
'-l',
type=int,
default=None,
help='If set, limits the number of processed lines.'
)
parser.add_argument(
'--print',
'-p',
action='store_true',
default=False,
help='If set, pretty prints before writing to output file.'
)
with bonobo.parse_args(parser) as options:
bonobo.run(
get_graph(_limit=options['limit'], _print=options['print']),
services=get_services()
)
import bonobo
from bonobo.examples.types.strings import get_graph
if __name__ == "__main__":
parser = bonobo.get_argument_parser()
with bonobo.parse_args(parser):
bonobo.run(get_graph())
def run(job, argparser):
parser = get_argument_parser(argparser)
with bonobo.parse_args(parser) as options:
bonobo.run(get_graph(job, **get_graph_options(options)),
services=get_services(job))
def main(setting, query, namespace, filename):
parser = bonobo.get_argument_parser()
with bonobo.parse_args(parser) as options:
bonobo.run(get_graph(query, namespace, filename, **options),
services=get_services(setting, **options))
import bonobo
import datetime
import time
def extract():
"""Placeholder, change, rename, remove... """
for x in range(60):
if x:
time.sleep(1)
yield datetime.datetime.now()
def get_graph():
graph = bonobo.Graph()
graph.add_chain(
extract,
print,
)
return graph
if __name__ == '__main__':
parser = bonobo.get_argument_parser()
with bonobo.parse_args(parser):
bonobo.run(get_graph())
import bonobo
from bonobo import examples
from bonobo.examples.files._services import get_services
def skip_comments(line):
line = line.strip()
if not line.startswith('#'):
yield line
def get_graph(*, _limit=(), _print=()):
return bonobo.Graph(
bonobo.FileReader('datasets/passwd.txt'),
skip_comments,
*_limit,
lambda s: s.split(':')[0],
*_print,
bonobo.FileWriter('usernames.txt', fs='fs.output'),
)
if __name__ == '__main__':
parser = examples.get_argument_parser()
with bonobo.parse_args(parser) as options:
bonobo.run(get_graph(**examples.get_graph_options(options)), services=get_services())
def main():
"""Execute the pipeline graph
"""
parser = bonobo.get_argument_parser()
with bonobo.parse_args(parser) as options:
bonobo.run(get_graph(**options), services=get_services(**options))
